From 5acf2a3de73ff3bc23494ff7cbb6b58385a585d5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 13 Mar 2020 05:52:51 +0000 Subject: [PATCH 0001/1498] Bump pytest from 5.3.5 to 5.4.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.3.5 to 5.4.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.3.5...5.4.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f83f84ea1e..38d37bd487 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ pylint==2.4.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.6 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in -pytest==5.3.5 # via -r test-requirements.in, pytest-cov +pytest==5.4.0 # via -r test-requirements.in, pytest-cov six==1.14.0 # via astroid, cryptography, packaging, prompt-toolkit, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in From 58bfacc327a0428514046078d351ab4b18117902 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2020 06:13:33 +0000 Subject: [PATCH 0002/1498] Bump pytest from 5.4.0 to 5.4.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.0 to 5.4.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.0...5.4.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 38d37bd487..b4245236be 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ pylint==2.4.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.6 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in -pytest==5.4.0 # via -r test-requirements.in, pytest-cov +pytest==5.4.1 # via -r test-requirements.in, pytest-cov six==1.14.0 # via astroid, cryptography, packaging, prompt-toolkit, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in From d362eabdb9fb302036a45116069a61cef400ef84 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2020 05:57:35 +0000 Subject: [PATCH 0003/1498] Bump coverage from 5.0.3 to 5.0.4 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.0.3 to 5.0.4. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.0.3...coverage-5.0.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b4245236be..0191ea0f5f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,7 @@ async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.1.0 # via ipython cffi==1.14.0 # via cryptography -coverage==5.0.3 # via pytest-cov +coverage==5.0.4 # via pytest-cov cryptography==2.8 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets entrypoints==0.3 # via flake8 From fc48165bca4e8319058644c5be2a9a41c7fa2b8b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2020 06:29:59 +0000 Subject: [PATCH 0004/1498] Bump wcwidth from 0.1.8 to 0.1.9 Bumps [wcwidth](https://github.com/jquast/wcwidth) from 0.1.8 to 0.1.9. - [Release notes](https://github.com/jquast/wcwidth/releases) - [Commits](https://github.com/jquast/wcwidth/compare/0.1.8...0.1.9) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 0191ea0f5f..3b716f646e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,6 +47,6 @@ sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in -wcwidth==0.1.8 # via prompt-toolkit, pytest +wcwidth==0.1.9 # via prompt-toolkit, pytest wrapt==1.11.2 # via astroid yapf==0.29.0 # via -r test-requirements.in From ed868c1c4da0755e4aff1fb29cba573f1e72ce09 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2020 06:14:03 +0000 Subject: [PATCH 0005/1498] Bump sphinxcontrib-trio from 1.1.0 to 1.1.1 Bumps [sphinxcontrib-trio](https://github.com/python-trio/sphinxcontrib-trio) from 1.1.0 to 1.1.1. - [Release notes](https://github.com/python-trio/sphinxcontrib-trio/releases) - [Commits](https://github.com/python-trio/sphinxcontrib-trio/compare/v1.1.0...v1.1.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 7e162227ea..4639b1f4d0 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -36,7 +36,7 @@ sphinxcontrib-htmlhelp==1.0.3 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sphinxcontrib-trio==1.1.0 # via -r docs-requirements.in +sphinxcontrib-trio==1.1.1 # via -r docs-requirements.in toml==0.10.0 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in urllib3==1.25.8 # via requests From 14c66fecd7ac3a8458bc32ffe733eadfac0f3eb5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 29 Mar 2020 19:38:44 -0700 Subject: [PATCH 0006/1498] Turn codecov back on Apparently this got accidentally disabled as part of a cleanup a while ago. (The problem is that the bash <(...) syntax assumes that curl is printing the file to stdout, but we're actually redirecting to a temporary file in order to handle retries.) --- ci.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 63f1355b3e..70f85ce1aa 100755 --- a/ci.sh +++ b/ci.sh @@ -267,7 +267,13 @@ else netsh winsock reset fi - bash <(curl-harder -o codecov.sh https://codecov.io/bash) -n "${JOB_NAME}" + # The codecov docs recommend something like 'bash <(curl ...)' to pipe the + # script directly into bash as its being downloaded. But, the codecov + # server is flaky, so we instead save to a temp file with retries, and + # wait until we've successfully fetched the whole script before trying to + # run it. + curl-harder -o codecov.sh https://codecov.io/bash + bash codecov.sh -n "${JOB_NAME}" $PASSED fi From 3c90e2d3e0f5baf38fe74ebc54ed0f609d55f96b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2020 06:13:10 +0000 Subject: [PATCH 0007/1498] Bump cryptography from 2.8 to 2.9 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.8 to 2.9. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.8...2.9) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3b716f646e..29c51c099b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ attrs==19.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.1.0 # via ipython cffi==1.14.0 # via cryptography coverage==5.0.4 # via pytest-cov -cryptography==2.8 # via pyopenssl, trustme +cryptography==2.9 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets entrypoints==0.3 # via flake8 flake8==3.7.9 # via -r test-requirements.in From 2a2252345162dd9bd0141df34334dd586f1f7b0e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 06:24:43 +0000 Subject: [PATCH 0008/1498] Bump pyparsing from 2.4.6 to 2.4.7 Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 2.4.6 to 2.4.7. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_2.4.6...pyparsing_2.4.7) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 4639b1f4d0..2a653c8fac 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -21,7 +21,7 @@ markupsafe==1.1.1 # via jinja2 outcome==1.0.1 # via -r docs-requirements.in packaging==20.3 # via sphinx pygments==2.6.1 # via sphinx -pyparsing==2.4.6 # via packaging +pyparsing==2.4.7 # via packaging pytz==2019.3 # via babel requests==2.23.0 # via sphinx six==1.14.0 # via packaging diff --git a/test-requirements.txt b/test-requirements.txt index 29c51c099b..6a672cb1c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,7 +39,7 @@ pyflakes==2.1.1 # via flake8 pygments==2.6.1 # via ipython pylint==2.4.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in -pyparsing==2.4.6 # via packaging +pyparsing==2.4.7 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in pytest==5.4.1 # via -r test-requirements.in, pytest-cov six==1.14.0 # via astroid, cryptography, packaging, prompt-toolkit, pyopenssl, traitlets From 51184abe5017498aef7b2466e2c79b3327f936fa Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 06:25:18 +0000 Subject: [PATCH 0009/1498] Bump certifi from 2019.11.28 to 2020.4.5.1 Bumps [certifi](https://github.com/certifi/python-certifi) from 2019.11.28 to 2020.4.5.1. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/commits) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 4639b1f4d0..e0523dbe76 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in attrs==19.3.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx -certifi==2019.11.28 # via requests +certifi==2020.4.5.1 # via requests chardet==3.0.4 # via requests click==7.1.1 # via towncrier docutils==0.16 # via sphinx From 45930f9d4c01ac785adef2613d3407e401ffcb50 Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Tue, 7 Apr 2020 09:33:04 +0800 Subject: [PATCH 0010/1498] [Fix][Docs] use trio.sleep instead of sleep --- docs/source/reference-core.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index f13d3440cb..d4bdd74edd 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -355,7 +355,7 @@ Here's an example:: print("starting...") with trio.move_on_after(5): with trio.move_on_after(10): - await sleep(20) + await trio.sleep(20) print("sleep finished without error") print("move_on_after(10) finished without error") print("move_on_after(5) finished without error") @@ -382,7 +382,7 @@ object representing this cancel scope, which we can use to check whether this scope caught a :exc:`Cancelled` exception:: with trio.move_on_after(5) as cancel_scope: - await sleep(10) + await trio.sleep(10) print(cancel_scope.cancelled_caught) # prints "True" The ``cancel_scope`` object also allows you to check or adjust this From 8a1bb7bd8b3f8c735014d605e2067f0629a50638 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2020 06:11:31 +0000 Subject: [PATCH 0011/1498] Bump coverage from 5.0.4 to 5.1 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.0.4 to 5.1. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.0.4...coverage-5.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6a672cb1c7..f33605c581 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,7 @@ async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.1.0 # via ipython cffi==1.14.0 # via cryptography -coverage==5.0.4 # via pytest-cov +coverage==5.1 # via pytest-cov cryptography==2.9 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets entrypoints==0.3 # via flake8 From 11101999a2ed8686d33d84a1a49727d179f551e1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 14 Apr 2020 06:05:27 +0000 Subject: [PATCH 0012/1498] Bump parso from 0.6.2 to 0.7.0 Bumps [parso](https://github.com/davidhalter/parso) from 0.6.2 to 0.7.0. - [Release notes](https://github.com/davidhalter/parso/releases) - [Changelog](https://github.com/davidhalter/parso/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/parso/compare/v0.6.2...v0.7.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f33605c581..7feee3242f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -26,7 +26,7 @@ mccabe==0.6.1 # via flake8, pylint more-itertools==8.2.0 # via pytest outcome==1.0.1 # via -r test-requirements.in packaging==20.3 # via pytest -parso==0.6.2 # via jedi +parso==0.7.0 # via jedi pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest From de20c1f2f0fa391c3386073e3a04aaa41f0d0c2a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 14 Apr 2020 06:06:09 +0000 Subject: [PATCH 0013/1498] Bump jinja2 from 2.11.1 to 2.11.2 Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.1 to 2.11.2. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.11.1...2.11.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 88b13dfe99..cf1109830c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ idna==2.9 # via -r docs-requirements.in, requests imagesize==1.2.0 # via sphinx immutables==0.11 # via -r docs-requirements.in incremental==17.5.0 # via towncrier -jinja2==2.11.1 # via sphinx, towncrier +jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.0.1 # via -r docs-requirements.in packaging==20.3 # via sphinx From 14ff57cfa6894b7c3c50113bf6889df03e7398d7 Mon Sep 17 00:00:00 2001 From: matham Date: Wed, 15 Apr 2020 18:37:58 -0400 Subject: [PATCH 0014/1498] Remove redundent calling thread tokens. --- trio/_threads.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index e5ffb74b7c..0d6cd02811 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -239,7 +239,6 @@ async def to_thread_run_sync(sync_fn, *args, cancellable=False, limiter=None): """ await trio.hazmat.checkpoint_if_cancelled() - token = trio.hazmat.current_trio_token() if limiter is None: limiter = current_default_thread_limiter() @@ -276,7 +275,7 @@ def worker_thread_fn(trio_token): try: result = outcome.capture(sync_fn, *args) try: - token.run_sync_soon(report_back_in_trio_thread_fn, result) + trio_token.run_sync_soon(report_back_in_trio_thread_fn, result) except trio.RunFinishedError: # The entire run finished, so our particular task is certainly # long gone -- it must have cancelled. From db699f367fc6c38754fcb4862126ffdc27c23e2a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2020 06:02:56 +0000 Subject: [PATCH 0015/1498] Bump urllib3 from 1.25.8 to 1.25.9 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.25.8 to 1.25.9. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/master/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.25.8...1.25.9) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index cf1109830c..58865b63c4 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -39,4 +39,4 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx sphinxcontrib-trio==1.1.1 # via -r docs-requirements.in toml==0.10.0 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in -urllib3==1.25.8 # via requests +urllib3==1.25.9 # via requests From 9614714c612bd76c9a684da80f74467b1449e50f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 19 Apr 2020 21:50:57 +0400 Subject: [PATCH 0016/1498] Remove deprecated open_cancel_scope --- trio/__init__.py | 5 ++--- trio/_core/__init__.py | 12 ++++++------ trio/_core/_run.py | 6 ------ trio/_core/tests/test_run.py | 7 ------- 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index d517bc7f18..679c8509be 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -18,9 +18,8 @@ from ._core import ( TrioInternalError, RunFinishedError, WouldBlock, Cancelled, BusyResourceError, ClosedResourceError, MultiError, run, open_nursery, - CancelScope, open_cancel_scope, current_effective_deadline, - TASK_STATUS_IGNORED, current_time, BrokenResourceError, EndOfChannel, - Nursery + CancelScope, current_effective_deadline, TASK_STATUS_IGNORED, current_time, + BrokenResourceError, EndOfChannel, Nursery ) from ._timeouts import ( diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index fda027e193..cb3d5d714f 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -17,12 +17,12 @@ # Imports that always exist from ._run import ( - Task, CancelScope, run, open_nursery, open_cancel_scope, checkpoint, - current_task, current_effective_deadline, checkpoint_if_cancelled, - TASK_STATUS_IGNORED, current_statistics, current_trio_token, reschedule, - remove_instrument, add_instrument, current_clock, current_root_task, - spawn_system_task, current_time, wait_all_tasks_blocked, wait_readable, - wait_writable, notify_closing, Nursery + Task, CancelScope, run, open_nursery, checkpoint, current_task, + current_effective_deadline, checkpoint_if_cancelled, TASK_STATUS_IGNORED, + current_statistics, current_trio_token, reschedule, remove_instrument, + add_instrument, current_clock, current_root_task, spawn_system_task, + current_time, wait_all_tasks_blocked, wait_readable, wait_writable, + notify_closing, Nursery ) # Has to come after _run to resolve a circular import diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 9bb115dadc..3ddfa5d524 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -619,12 +619,6 @@ def cancel_called(self): return self._cancel_called -@deprecated("0.11.0", issue=607, instead="trio.CancelScope") -def open_cancel_scope(*, deadline=inf, shield=False): - """Returns a context manager which creates a new cancellation scope.""" - return CancelScope(deadline=deadline, shield=shield) - - ################################################################ # Nursery and friends ################################################################ diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index fc03197133..3593793fe1 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -805,13 +805,6 @@ async def test_basic_timeout(mock_clock): await _core.checkpoint() -@pytest.mark.filterwarnings( - "ignore:.*trio.open_cancel_scope:trio.TrioDeprecationWarning" -) -async def test_cancel_scope_deprecated(recwarn): - assert isinstance(_core.open_cancel_scope(), _core.CancelScope) - - async def test_cancel_scope_nesting(): # Nested scopes: if two triggering at once, the outer one wins with _core.CancelScope() as scope1: From 0c3745d8bd17bce8488b7bd39418592b6e09f270 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sun, 19 Apr 2020 22:00:34 +0400 Subject: [PATCH 0017/1498] Add newsfragment --- newsfragments/1458.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1458.removal.rst diff --git a/newsfragments/1458.removal.rst b/newsfragments/1458.removal.rst new file mode 100644 index 0000000000..087a38c6d6 --- /dev/null +++ b/newsfragments/1458.removal.rst @@ -0,0 +1 @@ +Remove ``trio.open_cancel_scope`` which was deprecated in 0.11.0. From 34dc44d0424385c41c81d0fc27559d02012871b3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2020 06:24:11 +0000 Subject: [PATCH 0018/1498] Bump cryptography from 2.9 to 2.9.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.9 to 2.9.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.9...2.9.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7feee3242f..a660ad4d82 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ attrs==19.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.1.0 # via ipython cffi==1.14.0 # via cryptography coverage==5.1 # via pytest-cov -cryptography==2.9 # via pyopenssl, trustme +cryptography==2.9.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets entrypoints==0.3 # via flake8 flake8==3.7.9 # via -r test-requirements.in From 53510e3bba259e4164c181546086831a7c809670 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2020 06:07:34 +0000 Subject: [PATCH 0019/1498] Bump immutables from 0.11 to 0.12 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.11 to 0.12. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.11...v0.12) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 58865b63c4..f6b3a6359e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ click==7.1.1 # via towncrier docutils==0.16 # via sphinx idna==2.9 # via -r docs-requirements.in, requests imagesize==1.2.0 # via sphinx -immutables==0.11 # via -r docs-requirements.in +immutables==0.12 # via -r docs-requirements.in incremental==17.5.0 # via towncrier jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 diff --git a/test-requirements.txt b/test-requirements.txt index a660ad4d82..cec9927d2d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -16,7 +16,7 @@ decorator==4.4.2 # via ipython, traitlets entrypoints==0.3 # via flake8 flake8==3.7.9 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme -immutables==0.11 # via -r test-requirements.in +immutables==0.12 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.9.0 # via -r test-requirements.in isort==4.3.21 # via pylint From 0ee1bf475a5fc50b178556ae5e01e207df12375d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2020 06:08:19 +0000 Subject: [PATCH 0020/1498] Bump cryptography from 2.9.1 to 2.9.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.9.1 to 2.9.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.9.1...2.9.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a660ad4d82..d6cc0204be 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ attrs==19.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.1.0 # via ipython cffi==1.14.0 # via cryptography coverage==5.1 # via pytest-cov -cryptography==2.9.1 # via pyopenssl, trustme +cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets entrypoints==0.3 # via flake8 flake8==3.7.9 # via -r test-requirements.in From bc9a11609dc9a2da58487eddcd2235091140d55a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 24 Apr 2020 06:17:55 +0000 Subject: [PATCH 0021/1498] Bump yapf from 0.29.0 to 0.30.0 Bumps [yapf](https://github.com/google/yapf) from 0.29.0 to 0.30.0. - [Release notes](https://github.com/google/yapf/releases) - [Changelog](https://github.com/google/yapf/blob/master/CHANGELOG) - [Commits](https://github.com/google/yapf/compare/v0.29.0...v0.30.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.in | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 077eb70dcf..f03409d484 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -8,7 +8,7 @@ pylint # for pylint finding all symbols tests jedi # for jedi code completion tests # Tools -yapf ==0.29.0 # formatting +yapf ==0.30.0 # formatting flake8 astor # code generation diff --git a/test-requirements.txt b/test-requirements.txt index cc66fb37bf..e23ce7acd0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -49,4 +49,4 @@ traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in wcwidth==0.1.9 # via prompt-toolkit, pytest wrapt==1.11.2 # via astroid -yapf==0.29.0 # via -r test-requirements.in +yapf==0.30.0 # via -r test-requirements.in From 09ce2d77a58c7178900b900bbca5193c0f5d4637 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 24 Apr 2020 16:54:51 +0400 Subject: [PATCH 0022/1498] Run yapf 0.30.0 --- trio/tests/test_ssl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 0ac217bb4c..16b323e98b 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1255,8 +1255,8 @@ async def test_getpeercert(client_ctx): assert server.getpeercert() is None print(client.getpeercert()) assert ( - ("DNS", - "trio-test-1.example.org") in client.getpeercert()["subjectAltName"] + ("DNS", "trio-test-1.example.org") + in client.getpeercert()["subjectAltName"] ) From 7a82b562a4604de673abbcd4f1e4252d090d0cae Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 02:04:30 -0700 Subject: [PATCH 0023/1498] Port azure pipelines configuration to github actions --- .github/workflows/ci.yml | 90 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..93750278b0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,90 @@ +name: CI + +on: [pull_request, push] + +jobs: + Windows: + name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' + timeout-minutes: 20 + runs-on: 'windows-latest' + strategy: + matrix: + python: ['3.5', '3.6', '3.7', '3.8'] + arch: ['x86', 'x64'] + lsp: [''] + extra_name: [''] + include: + - python: '3.8' + arch: 'x64' + lsp: 'http://www.proxifier.com/download/ProxifierSetup.exe' + extra_name: ', with IFS LSP' + - python: '3.8' + arch: 'x64' + lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' + extra_name: ', with non-IFS LSP' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: '${{ matrix.python }}' + architecture: '${{ matrix.arch }}' + - name: Run tests + run: ./ci.sh + shell: bash + env: + LSP: '${{ matrix.lsp }}' + # Should match 'name:' up above + JOB_NAME: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' + + Linux: + name: 'Linux (${{ matrix.python }}${{ matrix.extra_name }})' + timeout-minutes: 10 + runs-on: 'ubuntu-latest' + strategy: + matrix: + python: ['3.5', '3.6', '3.7', '3.8'] + check_docs: ['0'] + check_formatting: ['0'] + extra_name: [''] + include: + - python: '3.8' + check_docs: '1' + extra_name: ', check docs' + - python: '3.8' + check_formatting: '1' + extra_name: ', check formatting' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: '${{ matrix.python }}' + - name: Run tests + run: ./ci.sh + env: + CHECK_DOCS: '${{ matrix.check_docs }}' + CHECK_FORMATTING: '${{ matrix.check_formatting }}' + # Should match 'name:' up above + JOB_NAME: 'Linux (${{ matrix.python }}${{ matrix.extra_name }})' + + macOS: + name: 'macOS (${{ matrix.python }})' + timeout-minutes: 10 + runs-on: 'macos-latest' + strategy: + matrix: + python: ['3.5', '3.6', '3.7', '3.8'] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: '${{ matrix.python }}' + - name: Run tests + run: ./ci.sh + env: + JOB_NAME: 'macOS (${{ matrix.python }})' From a1774369d13f305cf89804c9d8d78f95d6900cbe Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 04:21:25 -0700 Subject: [PATCH 0024/1498] only run on pull requests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93750278b0..0aac4c009f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: CI -on: [pull_request, push] +on: [pull_request] jobs: Windows: From 5fe170e517b6fe56a6a17eb95ceef30895c50f16 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2020 11:28:35 +0000 Subject: [PATCH 0025/1498] Bump jedi from 0.16.0 to 0.17.0 Bumps [jedi](https://github.com/davidhalter/jedi) from 0.16.0 to 0.17.0. - [Release notes](https://github.com/davidhalter/jedi/releases) - [Changelog](https://github.com/davidhalter/jedi/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/jedi/compare/v0.16.0...v0.17.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e23ce7acd0..b58b2f22c6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -20,7 +20,7 @@ immutables==0.12 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.9.0 # via -r test-requirements.in isort==4.3.21 # via pylint -jedi==0.16.0 # via -r test-requirements.in, ipython +jedi==0.17.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint more-itertools==8.2.0 # via pytest From 947e88dc274c0e9fe0ff3fe2d6ccf688bfb9e1b2 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 04:34:12 -0700 Subject: [PATCH 0026/1498] tiny comment cleanup --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0aac4c009f..132fdff097 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,4 +87,5 @@ jobs: - name: Run tests run: ./ci.sh env: + # Should match 'name:' up above JOB_NAME: 'macOS (${{ matrix.python }})' From 8b3c3adcf4d0ac9e15bc4d5f27dc03672746a370 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 04:35:06 -0700 Subject: [PATCH 0027/1498] Remove unused yapf import Needed in particular because it just started triggering a deprecation warning on python nightly. --- trio/_tools/gen_exports.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trio/_tools/gen_exports.py b/trio/_tools/gen_exports.py index cc0738177a..1340d049a2 100755 --- a/trio/_tools/gen_exports.py +++ b/trio/_tools/gen_exports.py @@ -10,7 +10,6 @@ import os from pathlib import Path import sys -import yapf.yapflib.yapf_api as formatter from textwrap import indent From 7fa11bdbcfee1c65d0bff590337ed4355ebbe44b Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 27 Apr 2020 10:37:49 +0400 Subject: [PATCH 0028/1498] Release 0.14.0 --- docs/source/history.rst | 45 ++++++++++++++++++++++++++++++++++ newsfragments/1272.feature.rst | 16 ------------ newsfragments/1308.bugfix.rst | 9 ------- newsfragments/1458.removal.rst | 1 - trio/_version.py | 2 +- 5 files changed, 46 insertions(+), 27 deletions(-) delete mode 100644 newsfragments/1272.feature.rst delete mode 100644 newsfragments/1308.bugfix.rst delete mode 100644 newsfragments/1458.removal.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index ed6cb2a7b7..65bb5dd54c 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,51 @@ Release history .. towncrier release notes start +Trio 0.14.0 (2020-04-27) +------------------------ + +Features +~~~~~~~~ + +- If you're using Trio's low-level interfaces like + `trio.hazmat.wait_readable` or similar, and then you close a socket or + file descriptor, you're supposed to call `trio.hazmat.notify_closing` + first so Trio can clean up properly. But what if you forget? In the + past, Trio would tend to either deadlock or explode spectacularly. + Now, it's much more robust to this situation, and should generally + survive. (But note that "survive" is not the same as "give you the + results you were expecting", so you should still call + `~trio.hazmat.notify_closing` when appropriate. This is about harm + reduction and making it easier to debug this kind of mistake, not + something you should rely on.) + + If you're using higher-level interfaces outside of the `trio.hazmat` + module, then you don't need to worry about any of this; those + intefaces already take care of calling `~trio.hazmat.notify_closing` + for you. (`#1272 `__) + + +Bugfixes +~~~~~~~~ + +- A bug related to the following methods has been introduced in version 0.12.0: + + - `trio.Path.iterdir` + - `trio.Path.glob` + - `trio.Path.rglob` + + The iteration of the blocking generators produced by pathlib was performed in + the trio thread. With this fix, the previous behavior is restored: the blocking + generators are converted into lists in a thread dedicated to blocking IO calls. (`#1308 `__) + + +Deprecations and Removals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Deprecate Python 3.5 (`#1408 `__) +- Remove ``trio.open_cancel_scope`` which was deprecated in 0.11.0. (`#1458 `__) + + Trio 0.13.0 (2019-11-02) ------------------------ diff --git a/newsfragments/1272.feature.rst b/newsfragments/1272.feature.rst deleted file mode 100644 index 3f540b93e5..0000000000 --- a/newsfragments/1272.feature.rst +++ /dev/null @@ -1,16 +0,0 @@ -If you're using Trio's low-level interfaces like -`trio.hazmat.wait_readable` or similar, and then you close a socket or -file descriptor, you're supposed to call `trio.hazmat.notify_closing` -first so Trio can clean up properly. But what if you forget? In the -past, Trio would tend to either deadlock or explode spectacularly. -Now, it's much more robust to this situation, and should generally -survive. (But note that "survive" is not the same as "give you the -results you were expecting", so you should still call -`~trio.hazmat.notify_closing` when appropriate. This is about harm -reduction and making it easier to debug this kind of mistake, not -something you should rely on.) - -If you're using higher-level interfaces outside of the `trio.hazmat` -module, then you don't need to worry about any of this; those -intefaces already take care of calling `~trio.hazmat.notify_closing` -for you. diff --git a/newsfragments/1308.bugfix.rst b/newsfragments/1308.bugfix.rst deleted file mode 100644 index db435e46c1..0000000000 --- a/newsfragments/1308.bugfix.rst +++ /dev/null @@ -1,9 +0,0 @@ -A bug related to the following methods has been introduced in version 0.12.0: - -- `trio.Path.iterdir` -- `trio.Path.glob` -- `trio.Path.rglob` - -The iteration of the blocking generators produced by pathlib was performed in -the trio thread. With this fix, the previous behavior is restored: the blocking -generators are converted into lists in a thread dedicated to blocking IO calls. diff --git a/newsfragments/1458.removal.rst b/newsfragments/1458.removal.rst deleted file mode 100644 index 087a38c6d6..0000000000 --- a/newsfragments/1458.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``trio.open_cancel_scope`` which was deprecated in 0.11.0. diff --git a/trio/_version.py b/trio/_version.py index 56ce2746f8..0705c99035 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.13.0+dev" +__version__ = "0.14.0" From d40b23e528828b7318739038a5e588b9c69e371f Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 05:29:40 -0700 Subject: [PATCH 0029/1498] Update test to use new, non-deprecated jedi.Script API --- trio/tests/test_exports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index abc6f07963..3650403179 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -92,7 +92,7 @@ def no_underscores(symbols): import jedi # Simulate typing "import trio; trio." script = jedi.Script("import {}; {}.".format(modname, modname)) - completions = script.completions() + completions = script.complete() static_names = no_underscores(c.name for c in completions) else: # pragma: no cover assert False From 3a5802e2a99c5b58335f3c29d68e0f4cb96733ff Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 05:30:03 -0700 Subject: [PATCH 0030/1498] Conditionally import SO_REUSEADDR, instead of conditionally deleting it We only expose trio.socket.SO_REUSEADDR on non-Windows platforms. We used to import it everywhere, and then delete it again on Windows. But apparently the latest versions of Jedi have grown smart enough to detect this deletion (but not its conditional nature), so they stopped being able to complete SO_REUSEADDR. Out-smart Jedi by skipping importing it on Windows, instead. --- trio/socket.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/trio/socket.py b/trio/socket.py index 266fc0dbda..3a0a61b4bf 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -115,10 +115,18 @@ # have: import socket as _stdlib_socket +_bad_symbols = set() +if _sys.platform == 'win32': + # See https://github.com/python-trio/trio/issues/39 + # Do not import for windows platform + # (you can still get it from stdlib socket, of course, if you want it) + _bad_symbols.add("SO_REUSEADDR") + globals().update( { _name: getattr(_stdlib_socket, _name) - for _name in _stdlib_socket.__all__ if _name.isupper() + for _name in _stdlib_socket.__all__ + if _name.isupper() and _name not in _bad_symbols } ) @@ -157,12 +165,6 @@ except ImportError: pass -if _sys.platform == 'win32': - # See https://github.com/python-trio/trio/issues/39 - # Do not import for windows platform - # (you can still get it from stdlib socket, of course, if you want it) - del SO_REUSEADDR - # get names used by Trio that we define on our own from ._socket import IPPROTO_IPV6 From bb68839e2b3ab2f5f9887bb1f820322ce0519cea Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2020 12:48:49 +0000 Subject: [PATCH 0031/1498] Bump pylint from 2.4.2 to 2.5.0 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.4.2 to 2.5.0. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.4.2...pylint-2.5.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index b58b2f22c6..360a6600f0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ # pip-compile --output-file test-requirements.txt test-requirements.in # astor==0.8.1 # via -r test-requirements.in -astroid==2.3.3 # via pylint +astroid==2.4.0 # via pylint async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.1.0 # via ipython @@ -37,7 +37,7 @@ pycodestyle==2.5.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.1.1 # via flake8 pygments==2.6.1 # via ipython -pylint==2.4.2 # via -r test-requirements.in +pylint==2.5.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in @@ -45,6 +45,7 @@ pytest==5.4.1 # via -r test-requirements.in, pytest-cov six==1.14.0 # via astroid, cryptography, packaging, prompt-toolkit, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in +toml==0.10.0 # via pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in wcwidth==0.1.9 # via prompt-toolkit, pytest From 7a804358091392e492a8960073889d0b395dbf79 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 27 Apr 2020 17:15:27 +0400 Subject: [PATCH 0032/1498] Bump version to v0.14.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index 0705c99035..7aff89c31d 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.14.0" +__version__ = "0.14.0+dev" From 615997553784af13330b4d6166b6c5bae4c57501 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 08:46:43 -0700 Subject: [PATCH 0033/1498] Delete azure-pipelines.yml Github Actions seems to be working correctly, and we can always restore this from history if we change our mind. I also "disabled" the pipeline in the Azure interface, so if we do need to re-enable it then we'll have to go fix that as well. --- azure-pipelines.yml | 126 -------------------------------------------- 1 file changed, 126 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index dabc5cb2d2..0000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,126 +0,0 @@ -trigger: - branches: - exclude: - - 'dependabot/*' - -jobs: - -- job: 'Windows' - pool: - vmImage: 'windows-latest' - timeoutInMinutes: 20 - strategy: - # Python version list: - # 64-bit: https://www.nuget.org/packages/python/ - # 32-bit: https://www.nuget.org/packages/pythonx86/ - matrix: - # The LSP tests can be super slow for some reason - like - # sometimes it just randomly takes 5 minutes to run the LSP - # installer. So we put them at the top, so they can get started - # earlier. - "with IFS LSP, Python 3.7, 64 bit": - python.version: '3.7.5' - python.pkg: 'python' - lsp: 'http://www.proxifier.com/download/ProxifierSetup.exe' - "with non-IFS LSP, Python 3.7, 64 bit": - python.version: '3.7.5' - python.pkg: 'python' - lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' - "Python 3.5, 32 bit": - python.version: '3.5.4' - python.pkg: 'pythonx86' - "Python 3.5, 64 bit": - python.version: '3.5.4' - python.pkg: 'python' - "Python 3.6, 32 bit": - python.version: '3.6.8' - python.pkg: 'pythonx86' - "Python 3.6, 64 bit": - python.version: '3.6.8' - python.pkg: 'python' - "Python 3.7, 32 bit": - python.version: '3.7.5' - python.pkg: 'pythonx86' - "Python 3.7, 64 bit": - python.version: '3.7.5' - python.pkg: 'python' - "Python 3.8, 32 bit": - python.version: '3.8.0' - python.pkg: 'pythonx86' - "Python 3.8, 64 bit": - python.version: '3.8.0' - python.pkg: 'python' - - steps: - - task: NuGetToolInstaller@0 - - - bash: ./ci.sh - displayName: "Run the actual tests" - - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'test-results.xml' - testRunTitle: 'Windows $(python.pkg) $(python.version)' - condition: succeededOrFailed() - -- job: 'Linux' - pool: - vmImage: 'ubuntu-latest' - timeoutInMinutes: 10 - strategy: - matrix: - "Check docs": - python.version: '3.8' - CHECK_DOCS: 1 - "Formatting and linting": - python.version: '3.8' - CHECK_FORMATTING: 1 - "Python 3.5": - python.version: '3.5' - "Python 3.6": - python.version: '3.6' - "Python 3.7": - python.version: '3.7' - "Python 3.8": - python.version: '3.8' - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - - - bash: ./ci.sh - displayName: "Run the actual tests" - - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'test-results.xml' - condition: succeededOrFailed() - -- job: 'macOS' - pool: - vmImage: 'macOS-latest' - timeoutInMinutes: 10 - strategy: - matrix: - "Python 3.5": - python.version: '3.5' - "Python 3.6": - python.version: '3.6' - "Python 3.7": - python.version: '3.7' - "Python 3.8": - python.version: '3.8' - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - - - bash: ./ci.sh - displayName: "Run the actual tests" - - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'test-results.xml' - condition: succeededOrFailed() From 5b1af22f915941dfb456117b596ed0872cc78694 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 27 Apr 2020 08:49:44 -0700 Subject: [PATCH 0034/1498] Add fail-fast: false to Github Actions config By default, as soon as a single job in a group fails, then GA will cancel all the other jobs. This is kind of confusing, e.g. I was just looking at a PR that had a Python 3.5-specific failure, but that wasn't immediately obvious, because GA only ran the 3.5 tests and then skipped all the others. So let's disable that behavior. --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 132fdff097..c7b540e1c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ jobs: timeout-minutes: 20 runs-on: 'windows-latest' strategy: + fail-fast: false matrix: python: ['3.5', '3.6', '3.7', '3.8'] arch: ['x86', 'x64'] @@ -43,6 +44,7 @@ jobs: timeout-minutes: 10 runs-on: 'ubuntu-latest' strategy: + fail-fast: false matrix: python: ['3.5', '3.6', '3.7', '3.8'] check_docs: ['0'] @@ -75,6 +77,7 @@ jobs: timeout-minutes: 10 runs-on: 'macos-latest' strategy: + fail-fast: false matrix: python: ['3.5', '3.6', '3.7', '3.8'] steps: From fd818ec9eb185b71cd7b7636ee102272cb16dce0 Mon Sep 17 00:00:00 2001 From: Micah Lyle Date: Mon, 27 Apr 2020 09:47:23 -0700 Subject: [PATCH 0035/1498] Make docstring slightly clearer in Nursery start --- trio/_core/_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 3ddfa5d524..c1ed65ed25 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -929,8 +929,8 @@ async def async_fn(arg1, arg2, \*, task_status=trio.TASK_STATUS_IGNORED): :meth:`start` is cancelled, then the child task is also cancelled. - When the child calls ``task_status.started()``, it's moved from - out from underneath :meth:`start` and into the given nursery. + When the child calls ``task_status.started()``, it's moved out + from underneath :meth:`start` and into the given nursery. If the child task passes a value to ``task_status.started(value)``, then :meth:`start` returns this From 6afb24da2edb3fd7472229ce8ad6fa09bee82d34 Mon Sep 17 00:00:00 2001 From: Micah Lyle Date: Mon, 27 Apr 2020 14:21:53 -0700 Subject: [PATCH 0036/1498] Quick buffering docs correction --- docs/source/reference-core.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index d4bdd74edd..227c67bd6a 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1306,7 +1306,7 @@ produces *backpressure*: if the channel producers are running faster than the consumers, then it forces the producers to slow down. You can disable buffering entirely, by doing -``open_memory_channel(0)``. In that case any task calls +``open_memory_channel(0)``. In that case any task that calls :meth:`~trio.abc.SendChannel.send` will wait until another task calls :meth:`~trio.abc.ReceiveChannel.receive`, and vice versa. This is similar to how channels work in the `classic Communicating Sequential Processes From acceec50985357c2631b9972be62cdd55191e945 Mon Sep 17 00:00:00 2001 From: Micah Lyle Date: Mon, 27 Apr 2020 14:23:12 -0700 Subject: [PATCH 0037/1498] Removed reference to `portal` in from thread example --- docs/source/reference-core/from-thread-example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-core/from-thread-example.py b/docs/source/reference-core/from-thread-example.py index 71a75d67bf..b95a0e00d3 100644 --- a/docs/source/reference-core/from-thread-example.py +++ b/docs/source/reference-core/from-thread-example.py @@ -21,7 +21,7 @@ async def main(): async with trio.open_nursery() as nursery: # In a background thread, run: - # thread_fn(portal, receive_from_trio, send_to_trio) + # thread_fn(receive_from_trio, send_to_trio) nursery.start_soon( trio.to_thread.run_sync, thread_fn, receive_from_trio, send_to_trio ) From cbae4d43b3010e7d7759349ecd96cd1716ecda6a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 28 Apr 2020 06:11:44 +0000 Subject: [PATCH 0038/1498] Bump wrapt from 1.11.2 to 1.12.1 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.11.2 to 1.12.1. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.11.2...1.12.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 360a6600f0..b912d752d3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -49,5 +49,5 @@ toml==0.10.0 # via pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in wcwidth==0.1.9 # via prompt-toolkit, pytest -wrapt==1.11.2 # via astroid +wrapt==1.12.1 # via astroid yapf==0.30.0 # via -r test-requirements.in From 8b8ff371d4473811ebf0da357674a31727dfb29b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 28 Apr 2020 06:12:08 +0000 Subject: [PATCH 0039/1498] Bump click from 7.1.1 to 7.1.2 Bumps [click](https://github.com/pallets/click) from 7.1.1 to 7.1.2. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/7.1.1...7.1.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f6b3a6359e..cdf0c30996 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -10,7 +10,7 @@ attrs==19.3.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx certifi==2020.4.5.1 # via requests chardet==3.0.4 # via requests -click==7.1.1 # via towncrier +click==7.1.2 # via towncrier docutils==0.16 # via sphinx idna==2.9 # via -r docs-requirements.in, requests imagesize==1.2.0 # via sphinx From c6d947068ca0692fc7ff83b30aaf170354f92724 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 8 Nov 2019 12:23:32 -0800 Subject: [PATCH 0040/1498] Use pidfd to implement Process.wait, when available --- newsfragments/1241.feature.rst | 4 +++ trio/_subprocess.py | 63 ++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 newsfragments/1241.feature.rst diff --git a/newsfragments/1241.feature.rst b/newsfragments/1241.feature.rst new file mode 100644 index 0000000000..8accb4235e --- /dev/null +++ b/newsfragments/1241.feature.rst @@ -0,0 +1,4 @@ +On Linux kernels v5.3 or newer, `trio.Process.wait` now uses `the +pidfd API `__ to track child +processes. This shouldn't have any user-visible change, but it makes +working with subprocesses faster and use less memory. diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 809af3f28a..74e920be02 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -1,5 +1,6 @@ import os import subprocess +import sys from typing import Optional from ._abc import AsyncResource, SendStream, ReceiveStream @@ -11,6 +12,39 @@ ) import trio +# Linux-specific, but has complex lifetime management stuff so we hard-code it +# here instead of hiding it behind the _subprocess_platform abstraction +can_try_pidfd_open = True +try: + from os import pidfd_open +except ImportError: + if sys.platform == "linux": + import ctypes + _cdll_for_pidfd_open = ctypes.CDLL(None, use_errno=True) + _cdll_for_pidfd_open.syscall.restype = ctypes.c_long + # fd and flags are actually int-sized, but the syscall() function + # always takes longs. (Except on x32 where long is 32-bits and syscall + # takes 64-bit arguments. But in the unlikely case that anyone is + # using x32, this will still work, b/c we only need to pass in 32 bits + # of data, and the C ABI doesn't distinguish between passing 32-bit vs + # 64-bit integers; our 32-bit values will get loaded into 64-bit + # registers where syscall() will find them.) + _cdll_for_pidfd_open.syscall.argtypes = [ + ctypes.c_long, # syscall number + ctypes.c_long, # fd + ctypes.c_long, # flags + ] + __NR_pidfd_open = 434 + + def pidfd_open(fd, flags): + result = _cdll_for_pidfd_open.syscall(__NR_pidfd_open, fd, flags) + if result < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return result + else: + can_try_pidfd_open = False + class Process(AsyncResource): r"""A child process. Like :class:`subprocess.Popen`, but async. @@ -151,6 +185,21 @@ def _init( if self.stderr is not None: os.close(stderr) + self._pidfd = None + if can_try_pidfd_open: + try: + fd = pidfd_open(self._proc.pid, 0) + except OSError: + # Well, we tried, but it didn't work (probably because we're + # running on an older kernel, or in an older sandbox, that + # hasn't been updated to support pidfd_open). We'll fall back + # on waitid instead. + pass + else: + # It worked! Wrap the raw fd up in a Python file object to + # make sure it'll get closed. + self._pidfd = open(fd) + if self.stdin is not None and self.stdout is not None: self.stdio = StapledStream(self.stdin, self.stdout) @@ -214,10 +263,14 @@ async def wait(self): if self.poll() is None: async with self._wait_lock: if self.poll() is None: - await wait_child_exiting(self) - self._proc.wait() + if self._pidfd is not None: + await trio.hazmat.wait_readable(self._pidfd) + else: + await wait_child_exiting(self) + self.poll() else: await trio.hazmat.checkpoint() + assert self.returncode is not None return self.returncode def poll(self): @@ -227,7 +280,11 @@ def poll(self): The exit status of the process, or ``None`` if it is still running; see :attr:`returncode`. """ - return self._proc.poll() + result = self._proc.poll() + if result is not None and self._pidfd is not None: + self._pidfd.close() + self._pidfd = None + return result def send_signal(self, sig): """Send signal ``sig`` to the process. From f3aa431e4ca27b6e4269bff5bb7bdcd5b6f3e66a Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 11 Feb 2020 23:52:53 -0800 Subject: [PATCH 0041/1498] Address feedback from review --- trio/_subprocess.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 74e920be02..3284813cfc 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -31,7 +31,7 @@ # registers where syscall() will find them.) _cdll_for_pidfd_open.syscall.argtypes = [ ctypes.c_long, # syscall number - ctypes.c_long, # fd + ctypes.c_long, # pid ctypes.c_long, # flags ] __NR_pidfd_open = 434 @@ -254,22 +254,31 @@ async def aclose(self): with trio.CancelScope(shield=True): await self.wait() + def _close_pidfd(self): + if self._pidfd is not None: + self._pidfd.close() + self._pidfd = None + async def wait(self): """Block until the process exits. Returns: The exit status of the process; see :attr:`returncode`. """ - if self.poll() is None: - async with self._wait_lock: - if self.poll() is None: - if self._pidfd is not None: - await trio.hazmat.wait_readable(self._pidfd) - else: - await wait_child_exiting(self) - self.poll() - else: - await trio.hazmat.checkpoint() + async with self._wait_lock: + if self.poll() is None: + if self._pidfd is not None: + await trio.hazmat.wait_readable(self._pidfd) + else: + await wait_child_exiting(self) + # We have to use .wait() here, not .poll(), because on macOS + # (and maybe other systems, who knows), there's a race + # condition inside the kernel that creates a tiny window where + # kqueue reports that the process has exited, but + # waitpid(WNOHANG) can't yet reap it. So this .wait() may + # actually block for a tiny fraction of a second. + self._proc.wait() + self._close_pidfd() assert self.returncode is not None return self.returncode @@ -281,9 +290,8 @@ def poll(self): running; see :attr:`returncode`. """ result = self._proc.poll() - if result is not None and self._pidfd is not None: - self._pidfd.close() - self._pidfd = None + if result is not None: + self._close_pidfd() return result def send_signal(self, sig): From c47255b2307d1194903d6fa2484a76a322f6fc9e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 28 Apr 2020 07:06:09 -0700 Subject: [PATCH 0042/1498] Fix wording in comment --- trio/_subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 3284813cfc..7e82b77318 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -22,7 +22,7 @@ import ctypes _cdll_for_pidfd_open = ctypes.CDLL(None, use_errno=True) _cdll_for_pidfd_open.syscall.restype = ctypes.c_long - # fd and flags are actually int-sized, but the syscall() function + # pid and flags are actually int-sized, but the syscall() function # always takes longs. (Except on x32 where long is 32-bits and syscall # takes 64-bit arguments. But in the unlikely case that anyone is # using x32, this will still work, b/c we only need to pass in 32 bits From 4e160ea174a15b5bb8eb71b4529c37c07d821109 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 28 Apr 2020 11:42:12 -0700 Subject: [PATCH 0043/1498] Use --insecure when fetching LSP installers in CI Fixes gh-1478 --- ci.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 70f85ce1aa..e5ad7f8875 100755 --- a/ci.sh +++ b/ci.sh @@ -237,7 +237,17 @@ else # up. if [ "$LSP" != "" ]; then echo "Installing LSP from ${LSP}" - curl-harder -o lsp-installer.exe "$LSP" + # We use --insecure because one of the LSP's has been observed to give + # cert verification errors: + # + # https://github.com/python-trio/trio/issues/1478 + # + # *Normally*, you should never ever use --insecure, especially when + # fetching an executable! But *in this case*, we're intentionally + # installing some untrustworthy quasi-malware onto into a sandboxed + # machine for testing. So MITM attacks are really the least of our + # worries. + curl-harder --insecure -o lsp-installer.exe "$LSP" # Double-slashes are how you tell windows-bash that you want a single # slash, and don't treat this as a unix-style filename that needs to # be replaced by a windows-style filename. From c615cc1ce9be75649d68db099728c9701e4fb708 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 29 Apr 2020 06:19:59 +0000 Subject: [PATCH 0044/1498] Bump pytz from 2019.3 to 2020.1 Bumps [pytz](https://github.com/stub42/pytz) from 2019.3 to 2020.1. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2019.3...release_2020.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index cdf0c30996..31e88d374b 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -22,7 +22,7 @@ outcome==1.0.1 # via -r docs-requirements.in packaging==20.3 # via sphinx pygments==2.6.1 # via sphinx pyparsing==2.4.7 # via packaging -pytz==2019.3 # via babel +pytz==2020.1 # via babel requests==2.23.0 # via sphinx six==1.14.0 # via packaging sniffio==1.1.0 # via -r docs-requirements.in From a434a00f98d52430d768b3c9169e646ad2715fc1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 29 Apr 2020 06:28:54 +0000 Subject: [PATCH 0045/1498] Bump sphinx from 2.4.4 to 3.0.3 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 2.4.4 to 3.0.3. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v2.4.4...v3.0.3) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 31e88d374b..00a1c52c21 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -29,7 +29,7 @@ sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.1.0 # via -r docs-requirements.in sphinx-rtd-theme==0.4.3 # via -r docs-requirements.in -sphinx==2.4.4 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx==3.0.3 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 2564020d710b1cf4e793725ee4bf017db9813e7e Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 14:41:30 +0400 Subject: [PATCH 0046/1498] Silence pytest junit_family warning --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index e6579a4f07..90b7a189bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,4 @@ xfail_strict = true faulthandler_timeout=60 markers = redistributors_should_skip: tests that should be skipped by downstream redistributors +junit_family = xunit2 From a20713270ffea7241e575446b310997fef1799e8 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 10:43:47 +0400 Subject: [PATCH 0047/1498] Stop testing Python 3.5 in CI --- .github/workflows/ci.yml | 6 +++--- .travis.yml | 7 ------- ci.sh | 4 ---- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7b540e1c4..819c61c5d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.5', '3.6', '3.7', '3.8'] + python: ['3.6', '3.7', '3.8'] arch: ['x86', 'x64'] lsp: [''] extra_name: [''] @@ -46,7 +46,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.5', '3.6', '3.7', '3.8'] + python: ['3.6', '3.7', '3.8'] check_docs: ['0'] check_formatting: ['0'] extra_name: [''] @@ -79,7 +79,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.5', '3.6', '3.7', '3.8'] + python: ['3.6', '3.7', '3.8'] steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml index 685b4a8e93..756d472eb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,13 +21,6 @@ jobs: env: - "JOB_NAME='Ubuntu 19.10, full VM'" - "VM_IMAGE=https://cloud-images.ubuntu.com/eoan/current/eoan-server-cloudimg-amd64.img" - # 3.5.0 and 3.5.1 have different __aiter__ semantics than all - # other versions, so we need to test them specially. Travis's - # newer images only provide 3.5.2+, so we have to request the old - # 'trusty' images. - - python: 3.5.0 - dist: trusty - - python: 3.5-dev - python: 3.6-dev - python: 3.7-dev - python: 3.8-dev diff --git a/ci.sh b/ci.sh index e5ad7f8875..23d10210c9 100755 --- a/ci.sh +++ b/ci.sh @@ -55,10 +55,6 @@ if [ "$AGENT_OS" = "Windows_NT" ]; then pydir="$PWD/pyinstall/${PYTHON_PKG}" export PATH="${pydir}/tools:${pydir}/tools/scripts:$PATH" - - # Fix an issue with the nuget python 3.5 packages - # https://github.com/python-trio/trio/pull/827#issuecomment-457433940 - rm -f "${pydir}/tools/pyvenv.cfg" || true fi ### Travis + macOS ### From 128acd8d7305dfa8e0776a5745392982163cb299 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 13:56:56 +0400 Subject: [PATCH 0048/1498] Test an early Python 3.6 version in CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 756d472eb5..576f0b9ae5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ jobs: env: - "JOB_NAME='Ubuntu 19.10, full VM'" - "VM_IMAGE=https://cloud-images.ubuntu.com/eoan/current/eoan-server-cloudimg-amd64.img" + - python: 3.6.1 # earliest 3.6 version available on Travis - python: 3.6-dev - python: 3.7-dev - python: 3.8-dev From d6e48f933c0af76a6f6f2ca552281fe96c0cd4a0 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 28 Apr 2020 17:45:39 +0400 Subject: [PATCH 0049/1498] Prevent Python 3.5 installations --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9ef41cb3e0..35d7500fc4 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,7 @@ # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: include_package_data=True, - python_requires=">=3.5", + python_requires=">=3.6", keywords=["async", "io", "networking", "trio"], classifiers=[ "Development Status :: 3 - Alpha", @@ -107,8 +107,9 @@ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Topic :: System :: Networking", "Framework :: Trio", ], From 609d3748fa3d4ad2382340b3441b254699166e8d Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 10:42:46 +0400 Subject: [PATCH 0050/1498] Stop referencing Python 3.5 in comments/docs --- README.rst | 2 +- docs/source/index.rst | 2 +- docs/source/tutorial.rst | 4 ++-- setup.py | 2 +- trio/_core/_entry_queue.py | 4 ++-- trio/_core/_run.py | 9 +++------ trio/_core/tests/test_ki.py | 4 +--- trio/_core/tests/test_run.py | 4 +--- trio/_highlevel_ssl_helpers.py | 8 ++------ trio/tests/test_path.py | 2 +- trio/tests/test_ssl.py | 7 +++---- 11 files changed, 18 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 4fe3ee5ebe..b1dae85404 100644 --- a/README.rst +++ b/README.rst @@ -102,7 +102,7 @@ demonstration of implementing the "Happy Eyeballs" algorithm in an older library versus Trio. **Cool, but will it work on my system?** Probably! As long as you have -some kind of Python 3.5-or-better (CPython or the latest PyPy3 are +some kind of Python 3.6-or-better (CPython or the latest PyPy3 are both fine), and are using Linux, macOS, or Windows, then Trio should absolutely work. *BSD and illumos likely work too, but we don't have testing infrastructure for them. And all of our dependencies are pure diff --git a/docs/source/index.rst b/docs/source/index.rst index 4f20067785..6e4f2f78e1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -44,7 +44,7 @@ chance to give feedback about any compatibility-breaking changes. Vital statistics: * Supported environments: Linux, macOS, or Windows running some kind of Python - 3.5-or-better (either CPython or PyPy3 is fine). \*BSD and + 3.6-or-better (either CPython or PyPy3 is fine). \*BSD and illumos likely work too, but are untested. * Install: ``python3 -m pip install -U trio`` (or on Windows, maybe diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 3b9255b236..e435966f3b 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -94,7 +94,7 @@ Okay, ready? Let's get started. Before you begin ---------------- -1. Make sure you're using Python 3.5 or newer. +1. Make sure you're using Python 3.6 or newer. 2. ``python3 -m pip install --upgrade trio`` (or on Windows, maybe ``py -3 -m pip install --upgrade trio`` – `details @@ -312,7 +312,7 @@ runs: >>>> # but forcing a garbage collection gives us a warning: >>>> import gc >>>> gc.collect() - /home/njs/pypy-3.5-nightly/lib-python/3/importlib/_bootstrap.py:191: RuntimeWarning: coroutine 'sleep' was never awaited + /home/njs/pypy-3.8-nightly/lib-python/3/importlib/_bootstrap.py:191: RuntimeWarning: coroutine 'sleep' was never awaited if _module_locks.get(name) is wr: # XXX PyPy fix? 0 >>>> diff --git a/setup.py b/setup.py index 35d7500fc4..3d9ba620a8 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ Vital statistics: * Supported environments: Linux, macOS, or Windows running some kind of Python - 3.5-or-better (either CPython or PyPy3 is fine). \\*BSD and illumos likely + 3.6-or-better (either CPython or PyPy3 is fine). \\*BSD and illumos likely work too, but are not tested. * Install: ``python3 -m pip install -U trio`` (or on Windows, maybe diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index 97b1c56fa4..ee20b6e562 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -179,8 +179,8 @@ def run_sync_soon(self, sync_fn, *args, idempotent=False): hashable, and Trio will make a best-effort attempt to discard any call submission which is equal to an already-pending call. Trio will make an attempt to process these in first-in first-out order, - but no guarantees. (Currently processing is FIFO on CPython 3.6 and - PyPy, but not CPython 3.5.) + but no guarantees. (Currently processing is FIFO on CPython and + PyPy.) Any ordering guarantees apply separately to ``idempotent=False`` and ``idempotent=True`` calls; there's no rule for how calls in the diff --git a/trio/_core/_run.py b/trio/_core/_run.py index c1ed65ed25..7f39e9aad8 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1256,12 +1256,9 @@ def _return_value_looks_like_wrong_library(value): # The protocol for detecting an asyncio Future-like object if getattr(value, "_asyncio_future_blocking", None) is not None: return True - # asyncio.Future doesn't have _asyncio_future_blocking until - # 3.5.3. We don't want to import asyncio, but this janky check - # should work well enough for our purposes. And it also catches - # tornado Futures and twisted Deferreds. By the time we're calling - # this function, we already know something has gone wrong, so a - # heuristic is pretty safe. + # This janky check catches tornado Futures and twisted Deferreds. + # By the time we're calling this function, we already know + # something has gone wrong, so a heuristic is pretty safe. if value.__class__.__name__ in ("Future", "Deferred"): return True return False diff --git a/trio/_core/tests/test_ki.py b/trio/_core/tests/test_ki.py index e0d3b97c50..b64057c496 100644 --- a/trio/_core/tests/test_ki.py +++ b/trio/_core/tests/test_ki.py @@ -504,9 +504,7 @@ def test_ki_wakes_us_up(): # # which contains the desired sequence. # - # Affected version of CPython include: - # - all versions of 3.5 (fix will not be backported) - # - 3.6.1 and earlier + # Affected version of CPython include 3.6.1 and earlier. # It's fixed in 3.6.2 and 3.7+ # # PyPy was never affected. diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 3593793fe1..cac776104d 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -38,9 +38,7 @@ async def sleep_forever(): # Some of our tests need to leak coroutines, and thus trigger the # "RuntimeWarning: coroutine '...' was never awaited" message. This context # manager should be used anywhere this happens to hide those messages, because -# (a) when expected they're clutter, (b) on CPython 3.5.x where x < 3, this -# warning can trigger a segfault if we run with warnings turned into errors: -# https://bugs.python.org/issue27811 +# when expected they're clutter. @contextmanager def ignore_coroutine_never_awaited_warnings(): with warnings.catch_warnings(): diff --git a/trio/_highlevel_ssl_helpers.py b/trio/_highlevel_ssl_helpers.py index 9b68e942f4..0df1665e5c 100644 --- a/trio/_highlevel_ssl_helpers.py +++ b/trio/_highlevel_ssl_helpers.py @@ -14,12 +14,8 @@ # if it's one we created, but not OK if it's one that was passed in... and # the one major protocol using NPN/ALPN is HTTP/2, which mandates that you use # a specially configured SSLContext anyway! I also thought maybe we could copy -# the given SSLContext and then mutate the copy, but it's no good: -# copy.copy(SSLContext) seems to succeed, but the state is not transferred! -# For example, with CPython 3.5, we have: -# ctx = ssl.create_default_context() -# assert ctx.check_hostname == True -# assert copy.copy(ctx).check_hostname == False +# the given SSLContext and then mutate the copy, but it's no good as SSLContext +# objects can't be copied: https://bugs.python.org/issue33023. # So... let's punt on that for now. Hopefully we'll be getting a new Python # TLS API soon and can revisit this then. async def open_ssl_over_tcp_stream( diff --git a/trio/tests/test_path.py b/trio/tests/test_path.py index 67d7c2957e..e6045a0a26 100644 --- a/trio/tests/test_path.py +++ b/trio/tests/test_path.py @@ -57,7 +57,7 @@ async def test_cmp_magic(cls_a, cls_b): assert not b == None # noqa -# upstream python3.5 bug: we should also test (pathlib.Path, trio.Path), but +# upstream python3.8 bug: we should also test (pathlib.Path, trio.Path), but # __*div__ does not properly raise NotImplementedError like the other comparison # magic, so trio.Path's implementation does not get dispatched cls_pairs = [ diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 16b323e98b..2111ffb931 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -175,13 +175,12 @@ def __init__(self, sleeper=None): ctx = SSL.Context(SSL.SSLv23_METHOD) # TLS 1.3 removes renegotiation support. Which is great for them, but # we still have to support versions before that, and that means we - # need to test renegotation support, which means we need to force this + # need to test renegotiation support, which means we need to force this # to use a lower version where this test server can trigger # renegotiations. Of course TLS 1.3 support isn't released yet, but # I'm told that this will work once it is. (And once it is we can - # remove the pragma: no cover too.) Alternatively, once we drop - # support for CPython 3.5 on macOS, then we could switch to using - # TLSv1_2_METHOD. + # remove the pragma: no cover too.) Alternatively, we could switch to + # using TLSv1_2_METHOD. # # Discussion: https://github.com/pyca/pyopenssl/issues/624 From b97059b10d97f1aace90e823a83cd3aeb0ffe042 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 15:33:16 +0400 Subject: [PATCH 0051/1498] Replace "3.6 or earlier" with "3.6" Since we no longer support Python 3.5. --- trio/_core/_run.py | 4 ++-- trio/_util.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 7f39e9aad8..10606c710a 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -60,7 +60,7 @@ def _public(fn): # On 3.7+, Context.run() is implemented in C and doesn't show up in -# tracebacks. On 3.6 and earlier, we use the contextvars backport, which is +# tracebacks. On 3.6, we use the contextvars backport, which is # currently implemented in Python and adds 1 frame to tracebacks. So this # function is a super-overkill version of "0 if sys.version_info >= (3, 7) # else 1". But if Context.run ever changes, we'll be ready! @@ -1906,7 +1906,7 @@ def run_impl(runner, async_fn, args): try: # We used to unwrap the Outcome object here and send/throw its # contents in directly, but it turns out that .throw() is - # buggy, at least on CPython 3.6 and earlier: + # buggy, at least on CPython 3.6: # https://bugs.python.org/issue29587 # https://bugs.python.org/issue29590 # So now we send in the Outcome object and unwrap it on the diff --git a/trio/_util.py b/trio/_util.py index 9204b83349..b48d080978 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -266,7 +266,7 @@ def __getitem__(self, _): # If a new class inherits from any ABC, then the new class's metaclass has to # inherit from ABCMeta. If a new class inherits from typing.Generic, and -# you're using Python 3.6 or earlier, then the new class's metaclass has to +# you're using Python 3.6, then the new class's metaclass has to # inherit from typing.GenericMeta. Some of the classes that want to use Final # or NoPublicConstructor inherit from ABCs and generics, so Final has to # inherit from these metaclasses. Fortunately, GenericMeta inherits from From c38265cf798f7cbff23778bdc78f5108f120391b Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 11:03:58 +0400 Subject: [PATCH 0052/1498] Stop unwrapping paths now that Python 3.5 is gone --- trio/_path.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/trio/_path.py b/trio/_path.py index bbadf9d874..fbe9db26f6 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -9,17 +9,6 @@ __all__ = ['Path'] -# python3.5 compat: __fspath__ does not exist in 3.5, so unwrap any trio.Path -# being passed to any wrapped method -def unwrap_paths(args): - new_args = [] - for arg in args: - if isinstance(arg, Path): - arg = arg._wrapped - new_args.append(arg) - return new_args - - # re-wrap return value from methods that return new instances of pathlib.Path def rewrap_path(value): if isinstance(value, pathlib.Path): @@ -30,7 +19,6 @@ def rewrap_path(value): def _forward_factory(cls, attr_name, attr): @wraps(attr) def wrapper(self, *args, **kwargs): - args = unwrap_paths(args) attr = getattr(self._wrapped, attr_name) value = attr(*args, **kwargs) return rewrap_path(value) @@ -69,7 +57,6 @@ async def wrapper(self, *args, **kwargs): def thread_wrapper_factory(cls, meth_name): @async_wraps(cls, cls._wraps, meth_name) async def wrapper(self, *args, **kwargs): - args = unwrap_paths(args) meth = getattr(self._wrapped, meth_name) func = partial(meth, *args, **kwargs) value = await trio.to_thread.run_sync(func) @@ -82,7 +69,6 @@ def classmethod_wrapper_factory(cls, meth_name): @classmethod @async_wraps(cls, cls._wraps, meth_name) async def wrapper(cls, *args, **kwargs): - args = unwrap_paths(args) meth = getattr(cls._wraps, meth_name) func = partial(meth, *args, **kwargs) value = await trio.to_thread.run_sync(func) @@ -168,8 +154,6 @@ class Path(metaclass=AsyncAutoWrapperType): _wrap_iter = ['glob', 'rglob', 'iterdir'] def __init__(self, *args): - args = unwrap_paths(args) - self._wrapped = pathlib.Path(*args) def __getattr__(self, name): From 24846dd4d13f608f9b2a316f0cbd80965cbbb127 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 11:05:22 +0400 Subject: [PATCH 0053/1498] Remove fspath backport now that Python 3.5 is gone --- trio/_file_io.py | 6 +- trio/_highlevel_open_unix_stream.py | 3 +- trio/_path.py | 8 +-- trio/_socket.py | 3 +- trio/_util.py | 60 ------------------ trio/tests/test_file_io.py | 4 +- trio/tests/test_path.py | 3 +- trio/tests/test_util.py | 95 +---------------------------- 8 files changed, 13 insertions(+), 169 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 32468af46b..ca1f003741 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -2,7 +2,7 @@ import io from .abc import AsyncResource -from ._util import aiter_compat, async_wraps, fspath +from ._util import aiter_compat, async_wraps import trio @@ -159,10 +159,6 @@ async def open_file( :func:`trio.Path.open` """ - # python3.5 compat - if isinstance(file, trio.Path): - file = fspath(file) - _file = wrap_file( await trio.to_thread.run_sync( io.open, file, mode, buffering, encoding, errors, newline, closefd, diff --git a/trio/_highlevel_open_unix_stream.py b/trio/_highlevel_open_unix_stream.py index 59141ebc38..08eef0c135 100644 --- a/trio/_highlevel_open_unix_stream.py +++ b/trio/_highlevel_open_unix_stream.py @@ -1,3 +1,4 @@ +import os from contextlib import contextmanager import trio @@ -44,6 +45,6 @@ async def open_unix_socket(filename,): # possible location to connect to sock = socket(AF_UNIX, SOCK_STREAM) with close_on_error(sock): - await sock.connect(trio._util.fspath(filename)) + await sock.connect(os.fspath(filename)) return trio.SocketStream(sock) diff --git a/trio/_path.py b/trio/_path.py index fbe9db26f6..2f34a38972 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -4,7 +4,7 @@ import pathlib import trio -from trio._util import async_wraps, fspath +from trio._util import async_wraps __all__ = ['Path'] @@ -169,7 +169,7 @@ def __repr__(self): return 'trio.Path({})'.format(repr(str(self))) def __fspath__(self): - return fspath(self._wrapped) + return os.fspath(self._wrapped) @wraps(pathlib.Path.open) async def open(self, *args, **kwargs): @@ -203,6 +203,4 @@ async def open(self, *args, **kwargs): # sense than inventing our own special docstring for this. del Path.absolute.__doc__ -# python3.5 compat -if hasattr(os, 'PathLike'): - os.PathLike.register(Path) +os.PathLike.register(Path) diff --git a/trio/_socket.py b/trio/_socket.py index 18403962d2..fe9ca242c1 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -7,7 +7,6 @@ import idna as _idna import trio -from ._util import fspath from . import _core @@ -512,7 +511,7 @@ async def _resolve_address(self, address, flags): elif self._sock.family == _stdlib_socket.AF_UNIX: await trio.hazmat.checkpoint() # unwrap path-likes - return fspath(address) + return _os.fspath(address) else: await trio.hazmat.checkpoint() diff --git a/trio/_util.py b/trio/_util.py index b48d080978..ce39d76bf9 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -177,66 +177,6 @@ def fix_one(qualname, name, obj): fix_one(objname, objname, obj) -# os.fspath is defined on Python 3.6+ but we need to support Python 3.5 too -# This is why we provide our own implementation. On Python 3.6+ we use the -# StdLib's version and on Python 3.5 our own version. -# Our own implementation implementation is based on PEP 519 while it has also -# been adapted to work with pathlib objects on python 3.5 -# The input typehint is removed as there is no os.PathLike on 3.5. -# See: https://www.python.org/dev/peps/pep-0519/#os - - -def fspath(path) -> t.Union[str, bytes]: - """Return the path representation of a path-like object. - - Returns - ------- - - If str or bytes is passed in, it is returned unchanged. - - If the os.PathLike interface is implemented it is used to get the path - representation. - - If the python version is 3.5 or earlier and a pathlib object is passed, - the object's string representation is returned. - - Raises - ------ - - Regardless of the input, if the path representation (e.g. the value - returned from __fspath__) is not str or bytes, TypeError is raised. - - If the provided path is not str, bytes, pathlib.PurePath or os.PathLike, - TypeError is raised. - """ - if isinstance(path, (str, bytes)): - return path - # Work from the object's type to match method resolution of other magic - # methods. - path_type = type(path) - # On python 3.5, pathlib objects don't have the __fspath__ method, - # but we still want to get their string representation. - if issubclass(path_type, pathlib.PurePath): - return str(path) - try: - path_repr = path_type.__fspath__(path) - except AttributeError: - if hasattr(path_type, '__fspath__'): - raise - else: - raise TypeError( - "expected str, bytes or os.PathLike object, " - "not " + path_type.__name__ - ) - if isinstance(path_repr, (str, bytes)): - return path_repr - else: - raise TypeError( - "expected {}.__fspath__() to return str or bytes, " - "not {}".format(path_type.__name__, - type(path_repr).__name__) - ) - - -if hasattr(os, "fspath"): - fspath = os.fspath # noqa - - class generic_function: """Decorator that makes a function indexable, to communicate non-inferrable generic type parameters to a static type checker. diff --git a/trio/tests/test_file_io.py b/trio/tests/test_file_io.py index 8f96e75aac..fd4fa648b4 100644 --- a/trio/tests/test_file_io.py +++ b/trio/tests/test_file_io.py @@ -1,4 +1,5 @@ import io +import os import pytest from unittest import mock @@ -6,13 +7,12 @@ import trio from trio import _core -from trio._util import fspath from trio._file_io import AsyncIOWrapper, _FILE_SYNC_ATTRS, _FILE_ASYNC_METHODS @pytest.fixture def path(tmpdir): - return fspath(tmpdir.join('test')) + return os.fspath(tmpdir.join('test')) @pytest.fixture diff --git a/trio/tests/test_path.py b/trio/tests/test_path.py index e6045a0a26..9bbbfd4df2 100644 --- a/trio/tests/test_path.py +++ b/trio/tests/test_path.py @@ -5,7 +5,6 @@ import trio from trio._path import AsyncAutoWrapperType as Type -from trio._util import fspath from trio._file_io import AsyncIOWrapper @@ -203,7 +202,7 @@ async def test_path_nonpath(): async def test_open_file_can_open_path(path): async with await trio.open_file(path, 'w') as f: - assert f.name == fspath(path) + assert f.name == os.fspath(path) async def test_globmethods(path): diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 5815ae3e42..f0f25a2ce6 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -1,23 +1,14 @@ -import os -import pathlib import signal -import sys import pytest import trio from .. import _core from .._util import ( - signal_raise, ConflictDetector, fspath, is_main_thread, generic_function, - Final, NoPublicConstructor + signal_raise, ConflictDetector, is_main_thread, generic_function, Final, + NoPublicConstructor ) -from ..testing import wait_all_tasks_blocked, assert_checkpoints - - -def raise_(exc): - """ Raise provided exception. - Just a helper for raising exceptions from lambdas. """ - raise exc +from ..testing import wait_all_tasks_blocked def test_signal_raise(): @@ -82,86 +73,6 @@ def test_module_metadata_is_fixed_up(): assert trio.to_thread.run_sync.__qualname__ == "run_sync" -# define a concrete class implementing the PathLike protocol -# Since we want to have compatibility with Python 3.5 we need -# to define the base class on runtime. -BaseKlass = os.PathLike if hasattr(os, "PathLike") else object - - -class ConcretePathLike(BaseKlass): - """ Class implementing the file system path protocol.""" - def __init__(self, path=""): - self.path = path - - def __fspath__(self): - return self.path - - -class TestFspath: - - # based on: - # https://github.com/python/cpython/blob/da6c3da6c33c6bf794f741e348b9c6d86cc43ec5/Lib/test/test_os.py#L3527-L3571 - - @pytest.mark.parametrize( - "path", (b'hello', b'goodbye', b'some/path/and/file') - ) - def test_return_bytes(self, path): - assert path == fspath(path) - - @pytest.mark.parametrize( - "path", ('hello', 'goodbye', 'some/path/and/file') - ) - def test_return_string(self, path): - assert path == fspath(path) - - @pytest.mark.parametrize( - "path", (pathlib.Path("/home"), pathlib.Path("C:\\windows")) - ) - def test_handle_pathlib(self, path): - assert str(path) == fspath(path) - - @pytest.mark.parametrize("path", ("path/like/object", b"path/like/object")) - def test_handle_pathlike_protocol(self, path): - pathlike = ConcretePathLike(path) - assert path == fspath(pathlike) - if sys.version_info > (3, 6): - assert issubclass(ConcretePathLike, os.PathLike) - assert isinstance(pathlike, os.PathLike) - - def test_argument_required(self): - with pytest.raises(TypeError): - fspath() - - def test_throw_error_at_multiple_arguments(self): - with pytest.raises(TypeError): - fspath(1, 2) - - @pytest.mark.parametrize( - "klass", (23, object(), int, type, os, type("blah", (), {})()) - ) - def test_throw_error_at_non_pathlike(self, klass): - with pytest.raises(TypeError): - fspath(klass) - - @pytest.mark.parametrize( - "exception, method", - [ - (TypeError, 1), # __fspath__ is not callable - (TypeError, lambda x: 23 - ), # __fspath__ returns a value other than str or bytes - (Exception, lambda x: raise_(Exception) - ), # __fspath__raises a random exception - (AttributeError, lambda x: raise_(AttributeError) - ), # __fspath__ raises AttributeError - ] - ) - def test_bad_pathlike_implementation(self, exception, method): - klass = type('foo', (), {}) - klass.__fspath__ = method - with pytest.raises(exception): - fspath(klass()) - - async def test_is_main_thread(): assert is_main_thread() From 26e681feb106c31f40bcb9141dc903339d515784 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 11:08:21 +0400 Subject: [PATCH 0054/1498] Remove aiter_compat now that Python 3.5 is gone --- trio/_abc.py | 3 --- trio/_core/_unbounded_queue.py | 2 -- trio/_core/tests/test_run.py | 2 -- trio/_file_io.py | 3 +-- trio/_signals.py | 5 +---- trio/_sync.py | 1 - trio/_util.py | 18 ------------------ 7 files changed, 2 insertions(+), 32 deletions(-) diff --git a/trio/_abc.py b/trio/_abc.py index 88c2ff1f70..8064e71169 100644 --- a/trio/_abc.py +++ b/trio/_abc.py @@ -1,6 +1,5 @@ from abc import ABCMeta, abstractmethod from typing import Generic, TypeVar -from ._util import aiter_compat import trio @@ -414,7 +413,6 @@ async def receive_some(self, max_bytes=None): """ - @aiter_compat def __aiter__(self): return self @@ -629,7 +627,6 @@ async def receive(self) -> ReceiveType: """ - @aiter_compat def __aiter__(self): return self diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index f5e2dda5c3..abe4d60c56 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -1,7 +1,6 @@ import attr from .. import _core -from .._util import aiter_compat from .._deprecate import deprecated __all__ = ["UnboundedQueue"] @@ -146,7 +145,6 @@ def statistics(self): tasks_waiting=self._lot.statistics().tasks_waiting ) - @aiter_compat def __aiter__(self): return self diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index cac776104d..e44cdbfb08 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -20,7 +20,6 @@ from ... import _core from ..._threads import to_thread_run_sync from ..._timeouts import sleep, fail_after -from ..._util import aiter_compat from ...testing import ( wait_all_tasks_blocked, Sequencer, @@ -2047,7 +2046,6 @@ def __init__(self, *largs): async def _accumulate(self, f, items, i): items[i] = await f() - @aiter_compat def __aiter__(self): return self diff --git a/trio/_file_io.py b/trio/_file_io.py index ca1f003741..10464589a9 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -2,7 +2,7 @@ import io from .abc import AsyncResource -from ._util import aiter_compat, async_wraps +from ._util import async_wraps import trio @@ -95,7 +95,6 @@ def __dir__(self): ) return attrs - @aiter_compat def __aiter__(self): return self diff --git a/trio/_signals.py b/trio/_signals.py index 2ebb4a0a5a..2f2793c43e 100644 --- a/trio/_signals.py +++ b/trio/_signals.py @@ -3,9 +3,7 @@ from collections import OrderedDict import trio -from ._util import ( - signal_raise, aiter_compat, is_main_thread, ConflictDetector -) +from ._util import signal_raise, is_main_thread, ConflictDetector __all__ = ["open_signal_receiver"] @@ -96,7 +94,6 @@ def deliver_next(): def _pending_signal_count(self): return len(self._pending) - @aiter_compat def __aiter__(self): return self diff --git a/trio/_sync.py b/trio/_sync.py index f34acd0858..e4af1be178 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -5,7 +5,6 @@ import trio -from ._util import aiter_compat from ._core import enable_ki_protection, ParkingLot from ._deprecate import deprecated diff --git a/trio/_util.py b/trio/_util.py index ce39d76bf9..088c4f8b37 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -68,24 +68,6 @@ def signal_raise(signum): signal.pthread_kill(threading.get_ident(), signum) -# Decorator to handle the change to __aiter__ in 3.5.2 -if sys.version_info < (3, 5, 2): - - def aiter_compat(aiter_impl): - # de-sugar decorator to fix Python 3.8 coverage issue - # https://github.com/python-trio/trio/pull/784#issuecomment-446438407 - async def __aiter__(*args, **kwargs): - return aiter_impl(*args, **kwargs) - - __aiter__ = wraps(aiter_impl)(__aiter__) - - return __aiter__ -else: - - def aiter_compat(aiter_impl): - return aiter_impl - - # See: #461 as to why this is needed. # The gist is that threading.main_thread() has the capability to lie to us # if somebody else edits the threading ident cache to replace the main From cb17cb4baec3042809d60476c6375cd1e314ab91 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 11:09:27 +0400 Subject: [PATCH 0055/1498] Assume enum.IntFlag exists now that Python 3.5 is gone --- trio/_core/_windows_cffi.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/trio/_core/_windows_cffi.py b/trio/_core/_windows_cffi.py index e2b95a9113..58845b86f4 100644 --- a/trio/_core/_windows_cffi.py +++ b/trio/_core/_windows_cffi.py @@ -1,10 +1,6 @@ import cffi import re import enum -try: - from enum import IntFlag -except ImportError: # python 3.5 - from enum import IntEnum as IntFlag ################################################################ # Functions and types @@ -264,7 +260,7 @@ class FileFlags(enum.IntEnum): TRUNCATE_EXISTING = 5 -class AFDPollFlags(IntFlag): +class AFDPollFlags(enum.IntFlag): # These are drawn from a combination of: # https://github.com/piscisaureus/wepoll/blob/master/src/afd.h # https://github.com/reactos/reactos/blob/master/sdk/include/reactos/drivers/afd/shared.h @@ -289,7 +285,7 @@ class WSAIoctls(enum.IntEnum): SIO_BSP_HANDLE_SELECT = 0x4800001C -class CompletionModes(IntFlag): +class CompletionModes(enum.IntFlag): FILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 0x1 FILE_SKIP_SET_EVENT_ON_HANDLE = 0x2 From 9c2cb7324c1f0ec641fa76b32c5cd89ca64002ee Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 11:11:46 +0400 Subject: [PATCH 0056/1498] Remove unneeded filterwarnings now that Python 3.5 is gone --- trio/tests/test_exports.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 3650403179..e7d894c12d 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -64,10 +64,6 @@ def public_namespaces(module): # https://github.com/PyCQA/astroid/issues/681 "ignore:the imp module is deprecated.*:DeprecationWarning" ) -@pytest.mark.filterwarnings( - # Same as above, but on Python 3.5 - "ignore:the imp module is deprecated.*:PendingDeprecationWarning" -) @pytest.mark.parametrize("modname", NAMESPACES) @pytest.mark.parametrize("tool", ["pylint", "jedi"]) def test_static_tool_sees_all_symbols(tool, modname): From 91747a074adec8f1b3db513849987ae4eff15d90 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 11:32:35 +0400 Subject: [PATCH 0057/1498] Guarantee run_sync_soon ordering now that Python 3.5 is gone --- trio/_core/_entry_queue.py | 4 +--- trio/_core/tests/test_run.py | 6 ------ 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index ee20b6e562..6833cc886f 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -178,9 +178,7 @@ def run_sync_soon(self, sync_fn, *args, idempotent=False): If ``idempotent=True``, then ``sync_fn`` and ``args`` must be hashable, and Trio will make a best-effort attempt to discard any call submission which is equal to an already-pending call. Trio - will make an attempt to process these in first-in first-out order, - but no guarantees. (Currently processing is FIFO on CPython and - PyPy.) + will process these in first-in first-out order. Any ordering guarantees apply separately to ``idempotent=False`` and ``idempotent=True`` calls; there's no rule for how calls in the diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index e44cdbfb08..a98e11ed8c 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1377,12 +1377,6 @@ def cb(x): for i in range(100): token.run_sync_soon(cb, i, idempotent=True) await wait_all_tasks_blocked() - if ( - sys.version_info < (3, 6) - and platform.python_implementation() == "CPython" - ): - # no order guarantees - record.sort() # Otherwise, we guarantee FIFO assert record == list(range(100)) From d22efed483b192b11c80c748871792ceba1ef986 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 14:41:08 +0400 Subject: [PATCH 0058/1498] Assume Python 3.6 ssl constants exist Since we no longer support Python 3.5. --- trio/_deprecated_ssl_reexports.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/trio/_deprecated_ssl_reexports.py b/trio/_deprecated_ssl_reexports.py index f86077e3fe..35d22f49f4 100644 --- a/trio/_deprecated_ssl_reexports.py +++ b/trio/_deprecated_ssl_reexports.py @@ -15,15 +15,10 @@ cert_time_to_seconds, CertificateError, create_default_context, DER_cert_to_PEM_cert, get_default_verify_paths, match_hostname, PEM_cert_to_DER_cert, Purpose, SSLEOFError, SSLError, SSLSyscallError, - SSLZeroReturnError + SSLZeroReturnError, AlertDescription, SSLErrorNumber, SSLSession, + VerifyFlags, VerifyMode, Options ) -# Added in python 3.6 -try: - from ssl import AlertDescription, SSLErrorNumber, SSLSession, VerifyFlags, VerifyMode, Options # noqa -except ImportError: - pass - # Added in python 3.7 try: from ssl import SSLCertVerificationError, TLSVersion # noqa From b7c3896c52a626121f7c748a8c3beb5e22b4bf87 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 15:34:00 +0400 Subject: [PATCH 0059/1498] Stop considering Python 3.5 SyntaxError --- trio/_core/tests/test_run.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index a98e11ed8c..3f8f40c60b 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2380,22 +2380,18 @@ def test_async_function_implemented_in_C(): # cr_frame, but C functions don't have Python frames. ns = {"_core": _core} - try: - exec( - dedent( - """ - async def agen_fn(record): - assert not _core.currently_ki_protected() - record.append("the generator ran") - yield - """ - ), - ns, - ) - except SyntaxError: - pytest.skip("Requires Python 3.6+") - else: - agen_fn = ns["agen_fn"] + exec( + dedent( + """ + async def agen_fn(record): + assert not _core.currently_ki_protected() + record.append("the generator ran") + yield + """ + ), + ns, + ) + agen_fn = ns["agen_fn"] run_record = [] agen = agen_fn(run_record) From 704d0734f8dcd06c1b1ac385f3db4d9ce6e563f5 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 5 Feb 2020 15:43:28 +0400 Subject: [PATCH 0060/1498] Use native async generators now that Python 3.5 is gone KeyboardInterrupt protection is still enabled for async_generator, since users could still do this with Python 3.6. We can always deprecate this later. --- trio/_core/tests/test_ki.py | 19 +++++++++++++++++++ trio/_core/tests/test_run.py | 4 +--- trio/_util.py | 2 -- trio/testing/_sequencer.py | 5 ++--- trio/tests/test_ssl.py | 12 ++++-------- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/trio/_core/tests/test_ki.py b/trio/_core/tests/test_ki.py index b64057c496..a63407484a 100644 --- a/trio/_core/tests/test_ki.py +++ b/trio/_core/tests/test_ki.py @@ -176,11 +176,30 @@ async def agen_unprotected2(): finally: assert not _core.currently_ki_protected() + # Native async generators + @_core.enable_ki_protection + async def agen_protected3(): + assert _core.currently_ki_protected() + try: + yield + finally: + assert _core.currently_ki_protected() + + @_core.disable_ki_protection + async def agen_unprotected3(): + assert not _core.currently_ki_protected() + try: + yield + finally: + assert not _core.currently_ki_protected() + for agen_fn in [ agen_protected1, agen_protected2, + agen_protected3, agen_unprotected1, agen_unprotected2, + agen_unprotected3, ]: async for _ in agen_fn(): # noqa assert not _core.currently_ki_protected() diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 3f8f40c60b..ee31799ea1 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -14,7 +14,6 @@ import outcome import sniffio import pytest -from async_generator import async_generator from .tutil import slow, check_sequence_matches, gc_collect_harder from ... import _core @@ -1753,9 +1752,8 @@ def generator_based_coro(): # pragma: no cover bad_call(len, [1, 2, 3]) assert "appears to be synchronous" in str(excinfo.value) - @async_generator async def async_gen(arg): # pragma: no cover - pass + yield with pytest.raises(TypeError) as excinfo: bad_call(async_gen, 0) diff --git a/trio/_util.py b/trio/_util.py index 088c4f8b37..b50a58036e 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -9,8 +9,6 @@ import typing as t import threading -import async_generator - # There's a dependency loop here... _core is allowed to use this file (in fact # it's the *only* file in the main trio/ package it's allowed to use), but # ConflictDetector needs checkpoint so it also has to import diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index 118969f83d..05b7560eec 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -1,7 +1,7 @@ from collections import defaultdict import attr -from async_generator import async_generator, yield_, asynccontextmanager +from async_generator import asynccontextmanager from .. import _core from .. import _util @@ -61,7 +61,6 @@ async def main(): _broken = attr.ib(default=False, init=False) @asynccontextmanager - @async_generator async def __call__(self, position: int): if position in self._claimed: raise RuntimeError( @@ -84,6 +83,6 @@ async def __call__(self, position: int): if self._broken: raise RuntimeError("sequence broken!") try: - await yield_() + yield finally: self._sequence_points[position + 1].set() diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 2111ffb931..9cdad56fe3 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -8,7 +8,7 @@ from OpenSSL import SSL import trustme -from async_generator import async_generator, yield_, asynccontextmanager +from async_generator import asynccontextmanager import trio from .. import _core @@ -138,7 +138,6 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): # (running in a thread). Useful for testing making connections with different # SSLContexts. @asynccontextmanager -@async_generator async def ssl_echo_server_raw(**kwargs): a, b = stdlib_socket.socketpair() async with trio.open_nursery() as nursery: @@ -151,19 +150,16 @@ async def ssl_echo_server_raw(**kwargs): partial(ssl_echo_serve_sync, b, **kwargs) ) - await yield_(SocketStream(tsocket.from_stdlib_socket(a))) + yield SocketStream(tsocket.from_stdlib_socket(a)) # Fixture that gives a properly set up SSLStream connected to a trio-test-1 # echo server (running in a thread) @asynccontextmanager -@async_generator async def ssl_echo_server(client_ctx, **kwargs): async with ssl_echo_server_raw(**kwargs) as sock: - await yield_( - SSLStream( - sock, client_ctx, server_hostname="trio-test-1.example.org" - ) + yield SSLStream( + sock, client_ctx, server_hostname="trio-test-1.example.org" ) From 3466d838d42df6b7304437807b50e87f4f34da84 Mon Sep 17 00:00:00 2001 From: Kyle Lawlor Date: Sat, 15 Feb 2020 18:37:09 -0500 Subject: [PATCH 0061/1498] Remove Python 3.5 deprecation warning --- trio/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index 679c8509be..1ad0e25362 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -169,10 +169,3 @@ __name__ + ".subprocess", _deprecated_subprocess_reexports.__dict__ ) del fixup_module_metadata - -import sys -if sys.version_info < (3, 6): - _deprecate.warn_deprecated( - "Support for Python 3.5", "0.14", issue=75, instead="Python 3.6+" - ) -del sys From 0097e3de89bfa94ed672e5f6d2d658713a61fe58 Mon Sep 17 00:00:00 2001 From: Kyle Lawlor Date: Wed, 19 Feb 2020 11:28:11 -0500 Subject: [PATCH 0062/1498] Expect constants supported by PyPy and CPython 3.6+ --- trio/_socket.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/trio/_socket.py b/trio/_socket.py index fe9ca242c1..a3ffe0315f 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -295,13 +295,7 @@ def _sniff_sockopts_for_fileno(family, type, proto, fileno): # and then we'll throw it away and construct a new one with the correct metadata. if not _sys.platform == "linux": return family, type, proto - try: - from socket import SO_DOMAIN, SO_PROTOCOL - except ImportError: - # Only available on 3.6 and above: - SO_PROTOCOL = 38 - SO_DOMAIN = 39 - from socket import SOL_SOCKET, SO_TYPE + from socket import SO_DOMAIN, SO_PROTOCOL, SOL_SOCKET, SO_TYPE sockobj = _stdlib_socket.socket(family, type, proto, fileno=fileno) try: family = sockobj.getsockopt(SOL_SOCKET, SO_DOMAIN) From ebb746dbab156c21b223f3d2b29c2ccfcb91a24d Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 29 Apr 2020 11:28:59 +0400 Subject: [PATCH 0063/1498] Add newsfragment --- newsfragments/75.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/75.removal.rst diff --git a/newsfragments/75.removal.rst b/newsfragments/75.removal.rst new file mode 100644 index 0000000000..b239384710 --- /dev/null +++ b/newsfragments/75.removal.rst @@ -0,0 +1 @@ +Remove support for Python 3.5. From d4715fc5ffbf8c08ebd4f508dc704d6a67264043 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 29 Apr 2020 14:22:19 -0700 Subject: [PATCH 0064/1498] Empty commit to attempt to un-wedge github From 5422a9e85663ed6b47000883171f283285e7ca73 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 29 Apr 2020 17:12:30 -0700 Subject: [PATCH 0065/1498] Remove stray word left in comment --- trio/_core/tests/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index ee31799ea1..4a263581f4 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1376,7 +1376,7 @@ def cb(x): for i in range(100): token.run_sync_soon(cb, i, idempotent=True) await wait_all_tasks_blocked() - # Otherwise, we guarantee FIFO + # We guarantee FIFO assert record == list(range(100)) From 1f6068617a3a7584fd167d923805eed388325c1d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 29 Apr 2020 17:18:13 -0700 Subject: [PATCH 0066/1498] Remove exec() trick used to hide code from py35 Now that we're dropping 3.5 support, we can just write the code directly in the source file. --- trio/_core/tests/test_run.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 4a263581f4..fa399d93b4 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2377,19 +2377,10 @@ def test_async_function_implemented_in_C(): # These used to crash because we'd try to mutate the coroutine object's # cr_frame, but C functions don't have Python frames. - ns = {"_core": _core} - exec( - dedent( - """ - async def agen_fn(record): - assert not _core.currently_ki_protected() - record.append("the generator ran") - yield - """ - ), - ns, - ) - agen_fn = ns["agen_fn"] + async def agen_fn(record): + assert not _core.currently_ki_protected() + record.append("the generator ran") + yield run_record = [] agen = agen_fn(run_record) From 3b4e8aaadeb99bd84b7e346635b445f86cdcb918 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2020 18:32:36 +0000 Subject: [PATCH 0067/1498] Bump ipython from 7.9.0 to 7.13.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.9.0 to 7.13.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.9.0...7.13.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b912d752d3..df9041404a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -18,7 +18,7 @@ flake8==3.7.9 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme immutables==0.12 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets -ipython==7.9.0 # via -r test-requirements.in +ipython==7.13.0 # via -r test-requirements.in isort==4.3.21 # via pylint jedi==0.17.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From 22038ac8516178480af7096acae11d2f7b711bb5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 30 Apr 2020 17:40:58 -0700 Subject: [PATCH 0068/1498] =?UTF-8?q?Rename=20trio.hazmat=20=E2=86=92=20tr?= =?UTF-8?q?io.lowlevel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: gh-476 --- docs/source/conf.py | 2 +- docs/source/design.rst | 9 +- docs/source/history.rst | 82 +++++++++---------- docs/source/index.rst | 2 +- docs/source/reference-core.rst | 4 +- ...ence-hazmat.rst => reference-lowlevel.rst} | 22 ++--- newsfragments/476.removal.rst | 3 + notes-to-self/afd-lab.py | 4 +- notes-to-self/print-task-tree.py | 2 +- notes-to-self/schedule-timing.py | 2 +- notes-to-self/socket-scaling.py | 4 +- notes-to-self/tiny-thread-pool.py | 2 +- notes-to-self/trace.py | 2 +- trio/__init__.py | 23 ++++-- trio/_abc.py | 10 +-- trio/_channel.py | 40 ++++----- trio/_core/__init__.py | 2 +- trio/_core/_entry_queue.py | 4 +- trio/_core/_exceptions.py | 2 +- trio/_core/_generated_run.py | 10 +-- trio/_core/_parking_lot.py | 4 +- trio/_core/_run.py | 16 ++-- trio/_core/_traps.py | 10 +-- trio/_core/_unbounded_queue.py | 2 +- trio/_core/tests/test_io.py | 24 +++--- trio/_file_io.py | 2 +- trio/_highlevel_socket.py | 8 +- trio/_signals.py | 6 +- trio/_socket.py | 10 +-- trio/_ssl.py | 12 +-- trio/_subprocess.py | 4 +- trio/_subprocess_platform/__init__.py | 2 +- trio/_sync.py | 46 +++++------ trio/_threads.py | 20 ++--- trio/_timeouts.py | 6 +- trio/_unix_pipes.py | 16 ++-- trio/{hazmat.py => lowlevel.py} | 0 trio/tests/test_exports.py | 4 +- trio/tests/test_highlevel_serve_listeners.py | 4 +- trio/tests/test_util.py | 4 +- 40 files changed, 221 insertions(+), 210 deletions(-) rename docs/source/{reference-hazmat.rst => reference-lowlevel.rst} (96%) create mode 100644 newsfragments/476.removal.rst rename trio/{hazmat.py => lowlevel.py} (100%) diff --git a/docs/source/conf.py b/docs/source/conf.py index 9870a88564..9ec78d6ea9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -32,7 +32,7 @@ ("py:class", "bytes-like"), ("py:class", "None"), # Was removed but still shows up in changelog - ("py:class", "trio.hazmat.RunLocal"), + ("py:class", "trio.lowlevel.RunLocal"), # trio.abc is documented at random places scattered throughout the docs ("py:mod", "trio.abc"), ("py:class", "math.inf"), diff --git a/docs/source/design.rst b/docs/source/design.rst index eb41d55c45..19db929101 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -437,8 +437,9 @@ and then the other ``trio.*`` modules are implemented in terms of the API it exposes. (If you want to see what this API looks like, then ``import trio; print(trio._core.__all__)``). Everything exported from ``trio._core`` is *also* exported as part of the ``trio``, -``trio.hazmat``, or ``trio.testing`` namespaces. (See their respective -``__init__.py`` files for details; there's a test to enforce this.) +``trio.lowlevel``, or ``trio.testing`` namespaces. (See their +respective ``__init__.py`` files for details; there's a test to +enforce this.) Rationale: currently, Trio is a new project in a novel part of the design space, so we don't make any stability guarantees. But the goal @@ -474,7 +475,7 @@ The most important submodule, where everything is integrated, is ``_run.py``. (This is also by far the largest submodule; it'd be nice to factor bits of it out where possible, but it's tricky because the core functionality genuinely is pretty intertwined.) Notably, this is -where cancel scopes, nurseries, and :class:`~trio.hazmat.Task` are +where cancel scopes, nurseries, and :class:`~trio.lowlevel.Task` are defined; it's also where the scheduler state and :func:`trio.run` live. @@ -524,7 +525,7 @@ several reasons: The ``IOManager`` layer provides a fairly raw exposure of the capabilities of each system, with public API functions that vary between different backends. (This is somewhat inspired by how :mod:`os` works.) These -public APIs are then exported as part of :mod:`trio.hazmat`, and +public APIs are then exported as part of :mod:`trio.lowlevel`, and higher-level APIs like :mod:`trio.socket` abstract over these system-specific APIs to provide a uniform experience. diff --git a/docs/source/history.rst b/docs/source/history.rst index 65bb5dd54c..8c119a59cc 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -12,20 +12,20 @@ Features ~~~~~~~~ - If you're using Trio's low-level interfaces like - `trio.hazmat.wait_readable` or similar, and then you close a socket or - file descriptor, you're supposed to call `trio.hazmat.notify_closing` + `trio.lowlevel.wait_readable` or similar, and then you close a socket or + file descriptor, you're supposed to call `trio.lowlevel.notify_closing` first so Trio can clean up properly. But what if you forget? In the past, Trio would tend to either deadlock or explode spectacularly. Now, it's much more robust to this situation, and should generally survive. (But note that "survive" is not the same as "give you the results you were expecting", so you should still call - `~trio.hazmat.notify_closing` when appropriate. This is about harm + `~trio.lowlevel.notify_closing` when appropriate. This is about harm reduction and making it easier to debug this kind of mistake, not something you should rely on.) - If you're using higher-level interfaces outside of the `trio.hazmat` + If you're using higher-level interfaces outside of the `trio.lowlevel` module, then you don't need to worry about any of this; those - intefaces already take care of calling `~trio.hazmat.notify_closing` + intefaces already take care of calling `~trio.lowlevel.notify_closing` for you. (`#1272 `__) @@ -60,8 +60,8 @@ Features `__ is generally the best way to implement async I/O operations – but it's historically been weak at providing ``select``\-style readiness - notifications, like `trio.hazmat.wait_readable` and - `~trio.hazmat.wait_writable`. We aren't willing to give those up, so + notifications, like `trio.lowlevel.wait_readable` and + `~trio.lowlevel.wait_writable`. We aren't willing to give those up, so previously Trio's Windows backend used a hybrid of ``select`` + IOCP. This was complex, slow, and had `limited scalability `__. @@ -162,7 +162,7 @@ Features like `current_time`. Static analysis tools like mypy and pylint should now be able to recognize and analyze all of Trio's top-level functions (though some class attributes are still dynamic... we're working on it). (`#805 `__) -- Add `trio.hazmat.FdStream` for wrapping a Unix file descriptor as a `~trio.abc.Stream`. (`#829 `__) +- Add `trio.lowlevel.FdStream` for wrapping a Unix file descriptor as a `~trio.abc.Stream`. (`#829 `__) - Trio now gives a reasonable traceback and error message in most cases when its invariants surrounding cancel scope nesting have been violated. (One common source of such violations is an async generator @@ -219,8 +219,8 @@ Deprecations and Removals `trio.to_thread.current_default_thread_limiter`. (`#810 `__) - Give up on trying to have different low-level waiting APIs on Unix and - Windows. All platforms now have `trio.hazmat.wait_readable`, - `trio.hazmat.wait_writable`, and `trio.hazmat.notify_closing`. The old + Windows. All platforms now have `trio.lowlevel.wait_readable`, + `trio.lowlevel.wait_writable`, and `trio.lowlevel.notify_closing`. The old platform-specific synonyms ``wait_socket_*``, ``notify_socket_closing``, and ``notify_fd_closing`` have been deprecated. (`#878 `__) @@ -331,7 +331,7 @@ Deprecations and Removals - Remove all the APIs deprecated in 0.9.0 or earlier (``trio.Queue``, ``trio.catch_signals()``, ``trio.BrokenStreamError``, and ``trio.ResourceBusyError``), except for ``trio.hazmat.UnboundedQueue``, - which stays for now since it is used by the obscure hazmat functions + which stays for now since it is used by the obscure lowlevel functions ``monitor_completion_queue()`` and ``monitor_kevent()``. (`#918 `__) @@ -358,7 +358,7 @@ Features convenience. (`#4 `__) - You can now create an unbounded :class:`CapacityLimiter` by initializing with `math.inf` (`#618 `__) -- New :mod:`trio.hazmat` features to allow cleanly switching live coroutine +- New :mod:`trio.lowlevel` features to allow cleanly switching live coroutine objects between Trio and other coroutine runners. Frankly, we're not even sure this is a good idea, but we want to `try it out in trio-asyncio `__, so here we are. @@ -473,9 +473,9 @@ Features ``notify_socket_close`` now work on bare socket descriptors, instead of requiring a :func:`socket.socket` object. (`#400 `__) -- If you're using :func:`trio.hazmat.wait_task_rescheduled` and other low-level +- If you're using :func:`trio.lowlevel.wait_task_rescheduled` and other low-level routines to implement a new sleeping primitive, you can now use the new - :data:`trio.hazmat.Task.custom_sleep_data` attribute to pass arbitrary data + :data:`trio.lowlevel.Task.custom_sleep_data` attribute to pass arbitrary data between the sleeping task, abort function, and waking task. (`#616 `__) @@ -516,7 +516,7 @@ Trio 0.6.0 (2018-08-13) Features ~~~~~~~~ -- Add :func:`trio.hazmat.WaitForSingleObject` async function to await Windows +- Add :func:`trio.lowlevel.WaitForSingleObject` async function to await Windows handles. (`#233 `__) - The `sniffio `__ library can now detect when Trio is running. (`#572 @@ -555,10 +555,10 @@ Features ``trio.hazmat.notify_socket_close``. If you're using Trio's built-in wrappers like :class:`~trio.SocketStream` or :mod:`trio.socket`, then you don't need to worry about this, but if you're using the low-level functions like - :func:`trio.hazmat.wait_readable`, you should make sure to call these + :func:`trio.lowlevel.wait_readable`, you should make sure to call these functions at appropriate times. (`#36 `__) -- Tasks created by :func:`~trio.hazmat.spawn_system_task` now no longer inherit +- Tasks created by :func:`~trio.lowlevel.spawn_system_task` now no longer inherit the creator's :mod:`contextvars` context, instead using one created at :func:`~trio.run`. (`#289 `__) @@ -585,11 +585,11 @@ Features - Add unix client socket support. (`#401 `__) - Add support for :mod:`contextvars` (see :ref:`task-local storage - `), and add :class:`trio.hazmat.RunVar` as a similar API + `), and add :class:`trio.lowlevel.RunVar` as a similar API for run-local variables. Deprecate ``trio.TaskLocal`` and ``trio.hazmat.RunLocal`` in favor of these new APIs. (`#420 `__) -- Add :func:`trio.hazmat.current_root_task` to get the root task. (`#452 +- Add :func:`trio.lowlevel.current_root_task` to get the root task. (`#452 `__) @@ -612,7 +612,7 @@ Deprecations and Removals Miscellaneous internal changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Simplify implementation of primitive traps like :func:`~trio.hazmat.wait_task_rescheduled` +- Simplify implementation of primitive traps like :func:`~trio.lowlevel.wait_task_rescheduled` (`#395 `__) @@ -734,7 +734,7 @@ Highlights * You know thread-local storage? Well, Trio now has an equivalent: :ref:`task-local storage `. There's also the related, but more obscure, run-local storage; see - :class:`~trio.hazmat.RunLocal`. (`#2 + :class:`~trio.lowlevel.RunLocal`. (`#2 `__) * Added a new :ref:`guide to for contributors `. @@ -815,7 +815,7 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 ``trio.BlockingTrioPortal`` * The hazmat function ``current_call_soon_thread_and_signal_safe`` - is being replaced by :class:`trio.hazmat.TrioToken` + is being replaced by :class:`trio.lowlevel.TrioToken` See `#68 `__ for details. @@ -840,14 +840,14 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 * ``task.wait`` This also lets us move a number of lower-level features out of the - main :mod:`trio` namespace and into :mod:`trio.hazmat`: + main :mod:`trio` namespace and into :mod:`trio.lowlevel`: - * ``trio.Task`` → :class:`trio.hazmat.Task` - * ``trio.current_task`` → :func:`trio.hazmat.current_task` - * ``trio.Result`` → ``trio.hazmat.Result`` - * ``trio.Value`` → ``trio.hazmat.Value`` - * ``trio.Error`` → ``trio.hazmat.Error`` - * ``trio.UnboundedQueue`` → ``trio.hazmat.UnboundedQueue`` + * ``trio.Task`` → :class:`trio.lowlevel.Task` + * ``trio.current_task`` → :func:`trio.lowlevel.current_task` + * ``trio.Result`` → ``trio.lowlevel.Result`` + * ``trio.Value`` → ``trio.lowlevel.Value`` + * ``trio.Error`` → ``trio.lowlevel.Error`` + * ``trio.UnboundedQueue`` → ``trio.lowlevel.UnboundedQueue`` In addition, several introspection attributes are being renamed: @@ -857,24 +857,24 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 See `#136 `__ for more details. -* To consolidate introspection functionality in :mod:`trio.hazmat`, +* To consolidate introspection functionality in :mod:`trio.lowlevel`, the following functions are moving: - * ``trio.current_clock`` → :func:`trio.hazmat.current_clock` - * ``trio.current_statistics`` → :func:`trio.hazmat.current_statistics` + * ``trio.current_clock`` → :func:`trio.lowlevel.current_clock` + * ``trio.current_statistics`` → :func:`trio.lowlevel.current_statistics` See `#317 `__ for more details. * It was decided that 0.1.0's "yield point" terminology was confusing; we now use :ref:`"checkpoint" ` instead. As part of - this, the following functions in :mod:`trio.hazmat` are changing + this, the following functions in :mod:`trio.lowlevel` are changing names: - * ``yield_briefly`` → :func:`~trio.hazmat.checkpoint` - * ``yield_briefly_no_cancel`` → :func:`~trio.hazmat.cancel_shielded_checkpoint` - * ``yield_if_cancelled`` → :func:`~trio.hazmat.checkpoint_if_cancelled` - * ``yield_indefinitely`` → :func:`~trio.hazmat.wait_task_rescheduled` + * ``yield_briefly`` → :func:`~trio.lowlevel.checkpoint` + * ``yield_briefly_no_cancel`` → :func:`~trio.lowlevel.cancel_shielded_checkpoint` + * ``yield_if_cancelled`` → :func:`~trio.lowlevel.checkpoint_if_cancelled` + * ``yield_indefinitely`` → :func:`~trio.lowlevel.wait_task_rescheduled` In addition, the following functions in :mod:`trio.testing` are changing names: @@ -890,8 +890,8 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 `__). * ``trio.current_instruments`` is deprecated. For adding or removing - instrumentation at run-time, see :func:`trio.hazmat.add_instrument` - and :func:`trio.hazmat.remove_instrument` (`#257 + instrumentation at run-time, see :func:`trio.lowlevel.add_instrument` + and :func:`trio.lowlevel.remove_instrument` (`#257 `__) Unfortunately, a limitation in PyPy3 5.8 breaks our deprecation @@ -922,7 +922,7 @@ Other changes * New exception ``ResourceBusyError`` -* The :class:`trio.hazmat.ParkingLot` class (which is used to +* The :class:`trio.lowlevel.ParkingLot` class (which is used to implement many of Trio's synchronization primitives) was rewritten to be simpler and faster (`#272 `__, `#287 @@ -989,7 +989,7 @@ Other changes `__) * PyCharm (and hopefully other IDEs) can now offer better completions - for the :mod:`trio` and :mod:`trio.hazmat` modules (`#314 + for the :mod:`trio` and :mod:`trio.lowlevel` modules (`#314 `__) * Trio now uses `yapf `__ to diff --git a/docs/source/index.rst b/docs/source/index.rst index 6e4f2f78e1..610c6b07de 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -77,7 +77,7 @@ Vital statistics: reference-core.rst reference-io.rst reference-testing.rst - reference-hazmat.rst + reference-lowlevel.rst design.rst history.rst contributing.rst diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 227c67bd6a..d868f9b1e7 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1433,9 +1433,9 @@ than the lower-level primitives discussed in this section. But if you need them, they're here. (If you find yourself reaching for these because you're trying to implement a new higher-level synchronization primitive, then you might also want to check out the facilities in -:mod:`trio.hazmat` for a more direct exposure of Trio's underlying +:mod:`trio.lowlevel` for a more direct exposure of Trio's underlying synchronization logic. All of classes discussed in this section are -implemented on top of the public APIs in :mod:`trio.hazmat`; they +implemented on top of the public APIs in :mod:`trio.lowlevel`; they don't have any special access to Trio's internals.) .. autoclass:: CapacityLimiter diff --git a/docs/source/reference-hazmat.rst b/docs/source/reference-lowlevel.rst similarity index 96% rename from docs/source/reference-hazmat.rst rename to docs/source/reference-lowlevel.rst index 7f32ebf985..0ffa2c2b21 100644 --- a/docs/source/reference-hazmat.rst +++ b/docs/source/reference-lowlevel.rst @@ -1,17 +1,17 @@ ======================================================= - Introspecting and extending Trio with ``trio.hazmat`` + Introspecting and extending Trio with ``trio.lowlevel`` ======================================================= -.. module:: trio.hazmat +.. module:: trio.lowlevel .. warning:: You probably don't want to use this module. -:mod:`trio.hazmat` is Trio's "hazardous materials" layer: it contains +:mod:`trio.lowlevel` is Trio's "hazardous materials" layer: it contains APIs useful for introspecting and extending Trio. If you're writing ordinary, everyday code, then you can ignore this module completely. But sometimes you need something a bit lower level. Here are some -examples of situations where you should reach for :mod:`trio.hazmat`: +examples of situations where you should reach for :mod:`trio.lowlevel`: * You want to implement a new :ref:`synchronization primitive ` that Trio doesn't (yet) provide, like a @@ -27,7 +27,7 @@ examples of situations where you should reach for :mod:`trio.hazmat`: * You need to interoperate with a C library whose API exposes raw file descriptors. -Using :mod:`trio.hazmat` isn't really *that* hazardous; in fact you're +Using :mod:`trio.lowlevel` isn't really *that* hazardous; in fact you're already using it – it's how most of the functionality described in previous chapters is implemented. The APIs described here have strictly defined and carefully documented semantics, and are perfectly @@ -113,7 +113,7 @@ Low-level I/O primitives ======================== Different environments expose different low-level APIs for performing -async I/O. :mod:`trio.hazmat` exposes these APIs in a relatively +async I/O. :mod:`trio.lowlevel` exposes these APIs in a relatively direct way, so as to allow maximum power and flexibility for higher level code. However, this means that the exact API provided may vary depending on what system Trio is running on. @@ -199,8 +199,8 @@ and want to bundle them together into a single bidirectional `~trio.abc.Stream`, then use `trio.StapledStream`:: bidirectional_stream = trio.StapledStream( - trio.hazmat.FdStream(write_fd), - trio.hazmat.FdStream(read_fd) + trio.lowlevel.FdStream(write_fd), + trio.lowlevel.FdStream(read_fd) ) .. autoclass:: FdStream @@ -452,15 +452,15 @@ this does serve to illustrate the basic structure of the self._blocked_tasks.append(task) def abort_fn(_): self._blocked_tasks.remove(task) - return trio.hazmat.Abort.SUCCEEDED - await trio.hazmat.wait_task_rescheduled(abort_fn) + return trio.lowlevel.Abort.SUCCEEDED + await trio.lowlevel.wait_task_rescheduled(abort_fn) self._held = True def release(self): self._held = False if self._blocked_tasks: woken_task = self._blocked_tasks.popleft() - trio.hazmat.reschedule(woken_task) + trio.lowlevel.reschedule(woken_task) Task API diff --git a/newsfragments/476.removal.rst b/newsfragments/476.removal.rst new file mode 100644 index 0000000000..e5c8a5619e --- /dev/null +++ b/newsfragments/476.removal.rst @@ -0,0 +1,3 @@ +It turns out that everyone got confused by the name ``trio.lowlevel``. +So that name has been deprecated, and the new name is +:mod:`trio.lowlevel`. diff --git a/notes-to-self/afd-lab.py b/notes-to-self/afd-lab.py index 58a6c22799..ed420dbdbd 100644 --- a/notes-to-self/afd-lab.py +++ b/notes-to-self/afd-lab.py @@ -96,7 +96,7 @@ class AFDLab: def __init__(self): self._afd = _afd_helper_handle() - trio.hazmat.register_with_iocp(self._afd) + trio.lowlevel.register_with_iocp(self._afd) async def afd_poll(self, sock, flags, *, exclusive=0): print(f"Starting a poll for {flags!r}") @@ -127,7 +127,7 @@ async def afd_poll(self, sock, flags, *, exclusive=0): raise try: - await trio.hazmat.wait_overlapped(self._afd, lpOverlapped) + await trio.lowlevel.wait_overlapped(self._afd, lpOverlapped) except: print(f"Poll for {flags!r}: {sys.exc_info()[1]!r}") raise diff --git a/notes-to-self/print-task-tree.py b/notes-to-self/print-task-tree.py index d4b6dd8da4..38e545853e 100644 --- a/notes-to-self/print-task-tree.py +++ b/notes-to-self/print-task-tree.py @@ -38,7 +38,7 @@ def current_root_task(): - task = trio.hazmat.current_task() + task = trio.lowlevel.current_task() while task.parent_nursery is not None: task = task.parent_nursery.parent_task return task diff --git a/notes-to-self/schedule-timing.py b/notes-to-self/schedule-timing.py index 441f579f7c..c3093066e2 100644 --- a/notes-to-self/schedule-timing.py +++ b/notes-to-self/schedule-timing.py @@ -10,7 +10,7 @@ async def reschedule_loop(depth): while RUNNING: LOOPS += 1 await trio.sleep(0) - #await trio.hazmat.cancel_shielded_checkpoint() + #await trio.lowlevel.cancel_shielded_checkpoint() else: await reschedule_loop(depth - 1) diff --git a/notes-to-self/socket-scaling.py b/notes-to-self/socket-scaling.py index 61527e2552..1a18141e17 100644 --- a/notes-to-self/socket-scaling.py +++ b/notes-to-self/socket-scaling.py @@ -49,11 +49,11 @@ def pt(desc, *, count=total, item="socket"): pt("socket creation") async with trio.open_nursery() as nursery: for s in sockets: - nursery.start_soon(trio.hazmat.wait_readable, s) + nursery.start_soon(trio.lowlevel.wait_readable, s) await trio.testing.wait_all_tasks_blocked() pt("spawning wait tasks") for _ in range(1000): - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() pt("scheduling 1000 times", count=1000, item="schedule") nursery.cancel_scope.cancel() pt("cancelling wait tasks") diff --git a/notes-to-self/tiny-thread-pool.py b/notes-to-self/tiny-thread-pool.py index 85afc4eb3e..b5de1c37b3 100644 --- a/notes-to-self/tiny-thread-pool.py +++ b/notes-to-self/tiny-thread-pool.py @@ -110,7 +110,7 @@ def _run_job(self, job): job.finish(*job.finish_args) -# Probably the interface should be: trio.hazmat.call_soon_in_worker_thread? +# Probably the interface should be: trio.lowlevel.call_soon_in_worker_thread? # Enqueueing work: # put into unbounded queue diff --git a/notes-to-self/trace.py b/notes-to-self/trace.py index 700e2e7e81..c024a36ba5 100644 --- a/notes-to-self/trace.py +++ b/notes-to-self/trace.py @@ -88,7 +88,7 @@ def after_task_step(self, task): def task_scheduled(self, task): try: - waker = trio.hazmat.current_task() + waker = trio.lowlevel.current_task() except RuntimeError: pass else: diff --git a/trio/__init__.py b/trio/__init__.py index 1ad0e25362..452dce130e 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -66,7 +66,7 @@ from ._deprecate import TrioDeprecationWarning # Submodules imported by default -from . import hazmat +from . import lowlevel from . import socket from . import abc from . import from_thread @@ -121,31 +121,38 @@ issue=810, instead=from_thread, ), + "hazmat": + _deprecate.DeprecatedAttribute( + lowlevel, + "0.15.0", + issue=476, + instead="trio.lowlevel", + ), } -_deprecate.enable_attribute_deprecations(hazmat.__name__) -hazmat.__deprecated_attributes__ = { +_deprecate.enable_attribute_deprecations(lowlevel.__name__) +lowlevel.__deprecated_attributes__ = { "wait_socket_readable": _deprecate.DeprecatedAttribute( - hazmat.wait_readable, + lowlevel.wait_readable, "0.12.0", issue=878, ), "wait_socket_writable": _deprecate.DeprecatedAttribute( - hazmat.wait_writable, + lowlevel.wait_writable, "0.12.0", issue=878, ), "notify_socket_close": _deprecate.DeprecatedAttribute( - hazmat.notify_closing, + lowlevel.notify_closing, "0.12.0", issue=878, ), "notify_fd_close": _deprecate.DeprecatedAttribute( - hazmat.notify_closing, + lowlevel.notify_closing, "0.12.0", issue=878, ), @@ -159,7 +166,7 @@ # - probably other stuff from ._util import fixup_module_metadata fixup_module_metadata(__name__, globals()) -fixup_module_metadata(hazmat.__name__, hazmat.__dict__) +fixup_module_metadata(lowlevel.__name__, lowlevel.__dict__) fixup_module_metadata(socket.__name__, socket.__dict__) fixup_module_metadata(abc.__name__, abc.__dict__) fixup_module_metadata(from_thread.__name__, from_thread.__dict__) diff --git a/trio/_abc.py b/trio/_abc.py index 8064e71169..504c145baa 100644 --- a/trio/_abc.py +++ b/trio/_abc.py @@ -79,7 +79,7 @@ def task_spawned(self, task): """Called when the given task is created. Args: - task (trio.hazmat.Task): The new task. + task (trio.lowlevel.Task): The new task. """ @@ -90,7 +90,7 @@ def task_scheduled(self, task): runnable tasks ahead of it. Args: - task (trio.hazmat.Task): The task that became runnable. + task (trio.lowlevel.Task): The task that became runnable. """ @@ -98,7 +98,7 @@ def before_task_step(self, task): """Called immediately before we resume running the given task. Args: - task (trio.hazmat.Task): The task that is about to run. + task (trio.lowlevel.Task): The task that is about to run. """ @@ -106,7 +106,7 @@ def after_task_step(self, task): """Called when we return to the main run loop after a task has yielded. Args: - task (trio.hazmat.Task): The task that just ran. + task (trio.lowlevel.Task): The task that just ran. """ @@ -114,7 +114,7 @@ def task_exited(self, task): """Called when the given task exits. Args: - task (trio.hazmat.Task): The finished task. + task (trio.lowlevel.Task): The finished task. """ diff --git a/trio/_channel.py b/trio/_channel.py index c902c485fa..1a24ece2e4 100644 --- a/trio/_channel.py +++ b/trio/_channel.py @@ -144,7 +144,7 @@ def send_nowait(self, value): assert not self._state.data task, _ = self._state.receive_tasks.popitem(last=False) task.custom_sleep_data._tasks.remove(task) - trio.hazmat.reschedule(task, Value(value)) + trio.lowlevel.reschedule(task, Value(value)) elif len(self._state.data) < self._state.max_buffer_size: self._state.data.append(value) else: @@ -157,16 +157,16 @@ async def send(self, value): Memory channels allow multiple tasks to call `send` at the same time. """ - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() try: self.send_nowait(value) except trio.WouldBlock: pass else: - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() return - task = trio.hazmat.current_task() + task = trio.lowlevel.current_task() self._tasks.add(task) self._state.send_tasks[task] = value task.custom_sleep_data = self @@ -174,9 +174,9 @@ async def send(self, value): def abort_fn(_): self._tasks.remove(task) del self._state.send_tasks[task] - return trio.hazmat.Abort.SUCCEEDED + return trio.lowlevel.Abort.SUCCEEDED - await trio.hazmat.wait_task_rescheduled(abort_fn) + await trio.lowlevel.wait_task_rescheduled(abort_fn) @enable_ki_protection def clone(self): @@ -210,11 +210,11 @@ def clone(self): @enable_ki_protection async def aclose(self): if self._closed: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return self._closed = True for task in self._tasks: - trio.hazmat.reschedule(task, Error(trio.ClosedResourceError())) + trio.lowlevel.reschedule(task, Error(trio.ClosedResourceError())) del self._state.send_tasks[task] self._tasks.clear() self._state.open_send_channels -= 1 @@ -222,9 +222,9 @@ async def aclose(self): assert not self._state.send_tasks for task in self._state.receive_tasks: task.custom_sleep_data._tasks.remove(task) - trio.hazmat.reschedule(task, Error(trio.EndOfChannel())) + trio.lowlevel.reschedule(task, Error(trio.EndOfChannel())) self._state.receive_tasks.clear() - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() @attr.s(eq=False, repr=False) @@ -255,7 +255,7 @@ def receive_nowait(self): if self._state.send_tasks: task, value = self._state.send_tasks.popitem(last=False) task.custom_sleep_data._tasks.remove(task) - trio.hazmat.reschedule(task) + trio.lowlevel.reschedule(task) self._state.data.append(value) # Fall through if self._state.data: @@ -273,16 +273,16 @@ async def receive(self): will get the second item sent, and so on. """ - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() try: value = self.receive_nowait() except trio.WouldBlock: pass else: - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() return value - task = trio.hazmat.current_task() + task = trio.lowlevel.current_task() self._tasks.add(task) self._state.receive_tasks[task] = None task.custom_sleep_data = self @@ -290,9 +290,9 @@ async def receive(self): def abort_fn(_): self._tasks.remove(task) del self._state.receive_tasks[task] - return trio.hazmat.Abort.SUCCEEDED + return trio.lowlevel.Abort.SUCCEEDED - return await trio.hazmat.wait_task_rescheduled(abort_fn) + return await trio.lowlevel.wait_task_rescheduled(abort_fn) @enable_ki_protection def clone(self): @@ -329,11 +329,11 @@ def clone(self): @enable_ki_protection async def aclose(self): if self._closed: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return self._closed = True for task in self._tasks: - trio.hazmat.reschedule(task, Error(trio.ClosedResourceError())) + trio.lowlevel.reschedule(task, Error(trio.ClosedResourceError())) del self._state.receive_tasks[task] self._tasks.clear() self._state.open_receive_channels -= 1 @@ -341,7 +341,7 @@ async def aclose(self): assert not self._state.receive_tasks for task in self._state.send_tasks: task.custom_sleep_data._tasks.remove(task) - trio.hazmat.reschedule(task, Error(trio.BrokenResourceError())) + trio.lowlevel.reschedule(task, Error(trio.BrokenResourceError())) self._state.send_tasks.clear() self._state.data.clear() - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index cb3d5d714f..4b3a088d1b 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -1,7 +1,7 @@ """ This namespace represents the core functionality that has to be built-in and deal with private internal data structures. Things in this namespace -are publicly available in either trio, trio.hazmat, or trio.testing. +are publicly available in either trio, trio.lowlevel, or trio.testing. """ from ._exceptions import ( diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index 6833cc886f..a8fd8e32bd 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -160,12 +160,12 @@ def run_sync_soon(self, sync_fn, *args, idempotent=False): If you need this, you'll have to build your own. The call is effectively run as part of a system task (see - :func:`~trio.hazmat.spawn_system_task`). In particular this means + :func:`~trio.lowlevel.spawn_system_task`). In particular this means that: * :exc:`KeyboardInterrupt` protection is *enabled* by default; if you want ``sync_fn`` to be interruptible by control-C, then you - need to use :func:`~trio.hazmat.disable_ki_protection` + need to use :func:`~trio.lowlevel.disable_ki_protection` explicitly. * If ``sync_fn`` raises an exception, then it's converted into a diff --git a/trio/_core/_exceptions.py b/trio/_core/_exceptions.py index 3cdaa2ca8e..b481f7c50d 100644 --- a/trio/_core/_exceptions.py +++ b/trio/_core/_exceptions.py @@ -5,7 +5,7 @@ class TrioInternalError(Exception): """Raised by :func:`run` if we encounter a bug in Trio, or (possibly) a - misuse of one of the low-level :mod:`trio.hazmat` APIs. + misuse of one of the low-level :mod:`trio.lowlevel` APIs. This should never happen! If you get this error, please file a bug. diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index 834346c0bf..75f61bfdc5 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -22,7 +22,7 @@ def current_statistics(): :data:`~math.inf` if there are no pending deadlines. * ``run_sync_soon_queue_size`` (int): The number of unprocessed callbacks queued via - :meth:`trio.hazmat.TrioToken.run_sync_soon`. + :meth:`trio.lowlevel.TrioToken.run_sync_soon`. * ``io_statistics`` (object): Some statistics from Trio's I/O backend. This always has an attribute ``backend`` which is a string naming which operating-system-specific I/O backend is in use; the @@ -85,7 +85,7 @@ def reschedule(task, next_send=_NO_SEND): to calling :func:`reschedule` once.) Args: - task (trio.hazmat.Task): the task to be rescheduled. Must be blocked + task (trio.lowlevel.Task): the task to be rescheduled. Must be blocked in a call to :func:`wait_task_rescheduled`. next_send (outcome.Outcome): the value (or error) to return (or raise) from :func:`wait_task_rescheduled`. @@ -196,18 +196,18 @@ async def test_lock_fairness(): nursery.start_soon(lock_taker, lock) # child hasn't run yet, we have the lock assert lock.locked() - assert lock._owner is trio.hazmat.current_task() + assert lock._owner is trio.lowlevel.current_task() await trio.testing.wait_all_tasks_blocked() # now the child has run and is blocked on lock.acquire(), we # still have the lock assert lock.locked() - assert lock._owner is trio.hazmat.current_task() + assert lock._owner is trio.lowlevel.current_task() lock.release() try: # The child has a prior claim, so we can't have it lock.acquire_nowait() except trio.WouldBlock: - assert lock._owner is not trio.hazmat.current_task() + assert lock._owner is not trio.lowlevel.current_task() print("PASS") else: print("FAIL") diff --git a/trio/_core/_parking_lot.py b/trio/_core/_parking_lot.py index d14e95e6ce..696ff90917 100644 --- a/trio/_core/_parking_lot.py +++ b/trio/_core/_parking_lot.py @@ -177,8 +177,8 @@ async def parker(lot): print("woken") async def main(): - lot1 = trio.hazmat.ParkingLot() - lot2 = trio.hazmat.ParkingLot() + lot1 = trio.lowlevel.ParkingLot() + lot2 = trio.lowlevel.ParkingLot() async with trio.open_nursery() as nursery: nursery.start_soon(parker, lot1) await trio.testing.wait_all_tasks_blocked() diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 10606c710a..11d65750e6 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -790,13 +790,13 @@ def __init__(self, parent_task, cancel_scope): @property def child_tasks(self): - """(`frozenset`): Contains all the child :class:`~trio.hazmat.Task` + """(`frozenset`): Contains all the child :class:`~trio.lowlevel.Task` objects which are still running.""" return frozenset(self._children) @property def parent_task(self): - "(`~trio.hazmat.Task`): The Task that opened this nursery." + "(`~trio.lowlevel.Task`): The Task that opened this nursery." return self._parent_task def _add_exc(self, exc): @@ -1144,7 +1144,7 @@ def current_statistics(self): :data:`~math.inf` if there are no pending deadlines. * ``run_sync_soon_queue_size`` (int): The number of unprocessed callbacks queued via - :meth:`trio.hazmat.TrioToken.run_sync_soon`. + :meth:`trio.lowlevel.TrioToken.run_sync_soon`. * ``io_statistics`` (object): Some statistics from Trio's I/O backend. This always has an attribute ``backend`` which is a string naming which operating-system-specific I/O backend is in use; the @@ -1210,7 +1210,7 @@ def reschedule(self, task, next_send=_NO_SEND): to calling :func:`reschedule` once.) Args: - task (trio.hazmat.Task): the task to be rescheduled. Must be blocked + task (trio.lowlevel.Task): the task to be rescheduled. Must be blocked in a call to :func:`wait_task_rescheduled`. next_send (outcome.Outcome): the value (or error) to return (or raise) from :func:`wait_task_rescheduled`. @@ -1575,18 +1575,18 @@ async def test_lock_fairness(): nursery.start_soon(lock_taker, lock) # child hasn't run yet, we have the lock assert lock.locked() - assert lock._owner is trio.hazmat.current_task() + assert lock._owner is trio.lowlevel.current_task() await trio.testing.wait_all_tasks_blocked() # now the child has run and is blocked on lock.acquire(), we # still have the lock assert lock.locked() - assert lock._owner is trio.hazmat.current_task() + assert lock._owner is trio.lowlevel.current_task() lock.release() try: # The child has a prior claim, so we can't have it lock.acquire_nowait() except trio.WouldBlock: - assert lock._owner is not trio.hazmat.current_task() + assert lock._owner is not trio.lowlevel.current_task() print("PASS") else: print("FAIL") @@ -2045,7 +2045,7 @@ async def checkpoint_if_cancelled(): Equivalent to (but potentially more efficient than):: if trio.current_deadline() == -inf: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() This is either a no-op, or else it allow other tasks to be scheduled and then raises :exc:`trio.Cancelled`. diff --git a/trio/_core/_traps.py b/trio/_core/_traps.py index a24a2f1742..f340481078 100644 --- a/trio/_core/_traps.py +++ b/trio/_core/_traps.py @@ -37,7 +37,7 @@ async def cancel_shielded_checkpoint(): Equivalent to (but potentially more efficient than):: with trio.CancelScope(shield=True): - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() """ return (await _async_yield(CancelShieldedCheckpoint)).unwrap() @@ -67,7 +67,7 @@ async def wait_task_rescheduled(abort_func): """Put the current task to sleep, with cancellation support. This is the lowest-level API for blocking in Trio. Every time a - :class:`~trio.hazmat.Task` blocks, it does so by calling this function + :class:`~trio.lowlevel.Task` blocks, it does so by calling this function (usually indirectly via some higher-level API). This is a tricky interface with no guard rails. If you can use @@ -96,7 +96,7 @@ async def wait_task_rescheduled(abort_func): def abort_func(raise_cancel): ... - return trio.hazmat.Abort.SUCCEEDED # or FAILED + return trio.lowlevel.Abort.SUCCEEDED # or FAILED It should attempt to clean up any state associated with this call, and in particular, arrange that :func:`reschedule` will *not* be called @@ -127,7 +127,7 @@ def abort_func(raise_cancel): # Catch the exception from raise_cancel and inject it into the task. # (This is what Trio does automatically for you if you return # Abort.SUCCEEDED.) - trio.hazmat.reschedule(task, outcome.capture(raise_cancel)) + trio.lowlevel.reschedule(task, outcome.capture(raise_cancel)) # Option 2: # wait to be woken by "someone", and then decide whether to raise @@ -137,7 +137,7 @@ def abort(inner_raise_cancel): nonlocal outer_raise_cancel outer_raise_cancel = inner_raise_cancel TRY_TO_CANCEL_OPERATION() - return trio.hazmat.Abort.FAILED + return trio.lowlevel.Abort.FAILED await wait_task_rescheduled(abort) if OPERATION_WAS_SUCCESSFULLY_CANCELLED: # raises the error diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index abe4d60c56..642a4a25ac 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -44,7 +44,7 @@ class UnboundedQueue: @deprecated( "0.9.0", issue=497, - thing="trio.hazmat.UnboundedQueue", + thing="trio.lowlevel.UnboundedQueue", instead="trio.open_memory_channel(math.inf)" ) def __init__(self): diff --git a/trio/_core/tests/test_io.py b/trio/_core/tests/test_io.py index 2adb3f9a65..fb459bea2d 100644 --- a/trio/_core/tests/test_io.py +++ b/trio/_core/tests/test_io.py @@ -48,9 +48,9 @@ def fileno_wrapper(fileobj): return fileno_wrapper -wait_readable_options = [trio.hazmat.wait_readable] -wait_writable_options = [trio.hazmat.wait_writable] -notify_closing_options = [trio.hazmat.notify_closing] +wait_readable_options = [trio.lowlevel.wait_readable] +wait_writable_options = [trio.lowlevel.wait_writable] +notify_closing_options = [trio.lowlevel.notify_closing] for options_list in [ wait_readable_options, wait_writable_options, notify_closing_options @@ -285,7 +285,7 @@ async def test_notify_closing_on_invalid_object(): got_oserror = False got_no_error = False try: - trio.hazmat.notify_closing(-1) + trio.lowlevel.notify_closing(-1) except OSError: got_oserror = True else: @@ -296,7 +296,7 @@ async def test_notify_closing_on_invalid_object(): async def test_wait_on_invalid_object(): # We definitely want to raise an error everywhere if you pass in an # invalid fd to wait_* - for wait in [trio.hazmat.wait_readable, trio.hazmat.wait_writable]: + for wait in [trio.lowlevel.wait_readable, trio.lowlevel.wait_writable]: with stdlib_socket.socket() as s: fileno = s.fileno() # We just closed the socket and don't do anything else in between, so @@ -377,7 +377,7 @@ async def allow_OSError(async_func, *args): with stdlib_socket.socket() as s: async with trio.open_nursery() as nursery: - nursery.start_soon(allow_OSError, trio.hazmat.wait_readable, s) + nursery.start_soon(allow_OSError, trio.lowlevel.wait_readable, s) await wait_all_tasks_blocked() s.close() await wait_all_tasks_blocked() @@ -389,7 +389,7 @@ async def allow_OSError(async_func, *args): # wait_readable pending until cancelled). with stdlib_socket.socket() as s, s.dup() as s2: # noqa: F841 async with trio.open_nursery() as nursery: - nursery.start_soon(allow_OSError, trio.hazmat.wait_readable, s) + nursery.start_soon(allow_OSError, trio.lowlevel.wait_readable, s) await wait_all_tasks_blocked() s.close() await wait_all_tasks_blocked() @@ -408,8 +408,8 @@ async def allow_OSError(async_func, *args): b.setblocking(False) fill_socket(a) async with trio.open_nursery() as nursery: - nursery.start_soon(allow_OSError, trio.hazmat.wait_readable, a) - nursery.start_soon(allow_OSError, trio.hazmat.wait_writable, a) + nursery.start_soon(allow_OSError, trio.lowlevel.wait_readable, a) + nursery.start_soon(allow_OSError, trio.lowlevel.wait_writable, a) await wait_all_tasks_blocked() a.close() nursery.cancel_scope.cancel() @@ -432,12 +432,12 @@ async def allow_OSError(async_func, *args): # definitely arrive, and when it does then we can assume that whatever # notification was going to arrive for 'a' has also arrived. async def wait_readable_a2_then_set(): - await trio.hazmat.wait_readable(a2) + await trio.lowlevel.wait_readable(a2) e.set() async with trio.open_nursery() as nursery: - nursery.start_soon(allow_OSError, trio.hazmat.wait_readable, a) - nursery.start_soon(allow_OSError, trio.hazmat.wait_writable, a) + nursery.start_soon(allow_OSError, trio.lowlevel.wait_readable, a) + nursery.start_soon(allow_OSError, trio.lowlevel.wait_writable, a) nursery.start_soon(wait_readable_a2_then_set) await wait_all_tasks_blocked() a.close() diff --git a/trio/_file_io.py b/trio/_file_io.py index 10464589a9..159e08be59 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -128,7 +128,7 @@ async def aclose(self): with trio.CancelScope(shield=True): await trio.to_thread.run_sync(self._wrapped.close) - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() async def open_file( diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index e816ea2dea..405375479c 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -109,7 +109,7 @@ async def send_all(self, data): raise trio.ClosedResourceError( "socket was already closed" ) - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return total_sent = 0 while total_sent < len(data): @@ -126,7 +126,7 @@ async def wait_send_all_might_not_block(self): async def send_eof(self): with self._send_conflict_detector: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() # On macOS, calling shutdown a second time raises ENOTCONN, but # send_eof needs to be idempotent. if self.socket.did_shutdown_SHUT_WR: @@ -144,7 +144,7 @@ async def receive_some(self, max_bytes=None): async def aclose(self): self.socket.close() - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() # __aenter__, __aexit__ inherited from HalfCloseableStream are OK @@ -389,4 +389,4 @@ async def aclose(self): """ self.socket.close() - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() diff --git a/trio/_signals.py b/trio/_signals.py index 2f2793c43e..6392eebd0f 100644 --- a/trio/_signals.py +++ b/trio/_signals.py @@ -59,7 +59,7 @@ class SignalReceiver: def __init__(self): # {signal num: None} self._pending = OrderedDict() - self._lot = trio.hazmat.ParkingLot() + self._lot = trio.lowlevel.ParkingLot() self._conflict_detector = ConflictDetector( "only one task can iterate on a signal receiver at a time" ) @@ -107,7 +107,7 @@ async def __anext__(self): if not self._pending: await self._lot.park() else: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() signum, _ = self._pending.popitem(last=False) return signum @@ -151,7 +151,7 @@ def open_signal_receiver(*signals): "Sorry, open_signal_receiver is only possible when running in " "Python interpreter's main thread" ) - token = trio.hazmat.current_trio_token() + token = trio.lowlevel.current_trio_token() queue = SignalReceiver() def handler(signum, _): diff --git a/trio/_socket.py b/trio/_socket.py index a3ffe0315f..843b540ad9 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -29,7 +29,7 @@ def _is_blocking_io_error(self, exc): return self._blocking_exc_override(exc) async def __aenter__(self): - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() async def __aexit__(self, etype, value, tb): if value is not None and self._is_blocking_io_error(value): @@ -37,7 +37,7 @@ async def __aexit__(self, etype, value, tb): # block return True else: - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() # Let the return or exception propagate return False @@ -446,7 +446,7 @@ def dup(self): def close(self): if self._sock.fileno() != -1: - trio.hazmat.notify_closing(self._sock) + trio.lowlevel.notify_closing(self._sock) self._sock.close() async def bind(self, address): @@ -503,12 +503,12 @@ async def _resolve_address(self, address, flags): "tuple" ) elif self._sock.family == _stdlib_socket.AF_UNIX: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() # unwrap path-likes return _os.fspath(address) else: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return address host, port, *_ = address # Special cases to match the stdlib, see gh-277 diff --git a/trio/_ssl.py b/trio/_ssl.py index 21093b54dc..c72e0fc3b0 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -432,7 +432,7 @@ def _check_status(self): async def _retry( self, fn, *args, ignore_want_read=False, is_handshake=False ): - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() yielded = False finished = False while not finished: @@ -598,7 +598,7 @@ async def _retry( self._incoming.write(data) self._inner_recv_count += 1 if not yielded: - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() return ret async def _do_handshake(self): @@ -670,7 +670,7 @@ async def receive_some(self, max_bytes=None): (_stdlib_ssl.SSLEOFError, _stdlib_ssl.SSLSyscallError) ) ): - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return b"" else: raise @@ -697,7 +697,7 @@ async def receive_some(self, max_bytes=None): self._https_compatible and isinstance(exc.__cause__, _stdlib_ssl.SSLEOFError) ): - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return b"" else: raise @@ -719,7 +719,7 @@ async def send_all(self, data): # SSLObject interprets write(b"") as an EOF for some reason, which # is not what we want. if not data: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return await self._retry(self._ssl_object.write, data) @@ -763,7 +763,7 @@ async def aclose(self): """ if self._state is _State.CLOSED: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return if self._state is _State.BROKEN or self._https_compatible: self._state = _State.CLOSED diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 7e82b77318..265d4f2ef9 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -268,7 +268,7 @@ async def wait(self): async with self._wait_lock: if self.poll() is None: if self._pidfd is not None: - await trio.hazmat.wait_readable(self._pidfd) + await trio.lowlevel.wait_readable(self._pidfd) else: await wait_child_exiting(self) # We have to use .wait() here, not .poll(), because on macOS @@ -380,7 +380,7 @@ async def open_process( """ # XX FIXME: move the process creation into a thread as soon as we're done # deprecating Process(...) - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() return Process._create( command, stdin=stdin, stdout=stdout, stderr=stderr, **options ) diff --git a/trio/_subprocess_platform/__init__.py b/trio/_subprocess_platform/__init__.py index b1db8499c6..68132cbf67 100644 --- a/trio/_subprocess_platform/__init__.py +++ b/trio/_subprocess_platform/__init__.py @@ -67,7 +67,7 @@ def create_pipe_from_child_output() -> Tuple[ReceiveStream, int]: try: if os.name == "posix": - from ..hazmat import FdStream + from ..lowlevel import FdStream def create_pipe_to_child_stdin(): # noqa: F811 rfd, wfd = os.pipe() diff --git a/trio/_sync.py b/trio/_sync.py index e4af1be178..7bb881f772 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -34,14 +34,14 @@ class Event: lost wakeups: it doesn't matter whether :meth:`set` gets called just before or after :meth:`wait`. If you want a lower-level wakeup primitive that doesn't have this protection, consider :class:`Condition` - or :class:`trio.hazmat.ParkingLot`. + or :class:`trio.lowlevel.ParkingLot`. .. note:: Unlike `threading.Event`, `trio.Event` has no `~threading.Event.clear` method. In Trio, once an `Event` has happened, it cannot un-happen. If you need to represent a series of events, consider creating a new `Event` object for each one (they're cheap!), or other synchronization methods like :ref:`channels ` or - `trio.hazmat.ParkingLot`. + `trio.lowlevel.ParkingLot`. """ @@ -77,7 +77,7 @@ async def wait(self): """ if self._flag: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() else: await self._lot.park() @@ -244,7 +244,7 @@ def acquire_nowait(self): tokens. """ - self.acquire_on_behalf_of_nowait(trio.hazmat.current_task()) + self.acquire_on_behalf_of_nowait(trio.lowlevel.current_task()) @enable_ki_protection def acquire_on_behalf_of_nowait(self, borrower): @@ -252,7 +252,7 @@ def acquire_on_behalf_of_nowait(self, borrower): blocking. Args: - borrower: A :class:`trio.hazmat.Task` or arbitrary opaque object + borrower: A :class:`trio.lowlevel.Task` or arbitrary opaque object used to record who is borrowing this token. This is used by :func:`trio.to_thread.run_sync` to allow threads to "hold tokens", with the intention in the future of using it to `allow @@ -284,7 +284,7 @@ async def acquire(self): tokens. """ - await self.acquire_on_behalf_of(trio.hazmat.current_task()) + await self.acquire_on_behalf_of(trio.lowlevel.current_task()) @enable_ki_protection async def acquire_on_behalf_of(self, borrower): @@ -292,7 +292,7 @@ async def acquire_on_behalf_of(self, borrower): necessary. Args: - borrower: A :class:`trio.hazmat.Task` or arbitrary opaque object + borrower: A :class:`trio.lowlevel.Task` or arbitrary opaque object used to record who is borrowing this token; see :meth:`acquire_on_behalf_of_nowait` for details. @@ -301,11 +301,11 @@ async def acquire_on_behalf_of(self, borrower): tokens. """ - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() try: self.acquire_on_behalf_of_nowait(borrower) except trio.WouldBlock: - task = trio.hazmat.current_task() + task = trio.lowlevel.current_task() self._pending_borrowers[task] = borrower try: await self._lot.park() @@ -313,7 +313,7 @@ async def acquire_on_behalf_of(self, borrower): self._pending_borrowers.pop(task) raise else: - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() @enable_ki_protection def release(self): @@ -324,7 +324,7 @@ def release(self): sack's tokens. """ - self.release_on_behalf_of(trio.hazmat.current_task()) + self.release_on_behalf_of(trio.lowlevel.current_task()) @enable_ki_protection def release_on_behalf_of(self, borrower): @@ -411,7 +411,7 @@ def __init__(self, initial_value, *, max_value=None): # Invariants: # bool(self._lot) implies self._value == 0 # (or equivalently: self._value > 0 implies not self._lot) - self._lot = trio.hazmat.ParkingLot() + self._lot = trio.lowlevel.ParkingLot() self._value = initial_value self._max_value = max_value @@ -460,13 +460,13 @@ async def acquire(self): letting it drop below zero. """ - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() try: self.acquire_nowait() except trio.WouldBlock: await self._lot.park() else: - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() @enable_ki_protection def release(self): @@ -554,7 +554,7 @@ def acquire_nowait(self): """ - task = trio.hazmat.current_task() + task = trio.lowlevel.current_task() if self._owner is task: raise RuntimeError("attempt to re-acquire an already held Lock") elif self._owner is None and not self._lot: @@ -568,7 +568,7 @@ async def acquire(self): """Acquire the lock, blocking if necessary. """ - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() try: self.acquire_nowait() except trio.WouldBlock: @@ -577,7 +577,7 @@ async def acquire(self): # lock as well. await self._lot.park() else: - await trio.hazmat.cancel_shielded_checkpoint() + await trio.lowlevel.cancel_shielded_checkpoint() @enable_ki_protection def release(self): @@ -587,7 +587,7 @@ def release(self): RuntimeError: if the calling task does not hold the lock. """ - task = trio.hazmat.current_task() + task = trio.lowlevel.current_task() if task is not self._owner: raise RuntimeError("can't release a Lock you don't own") if self._lot: @@ -601,7 +601,7 @@ def statistics(self): Currently the following fields are defined: * ``locked``: boolean indicating whether the lock is held. - * ``owner``: the :class:`trio.hazmat.Task` currently holding the lock, + * ``owner``: the :class:`trio.lowlevel.Task` currently holding the lock, or None if the lock is not held. * ``tasks_waiting``: The number of tasks blocked on this lock's :meth:`acquire` method. @@ -704,7 +704,7 @@ def __init__(self, lock=None): if not type(lock) is Lock: raise TypeError("lock must be a trio.Lock") self._lock = lock - self._lot = trio.hazmat.ParkingLot() + self._lot = trio.lowlevel.ParkingLot() def locked(self): """Check whether the underlying lock is currently held. @@ -760,7 +760,7 @@ async def wait(self): RuntimeError: if the calling task does not hold the lock. """ - if trio.hazmat.current_task() is not self._lock._owner: + if trio.lowlevel.current_task() is not self._lock._owner: raise RuntimeError("must hold the lock to wait") self.release() # NOTE: we go to sleep on self._lot, but we'll wake up on @@ -782,7 +782,7 @@ def notify(self, n=1): RuntimeError: if the calling task does not hold the lock. """ - if trio.hazmat.current_task() is not self._lock._owner: + if trio.lowlevel.current_task() is not self._lock._owner: raise RuntimeError("must hold the lock to notify") self._lot.repark(self._lock._lot, count=n) @@ -793,7 +793,7 @@ def notify_all(self): RuntimeError: if the calling task does not hold the lock. """ - if trio.hazmat.current_task() is not self._lock._owner: + if trio.lowlevel.current_task() is not self._lock._owner: raise RuntimeError("must hold the lock to notify") self._lot.repark_all(self._lock._lot) diff --git a/trio/_threads.py b/trio/_threads.py index 0d6cd02811..811bc526a0 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -17,7 +17,7 @@ class BlockingTrioPortal: def __init__(self, trio_token=None): if trio_token is None: - trio_token = trio.hazmat.current_trio_token() + trio_token = trio.lowlevel.current_trio_token() self._trio_token = trio_token def run(self, afn, *args): @@ -238,14 +238,14 @@ async def to_thread_run_sync(sync_fn, *args, cancellable=False, limiter=None): Exception: Whatever ``sync_fn(*args)`` raises. """ - await trio.hazmat.checkpoint_if_cancelled() + await trio.lowlevel.checkpoint_if_cancelled() if limiter is None: limiter = current_default_thread_limiter() # Holds a reference to the task that's blocked in this function waiting # for the result – or None if this function was cancelled and we should # discard the result. - task_register = [trio.hazmat.current_task()] + task_register = [trio.lowlevel.current_task()] name = "trio-worker-{}".format(next(_thread_counter)) placeholder = ThreadPlaceholder(name) @@ -264,7 +264,7 @@ def do_release_then_return_result(): result = outcome.capture(do_release_then_return_result) if task_register[0] is not None: - trio.hazmat.reschedule(task_register[0], result) + trio.lowlevel.reschedule(task_register[0], result) # This is the function that runs in the worker thread to do the actual # work and then schedule the call to report_back_in_trio_thread_fn @@ -287,7 +287,7 @@ def worker_thread_fn(trio_token): try: # daemon=True because it might get left behind if we cancel, and in # this case shouldn't block process exit. - current_trio_token = trio.hazmat.current_trio_token() + current_trio_token = trio.lowlevel.current_trio_token() thread = threading.Thread( target=worker_thread_fn, args=(current_trio_token,), @@ -302,11 +302,11 @@ def worker_thread_fn(trio_token): def abort(_): if cancellable: task_register[0] = None - return trio.hazmat.Abort.SUCCEEDED + return trio.lowlevel.Abort.SUCCEEDED else: - return trio.hazmat.Abort.FAILED + return trio.lowlevel.Abort.FAILED - return await trio.hazmat.wait_task_rescheduled(abort) + return await trio.lowlevel.wait_task_rescheduled(abort) def _run_fn_as_system_task(cb, fn, *args, trio_token=None): @@ -332,7 +332,7 @@ def _run_fn_as_system_task(cb, fn, *args, trio_token=None): # thread local storage (or the absence of) is sufficient to check if trio # is running in a thread or not. try: - trio.hazmat.current_task() + trio.lowlevel.current_task() except RuntimeError: pass else: @@ -385,7 +385,7 @@ async def unprotected_afn(): async def await_in_trio_thread_task(): q.put_nowait(await outcome.acapture(unprotected_afn)) - trio.hazmat.spawn_system_task(await_in_trio_thread_task, name=afn) + trio.lowlevel.spawn_system_task(await_in_trio_thread_task, name=afn) return _run_fn_as_system_task(callback, afn, *args, trio_token=trio_token) diff --git a/trio/_timeouts.py b/trio/_timeouts.py index 25cef08d0d..eabc6befbb 100644 --- a/trio/_timeouts.py +++ b/trio/_timeouts.py @@ -48,8 +48,8 @@ async def sleep_forever(): Equivalent to calling ``await sleep(math.inf)``. """ - await trio.hazmat.wait_task_rescheduled( - lambda _: trio.hazmat.Abort.SUCCEEDED + await trio.lowlevel.wait_task_rescheduled( + lambda _: trio.lowlevel.Abort.SUCCEEDED ) @@ -83,7 +83,7 @@ async def sleep(seconds): if seconds < 0: raise ValueError("duration must be non-negative") if seconds == 0: - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() else: await sleep_until(trio.current_time() + seconds) diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index fb6515d8df..52df9b81cc 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -7,7 +7,7 @@ import trio if os.name != "posix": - # We raise an error here rather than gating the import in hazmat.py + # We raise an error here rather than gating the import in lowlevel.py # in order to keep jedi static analysis happy. raise ImportError @@ -69,9 +69,9 @@ def __del__(self): async def aclose(self): if not self.closed: - trio.hazmat.notify_closing(self.fd) + trio.lowlevel.notify_closing(self.fd) self._raw_close() - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() class FdStream(Stream): @@ -120,7 +120,7 @@ async def send_all(self, data: bytes): # should raise if self._fd_holder.closed: raise trio.ClosedResourceError("file was already closed") - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() length = len(data) # adapted from the SocketStream code with memoryview(data) as view: @@ -130,7 +130,7 @@ async def send_all(self, data: bytes): try: sent += os.write(self._fd_holder.fd, remaining) except BlockingIOError: - await trio.hazmat.wait_writable(self._fd_holder.fd) + await trio.lowlevel.wait_writable(self._fd_holder.fd) except OSError as e: if e.errno == errno.EBADF: raise trio.ClosedResourceError( @@ -144,7 +144,7 @@ async def wait_send_all_might_not_block(self) -> None: if self._fd_holder.closed: raise trio.ClosedResourceError("file was already closed") try: - await trio.hazmat.wait_writable(self._fd_holder.fd) + await trio.lowlevel.wait_writable(self._fd_holder.fd) except BrokenPipeError as e: # kqueue: raises EPIPE on wait_writable instead # of sending, which is annoying @@ -160,12 +160,12 @@ async def receive_some(self, max_bytes=None) -> bytes: if max_bytes < 1: raise ValueError("max_bytes must be integer >= 1") - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() while True: try: data = os.read(self._fd_holder.fd, max_bytes) except BlockingIOError: - await trio.hazmat.wait_readable(self._fd_holder.fd) + await trio.lowlevel.wait_readable(self._fd_holder.fd) except OSError as e: if e.errno == errno.EBADF: raise trio.ClosedResourceError( diff --git a/trio/hazmat.py b/trio/lowlevel.py similarity index 100% rename from trio/hazmat.py rename to trio/lowlevel.py diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index e7d894c12d..6085182a0e 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -13,7 +13,7 @@ def test_core_is_properly_reexported(): # Each export from _core should be re-exported by exactly one of these # three modules: - sources = [trio, trio.hazmat, trio.testing] + sources = [trio, trio.lowlevel, trio.testing] for symbol in dir(_core): if symbol.startswith('_') or symbol == 'tests': continue @@ -97,7 +97,7 @@ def no_underscores(symbols): # runtime set: # - static tools are sometimes sloppy and include deleted names # - some symbols are platform-specific at runtime, but always show up in - # static analysis (e.g. in trio.socket or trio.hazmat) + # static analysis (e.g. in trio.socket or trio.lowlevel) # So we check that the runtime names are a subset of the static names. missing_names = runtime_names - static_names if missing_names: # pragma: no cover diff --git a/trio/tests/test_highlevel_serve_listeners.py b/trio/tests/test_highlevel_serve_listeners.py index ead37cac87..e26a65605f 100644 --- a/trio/tests/test_highlevel_serve_listeners.py +++ b/trio/tests/test_highlevel_serve_listeners.py @@ -23,7 +23,7 @@ async def connect(self): return client async def accept(self): - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() assert not self.closed if self.accept_hook is not None: await self.accept_hook() @@ -33,7 +33,7 @@ async def accept(self): async def aclose(self): self.closed = True - await trio.hazmat.checkpoint() + await trio.lowlevel.checkpoint() async def test_serve_listeners_basic(): diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index f0f25a2ce6..90aec096b1 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -55,12 +55,12 @@ def test_module_metadata_is_fixed_up(): assert trio.Cancelled.__module__ == "trio" assert trio.open_nursery.__module__ == "trio" assert trio.abc.Stream.__module__ == "trio.abc" - assert trio.hazmat.wait_task_rescheduled.__module__ == "trio.hazmat" + assert trio.lowlevel.wait_task_rescheduled.__module__ == "trio.lowlevel" import trio.testing assert trio.testing.trio_test.__module__ == "trio.testing" # Also check methods - assert trio.hazmat.ParkingLot.__init__.__module__ == "trio.hazmat" + assert trio.lowlevel.ParkingLot.__init__.__module__ == "trio.lowlevel" assert trio.abc.Stream.send_all.__module__ == "trio.abc" # And names From 599e170c25c8f9ccb60ff10483c32c19b6291efc Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 30 Apr 2020 17:49:22 -0700 Subject: [PATCH 0069/1498] Update reference-lowlevel text to remove hazmat-y language --- docs/source/reference-lowlevel.rst | 40 ++++++++++++++---------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 0ffa2c2b21..4c7677f1e8 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -1,44 +1,40 @@ -======================================================= +========================================================= Introspecting and extending Trio with ``trio.lowlevel`` -======================================================= +========================================================= .. module:: trio.lowlevel -.. warning:: - You probably don't want to use this module. - -:mod:`trio.lowlevel` is Trio's "hazardous materials" layer: it contains -APIs useful for introspecting and extending Trio. If you're writing -ordinary, everyday code, then you can ignore this module completely. -But sometimes you need something a bit lower level. Here are some -examples of situations where you should reach for :mod:`trio.lowlevel`: +:mod:`trio.lowlevel` contains low-level APIs for introspecting and +extending Trio. If you're writing ordinary, everyday code, then you +can ignore this module completely. But sometimes you need something a +bit lower level. Here are some examples of situations where you should +reach for :mod:`trio.lowlevel`: * You want to implement a new :ref:`synchronization primitive ` that Trio doesn't (yet) provide, like a reader-writer lock. * You want to extract low-level metrics to monitor the health of your application. -* You want to add support for a low-level operating system interface - that Trio doesn't (yet) expose, like watching a filesystem directory - for changes. +* You want to use a low-level operating system interface that Trio + doesn't (yet) provide its own wrappers for, like watching a + filesystem directory for changes. * You want to implement an interface for calling between Trio and another event loop within the same process. * You're writing a debugger and want to visualize Trio's task tree. * You need to interoperate with a C library whose API exposes raw file descriptors. -Using :mod:`trio.lowlevel` isn't really *that* hazardous; in fact you're -already using it – it's how most of the functionality described in -previous chapters is implemented. The APIs described here have -strictly defined and carefully documented semantics, and are perfectly -safe – *if* you read carefully and take proper precautions. Some of -those strict semantics have `nasty big pointy teeth +You don't need to be scared of :mod:`trio.lowlevel`, as long as you +take proper precautions. These are real public APIs, with strictly +defined and carefully documented semantics. But, you should read +carefully and take proper precautions. Some of those strict semantics +have `nasty big pointy teeth `__. If you make a mistake, Trio may not be able to handle it gracefully; conventions and guarantees that are followed strictly in the rest of Trio do not -always apply. Using this module makes it your responsibility to think -through and handle the nasty cases to expose a friendly Trio-style API -to your users. +always apply. When you use this module, it's your job to think about +how you're going to handle the tricky cases so you can expose a +friendly Trio-style API to your users. Debugging and instrumentation From dc297526512e4fa7e9c443fdb24fa9738515b111 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 30 Apr 2020 17:52:29 -0700 Subject: [PATCH 0070/1498] Run yapf --- trio/_channel.py | 4 +++- trio/_unix_pipes.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/trio/_channel.py b/trio/_channel.py index 1a24ece2e4..3ec404a8a2 100644 --- a/trio/_channel.py +++ b/trio/_channel.py @@ -341,7 +341,9 @@ async def aclose(self): assert not self._state.receive_tasks for task in self._state.send_tasks: task.custom_sleep_data._tasks.remove(task) - trio.lowlevel.reschedule(task, Error(trio.BrokenResourceError())) + trio.lowlevel.reschedule( + task, Error(trio.BrokenResourceError()) + ) self._state.send_tasks.clear() self._state.data.clear() await trio.lowlevel.checkpoint() diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index 52df9b81cc..6a961d027d 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -130,7 +130,9 @@ async def send_all(self, data: bytes): try: sent += os.write(self._fd_holder.fd, remaining) except BlockingIOError: - await trio.lowlevel.wait_writable(self._fd_holder.fd) + await trio.lowlevel.wait_writable( + self._fd_holder.fd + ) except OSError as e: if e.errno == errno.EBADF: raise trio.ClosedResourceError( From 47098e523285f6b6f482b6a9aa1881ef4ed7ffd3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 1 May 2020 06:48:00 +0000 Subject: [PATCH 0071/1498] Bump prompt-toolkit from 2.0.10 to 3.0.5 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 2.0.10 to 3.0.5. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/2.0.10...3.0.5) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index df9041404a..e538cccac4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -30,7 +30,7 @@ parso==0.7.0 # via jedi pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==2.0.10 # via ipython +prompt-toolkit==3.0.5 # via ipython ptyprocess==0.6.0 # via pexpect py==1.8.1 # via pytest pycodestyle==2.5.0 # via flake8 @@ -42,7 +42,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in pytest==5.4.1 # via -r test-requirements.in, pytest-cov -six==1.14.0 # via astroid, cryptography, packaging, prompt-toolkit, pyopenssl, traitlets +six==1.14.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in toml==0.10.0 # via pylint From 73d0bd4fe257572dfdf66f908fc0a253076ae023 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 1 May 2020 14:24:19 -0700 Subject: [PATCH 0072/1498] Fix over-zealous search/replace --- newsfragments/476.removal.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/476.removal.rst b/newsfragments/476.removal.rst index e5c8a5619e..c4f8331f8c 100644 --- a/newsfragments/476.removal.rst +++ b/newsfragments/476.removal.rst @@ -1,3 +1,3 @@ -It turns out that everyone got confused by the name ``trio.lowlevel``. +It turns out that everyone got confused by the name ``trio.hazmat``. So that name has been deprecated, and the new name is :mod:`trio.lowlevel`. From cdc2118209d149e3c3ca9743e854311968ebb40f Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 1 May 2020 14:26:31 -0700 Subject: [PATCH 0073/1498] Fix 'from trio.hazmat import ...' --- trio/__init__.py | 4 ++++ trio/hazmat.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 trio/hazmat.py diff --git a/trio/__init__.py b/trio/__init__.py index 452dce130e..30e42da97e 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -121,6 +121,10 @@ issue=810, instead=from_thread, ), + # NOTE: when you remove this, you should also remove the file + # trio/hazmat.py. For details on why we have both, see: + # + # https://github.com/python-trio/trio/pull/1484#issuecomment-622574499 "hazmat": _deprecate.DeprecatedAttribute( lowlevel, diff --git a/trio/hazmat.py b/trio/hazmat.py new file mode 100644 index 0000000000..1d079ae8f8 --- /dev/null +++ b/trio/hazmat.py @@ -0,0 +1,13 @@ +# This file is a hack to work around a quirk in Python's import machinery: +# +# https://github.com/python-trio/trio/pull/1484#issuecomment-622574499 +# +# It should be removed at the same time as the _DeprecatedAttribute("hazmat", +# ...) inside trio/__init__.py. + +from ._deprecate import warn_deprecated + +warn_deprecated("trio.hazmat", "0.15.0", issue=476, instead="trio.lowlevel") +del warn_deprecated + +from .lowlevel import * From f2553c28cb3056713e4b84e7bd1ce861428c1fa6 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 1 May 2020 14:28:21 -0700 Subject: [PATCH 0074/1498] tweak wording --- docs/source/reference-lowlevel.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 4c7677f1e8..53e5cf8a8c 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -26,9 +26,10 @@ reach for :mod:`trio.lowlevel`: You don't need to be scared of :mod:`trio.lowlevel`, as long as you take proper precautions. These are real public APIs, with strictly -defined and carefully documented semantics. But, you should read -carefully and take proper precautions. Some of those strict semantics -have `nasty big pointy teeth +defined and carefully documented semantics. They're the same tools we +use to implement all the nice high-level APIs in the :mod:`trio` +namespace. But, be careful. Some of those strict semantics have `nasty +big pointy teeth `__. If you make a mistake, Trio may not be able to handle it gracefully; conventions and guarantees that are followed strictly in the rest of Trio do not From 29d50ad6153eb7978361f7f4e0beb8a9b44cb2c5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 1 May 2020 18:35:51 -0700 Subject: [PATCH 0075/1498] Redo history file to preserve links while avoiding anachronistic text --- docs/source/history.rst | 65 +++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/docs/source/history.rst b/docs/source/history.rst index 8c119a59cc..320ecbab9a 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -12,8 +12,9 @@ Features ~~~~~~~~ - If you're using Trio's low-level interfaces like - `trio.lowlevel.wait_readable` or similar, and then you close a socket or - file descriptor, you're supposed to call `trio.lowlevel.notify_closing` + `trio.hazmat.wait_readable ` or similar, and then you close a socket or + file descriptor, you're supposed to call `trio.hazmat.notify_closing + ` first so Trio can clean up properly. But what if you forget? In the past, Trio would tend to either deadlock or explode spectacularly. Now, it's much more robust to this situation, and should generally @@ -23,7 +24,7 @@ Features reduction and making it easier to debug this kind of mistake, not something you should rely on.) - If you're using higher-level interfaces outside of the `trio.lowlevel` + If you're using higher-level interfaces outside of the `trio.hazmat ` module, then you don't need to worry about any of this; those intefaces already take care of calling `~trio.lowlevel.notify_closing` for you. (`#1272 `__) @@ -60,7 +61,7 @@ Features `__ is generally the best way to implement async I/O operations – but it's historically been weak at providing ``select``\-style readiness - notifications, like `trio.lowlevel.wait_readable` and + notifications, like `trio.hazmat.wait_readable ` and `~trio.lowlevel.wait_writable`. We aren't willing to give those up, so previously Trio's Windows backend used a hybrid of ``select`` + IOCP. This was complex, slow, and had `limited scalability @@ -162,7 +163,7 @@ Features like `current_time`. Static analysis tools like mypy and pylint should now be able to recognize and analyze all of Trio's top-level functions (though some class attributes are still dynamic... we're working on it). (`#805 `__) -- Add `trio.lowlevel.FdStream` for wrapping a Unix file descriptor as a `~trio.abc.Stream`. (`#829 `__) +- Add `trio.hazmat.FdStream ` for wrapping a Unix file descriptor as a `~trio.abc.Stream`. (`#829 `__) - Trio now gives a reasonable traceback and error message in most cases when its invariants surrounding cancel scope nesting have been violated. (One common source of such violations is an async generator @@ -219,8 +220,9 @@ Deprecations and Removals `trio.to_thread.current_default_thread_limiter`. (`#810 `__) - Give up on trying to have different low-level waiting APIs on Unix and - Windows. All platforms now have `trio.lowlevel.wait_readable`, - `trio.lowlevel.wait_writable`, and `trio.lowlevel.notify_closing`. The old + Windows. All platforms now have `trio.hazmat.wait_readable `, + `trio.hazmat.wait_writable `, and + `trio.hazmat.notify_closing `. The old platform-specific synonyms ``wait_socket_*``, ``notify_socket_closing``, and ``notify_fd_closing`` have been deprecated. (`#878 `__) @@ -358,7 +360,7 @@ Features convenience. (`#4 `__) - You can now create an unbounded :class:`CapacityLimiter` by initializing with `math.inf` (`#618 `__) -- New :mod:`trio.lowlevel` features to allow cleanly switching live coroutine +- New :mod:`trio.hazmat ` features to allow cleanly switching live coroutine objects between Trio and other coroutine runners. Frankly, we're not even sure this is a good idea, but we want to `try it out in trio-asyncio `__, so here we are. @@ -473,9 +475,9 @@ Features ``notify_socket_close`` now work on bare socket descriptors, instead of requiring a :func:`socket.socket` object. (`#400 `__) -- If you're using :func:`trio.lowlevel.wait_task_rescheduled` and other low-level +- If you're using :func:`trio.hazmat.wait_task_rescheduled ` and other low-level routines to implement a new sleeping primitive, you can now use the new - :data:`trio.lowlevel.Task.custom_sleep_data` attribute to pass arbitrary data + :data:`trio.hazmat.Task.custom_sleep_data ` attribute to pass arbitrary data between the sleeping task, abort function, and waking task. (`#616 `__) @@ -516,7 +518,7 @@ Trio 0.6.0 (2018-08-13) Features ~~~~~~~~ -- Add :func:`trio.lowlevel.WaitForSingleObject` async function to await Windows +- Add :func:`trio.hazmat.WaitForSingleObject ` async function to await Windows handles. (`#233 `__) - The `sniffio `__ library can now detect when Trio is running. (`#572 @@ -555,7 +557,7 @@ Features ``trio.hazmat.notify_socket_close``. If you're using Trio's built-in wrappers like :class:`~trio.SocketStream` or :mod:`trio.socket`, then you don't need to worry about this, but if you're using the low-level functions like - :func:`trio.lowlevel.wait_readable`, you should make sure to call these + :func:`trio.hazmat.wait_readable `, you should make sure to call these functions at appropriate times. (`#36 `__) - Tasks created by :func:`~trio.lowlevel.spawn_system_task` now no longer inherit @@ -585,11 +587,11 @@ Features - Add unix client socket support. (`#401 `__) - Add support for :mod:`contextvars` (see :ref:`task-local storage - `), and add :class:`trio.lowlevel.RunVar` as a similar API + `), and add :class:`trio.hazmat.RunVar ` as a similar API for run-local variables. Deprecate ``trio.TaskLocal`` and ``trio.hazmat.RunLocal`` in favor of these new APIs. (`#420 `__) -- Add :func:`trio.lowlevel.current_root_task` to get the root task. (`#452 +- Add :func:`trio.hazmat.current_root_task ` to get the root task. (`#452 `__) @@ -815,7 +817,7 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 ``trio.BlockingTrioPortal`` * The hazmat function ``current_call_soon_thread_and_signal_safe`` - is being replaced by :class:`trio.lowlevel.TrioToken` + is being replaced by :class:`trio.hazmat.TrioToken ` See `#68 `__ for details. @@ -840,14 +842,14 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 * ``task.wait`` This also lets us move a number of lower-level features out of the - main :mod:`trio` namespace and into :mod:`trio.lowlevel`: + main :mod:`trio` namespace and into :mod:`trio.hazmat `: - * ``trio.Task`` → :class:`trio.lowlevel.Task` - * ``trio.current_task`` → :func:`trio.lowlevel.current_task` - * ``trio.Result`` → ``trio.lowlevel.Result`` - * ``trio.Value`` → ``trio.lowlevel.Value`` - * ``trio.Error`` → ``trio.lowlevel.Error`` - * ``trio.UnboundedQueue`` → ``trio.lowlevel.UnboundedQueue`` + * ``trio.Task`` → :class:`trio.hazmat.Task ` + * ``trio.current_task`` → :func:`trio.hazmat.current_task ` + * ``trio.Result`` → ``trio.hazmat.Result`` + * ``trio.Value`` → ``trio.hazmat.Value`` + * ``trio.Error`` → ``trio.hazmat.Error`` + * ``trio.UnboundedQueue`` → ``trio.hazmat.UnboundedQueue`` In addition, several introspection attributes are being renamed: @@ -857,21 +859,22 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 See `#136 `__ for more details. -* To consolidate introspection functionality in :mod:`trio.lowlevel`, +* To consolidate introspection functionality in :mod:`trio.hazmat `, the following functions are moving: - * ``trio.current_clock`` → :func:`trio.lowlevel.current_clock` - * ``trio.current_statistics`` → :func:`trio.lowlevel.current_statistics` + * ``trio.current_clock`` → :func:`trio.hazmat.current_clock ` + * ``trio.current_statistics`` → + :func:`trio.hazmat.current_statistics ` See `#317 `__ for more details. * It was decided that 0.1.0's "yield point" terminology was confusing; we now use :ref:`"checkpoint" ` instead. As part of - this, the following functions in :mod:`trio.lowlevel` are changing + this, the following functions in :mod:`trio.hazmat ` are changing names: - * ``yield_briefly`` → :func:`~trio.lowlevel.checkpoint` + * ``yield_briefly`` → :func:`~trio.hazmat.checkpoint ` * ``yield_briefly_no_cancel`` → :func:`~trio.lowlevel.cancel_shielded_checkpoint` * ``yield_if_cancelled`` → :func:`~trio.lowlevel.checkpoint_if_cancelled` * ``yield_indefinitely`` → :func:`~trio.lowlevel.wait_task_rescheduled` @@ -890,8 +893,8 @@ Upcoming breaking changes with warnings (i.e., stuff that in 0.2.0 `__). * ``trio.current_instruments`` is deprecated. For adding or removing - instrumentation at run-time, see :func:`trio.lowlevel.add_instrument` - and :func:`trio.lowlevel.remove_instrument` (`#257 + instrumentation at run-time, see :func:`trio.hazmat.add_instrument ` + and :func:`trio.hazmat.remove_instrument ` (`#257 `__) Unfortunately, a limitation in PyPy3 5.8 breaks our deprecation @@ -922,7 +925,7 @@ Other changes * New exception ``ResourceBusyError`` -* The :class:`trio.lowlevel.ParkingLot` class (which is used to +* The :class:`trio.hazmat.ParkingLot ` class (which is used to implement many of Trio's synchronization primitives) was rewritten to be simpler and faster (`#272 `__, `#287 @@ -989,7 +992,7 @@ Other changes `__) * PyCharm (and hopefully other IDEs) can now offer better completions - for the :mod:`trio` and :mod:`trio.lowlevel` modules (`#314 + for the :mod:`trio` and :mod:`trio.hazmat ` modules (`#314 `__) * Trio now uses `yapf `__ to From 33337f04da1326bda962a73860b505ccaa5fe73e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 1 May 2020 22:14:04 -0700 Subject: [PATCH 0076/1498] Enable GHA CI on pushes to the master branch We don't really care about this in general, because we run CI on everything before its merged. But codecov cares, because it needs this to figure out the "baseline" for coverage comparisons. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 819c61c5d1..f7d43ccdc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,10 @@ name: CI -on: [pull_request] +on: + push: + branches: + - master + pull_request: jobs: Windows: From fa639d4ee75ec360e2b98eb20e405dbf1cafb919 Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Sat, 2 May 2020 12:56:00 +0200 Subject: [PATCH 0077/1498] Add DeFramed to Trio's awesomeness. --- docs/source/awesome-trio-libraries.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 6d0d649a59..d136d90dfe 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -32,6 +32,7 @@ Web and HTML * `quart-trio `__ - Like Flask, but for Trio. A simple and powerful framework for building async web applications and REST APIs. Tip: this is an ASGI-based framework, so you'll also need an HTTP server with ASGI support. * `hypercorn `__ - An HTTP server for hosting your ASGI apps. Supports HTTP/1.1, HTTP/2, HTTP/3, and Websockets. Can be run as a standalone server, or embedded in a larger Trio app. Use it with ``quart-trio``, or any other Trio-compatible ASGI framework. * `httpx `__ - HTTPX is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. +* `DeFramed `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi Date: Sat, 2 May 2020 23:15:27 +0200 Subject: [PATCH 0078/1498] typo --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index d136d90dfe..0eead76b76 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -32,7 +32,7 @@ Web and HTML * `quart-trio `__ - Like Flask, but for Trio. A simple and powerful framework for building async web applications and REST APIs. Tip: this is an ASGI-based framework, so you'll also need an HTTP server with ASGI support. * `hypercorn `__ - An HTTP server for hosting your ASGI apps. Supports HTTP/1.1, HTTP/2, HTTP/3, and Websockets. Can be run as a standalone server, or embedded in a larger Trio app. Use it with ``quart-trio``, or any other Trio-compatible ASGI framework. * `httpx `__ - HTTPX is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. -* `DeFramed `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi `__ GUI library. Database From cc9f1f1ab793a1dfd68ba63dcb78d8677fce653f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 4 May 2020 09:57:17 +0400 Subject: [PATCH 0079/1498] docs: switch to add_css_file add_stylesheet was deprecated in Sphinx 3.0 --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 9870a88564..1781cc3c01 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -47,7 +47,7 @@ # ...note that this has since grown to contain a bunch of other CSS hacks too # though. def setup(app): - app.add_stylesheet("hackrtd.css") + app.add_css_file("hackrtd.css") # -- General configuration ------------------------------------------------ From 43405f76b32eb4c7a83f9fb2aad2821873539a5c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 4 May 2020 06:52:00 +0000 Subject: [PATCH 0080/1498] Bump ipython from 7.13.0 to 7.14.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.13.0 to 7.14.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.13.0...7.14.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e538cccac4..ea090b6562 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -18,7 +18,7 @@ flake8==3.7.9 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme immutables==0.12 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets -ipython==7.13.0 # via -r test-requirements.in +ipython==7.14.0 # via -r test-requirements.in isort==4.3.21 # via pylint jedi==0.17.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From 94702777b752398df6c39da617a55093f5efadeb Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Mon, 4 May 2020 01:00:18 -0700 Subject: [PATCH 0081/1498] Fix typo in docs: issue 29587 -> issue 29256. --- docs/source/design.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 19db929101..6251f22cdb 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -498,7 +498,7 @@ several reasons: * Controlling our own fate: I/O handling is pretty core to what Trio is about, and :mod:`selectors` is (as of 2017-03-01) somewhat buggy - (e.g. `issue 29587 `__, `issue + (e.g. `issue 29256 `__, `issue 29255 `__). Which isn't a big deal on its own, but since :mod:`selectors` is part of the standard library we can't fix it and ship an updated version; we're stuck From e3cbde06b749a7109e0173d12b50f66e2b0a3886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 4 May 2020 20:10:53 +0300 Subject: [PATCH 0082/1498] Prevented the installation of cffi on PyPy --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3d9ba620a8..852fa616b2 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,8 @@ "sniffio", # cffi 1.12 adds from_buffer(require_writable=True) and ffi.release() # cffi 1.14 fixes memory leak inside ffi.getwinerror() - "cffi>=1.14; os_name == 'nt'", # "cffi is required on windows" + # cffi is required on Windows, except on PyPy where it is built-in + "cffi>=1.14; os_name == 'nt' and implementation_name != 'pypy'", "contextvars>=2.1; python_version < '3.7'" ], # This means, just install *everything* you see under trio/, even if it From 11bfbde3210b71540a7be6b1873e543bc6b14249 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 5 May 2020 06:17:45 +0000 Subject: [PATCH 0083/1498] Bump sphinxcontrib-trio from 1.1.1 to 1.1.2 Bumps [sphinxcontrib-trio](https://github.com/python-trio/sphinxcontrib-trio) from 1.1.1 to 1.1.2. - [Release notes](https://github.com/python-trio/sphinxcontrib-trio/releases) - [Commits](https://github.com/python-trio/sphinxcontrib-trio/compare/v1.1.1...v1.1.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 31e88d374b..dcb85af4f3 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -36,7 +36,7 @@ sphinxcontrib-htmlhelp==1.0.3 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sphinxcontrib-trio==1.1.1 # via -r docs-requirements.in +sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in toml==0.10.0 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in urllib3==1.25.9 # via requests From 306b37a7613f0f6ec0135334e841e0bf60e037ba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 6 May 2020 06:22:37 +0000 Subject: [PATCH 0084/1498] Bump pylint from 2.5.0 to 2.5.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.5.0 to 2.5.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.5.0...pylint-2.5.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ea090b6562..f92f1c81ce 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -37,7 +37,7 @@ pycodestyle==2.5.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.1.1 # via flake8 pygments==2.6.1 # via ipython -pylint==2.5.0 # via -r test-requirements.in +pylint==2.5.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in From 05ef8f94bcf8c2a403d15e7b8aff27948cd54e49 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 6 May 2020 06:23:33 +0000 Subject: [PATCH 0085/1498] Bump astroid from 2.4.0 to 2.4.1 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.4.0 to 2.4.1. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/astroid-2.4.0...astroid-2.4.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ea090b6562..76ddb57056 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ # pip-compile --output-file test-requirements.txt test-requirements.in # astor==0.8.1 # via -r test-requirements.in -astroid==2.4.0 # via pylint +astroid==2.4.1 # via pylint async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.1.0 # via ipython From 2265ddeb15db89592406bc52eb6313cdd9a4ded7 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 6 May 2020 00:24:49 -0700 Subject: [PATCH 0086/1498] Remove support for calling trio.Process() directly Close gh-1109 --- newsfragments/1109.feature.rst | 3 + newsfragments/1109.removal.rst | 3 + trio/_subprocess.py | 166 +++++++++++++++------------------ trio/tests/test_subprocess.py | 8 -- 4 files changed, 81 insertions(+), 99 deletions(-) create mode 100644 newsfragments/1109.feature.rst create mode 100644 newsfragments/1109.removal.rst diff --git a/newsfragments/1109.feature.rst b/newsfragments/1109.feature.rst new file mode 100644 index 0000000000..cbbef01ba3 --- /dev/null +++ b/newsfragments/1109.feature.rst @@ -0,0 +1,3 @@ +It turns out that creating a subprocess can block the parent process +for a surprisingly long time. So `trio.open_process` now uses a worker +thread to avoid blocking the event loop. diff --git a/newsfragments/1109.removal.rst b/newsfragments/1109.removal.rst new file mode 100644 index 0000000000..65344f64b8 --- /dev/null +++ b/newsfragments/1109.removal.rst @@ -0,0 +1,3 @@ +If you want to create a `trio.Process` object, you now have to call +`trio.open_process`; calling ``trio.Process()`` directly was +deprecated in v0.12.0 and has now been removed. diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 265d4f2ef9..6983c839c7 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -2,6 +2,7 @@ import subprocess import sys from typing import Optional +from functools import partial from ._abc import AsyncResource, SendStream, ReceiveStream from ._highlevel_generic import StapledStream @@ -10,6 +11,7 @@ wait_child_exiting, create_pipe_to_child_stdin, create_pipe_from_child_output ) +from ._util import NoPublicConstructor import trio # Linux-specific, but has complex lifetime management stuff so we hard-code it @@ -46,7 +48,7 @@ def pidfd_open(fd, flags): can_try_pidfd_open = False -class Process(AsyncResource): +class Process(AsyncResource, metaclass=NoPublicConstructor): r"""A child process. Like :class:`subprocess.Popen`, but async. This class has no public constructor. To create a child process, use @@ -100,91 +102,18 @@ class Process(AsyncResource): # arbitrarily many threads if wait() keeps getting cancelled. _wait_for_exit_data = None - # After the deprecation period: - # - delete __init__ and _create - # - add metaclass=NoPublicConstructor - # - rename _init to __init__ - # - move most of the code into open_process() - # - put the subprocess.Popen(...) call into a thread - def __init__(self, *args, **kwargs): - trio._deprecate.warn_deprecated( - "directly constructing Process objects", - "0.12.0", - issue=1109, - instead="trio.open_process" - ) - self._init(*args, **kwargs) - - @classmethod - def _create(cls, *args, **kwargs): - self = cls.__new__(cls) - self._init(*args, **kwargs) - return self - - def _init( - self, command, *, stdin=None, stdout=None, stderr=None, **options - ): - for key in ( - 'universal_newlines', 'text', 'encoding', 'errors', 'bufsize' - ): - if options.get(key): - raise TypeError( - "trio.Process only supports communicating over " - "unbuffered byte streams; the '{}' option is not supported" - .format(key) - ) - - self.stdin = None # type: Optional[SendStream] - self.stdout = None # type: Optional[ReceiveStream] - self.stderr = None # type: Optional[ReceiveStream] - self.stdio = None # type: Optional[StapledStream] + def __init__(self, popen, stdin, stdout, stderr): + self._proc = popen + self.stdin = stdin # type: Optional[SendStream] + self.stdout = stdout # type: Optional[ReceiveStream] + self.stderr = stderr # type: Optional[ReceiveStream] - if os.name == "posix": - if isinstance(command, str) and not options.get("shell"): - raise TypeError( - "command must be a sequence (not a string) if shell=False " - "on UNIX systems" - ) - if not isinstance(command, str) and options.get("shell"): - raise TypeError( - "command must be a string (not a sequence) if shell=True " - "on UNIX systems" - ) + self.stdio = None # type: Optional[StapledStream] + if self.stdin is not None and self.stdout is not None: + self.stdio = StapledStream(self.stdin, self.stdout) self._wait_lock = Lock() - if stdin == subprocess.PIPE: - self.stdin, stdin = create_pipe_to_child_stdin() - if stdout == subprocess.PIPE: - self.stdout, stdout = create_pipe_from_child_output() - if stderr == subprocess.STDOUT: - # If we created a pipe for stdout, pass the same pipe for - # stderr. If stdout was some non-pipe thing (DEVNULL or a - # given FD), pass the same thing. If stdout was passed as - # None, keep stderr as STDOUT to allow subprocess to dup - # our stdout. Regardless of which of these is applicable, - # don't create a new Trio stream for stderr -- if stdout - # is piped, stderr will be intermixed on the stdout stream. - if stdout is not None: - stderr = stdout - elif stderr == subprocess.PIPE: - self.stderr, stderr = create_pipe_from_child_output() - - try: - self._proc = subprocess.Popen( - command, stdin=stdin, stdout=stdout, stderr=stderr, **options - ) - finally: - # Close the parent's handle for each child side of a pipe; - # we want the child to have the only copy, so that when - # it exits we can read EOF on our side. - if self.stdin is not None: - os.close(stdin) - if self.stdout is not None: - os.close(stdout) - if self.stderr is not None: - os.close(stderr) - self._pidfd = None if can_try_pidfd_open: try: @@ -200,9 +129,6 @@ def _init( # make sure it'll get closed. self._pidfd = open(fd) - if self.stdin is not None and self.stdout is not None: - self.stdio = StapledStream(self.stdin, self.stdout) - self.args = self._proc.args self.pid = self._proc.pid @@ -378,12 +304,70 @@ async def open_process( specified command could not be found. """ - # XX FIXME: move the process creation into a thread as soon as we're done - # deprecating Process(...) - await trio.lowlevel.checkpoint() - return Process._create( - command, stdin=stdin, stdout=stdout, stderr=stderr, **options - ) + for key in ('universal_newlines', 'text', 'encoding', 'errors', 'bufsize'): + if options.get(key): + raise TypeError( + "trio.Process only supports communicating over " + "unbuffered byte streams; the '{}' option is not supported" + .format(key) + ) + + if os.name == "posix": + if isinstance(command, str) and not options.get("shell"): + raise TypeError( + "command must be a sequence (not a string) if shell=False " + "on UNIX systems" + ) + if not isinstance(command, str) and options.get("shell"): + raise TypeError( + "command must be a string (not a sequence) if shell=True " + "on UNIX systems" + ) + + trio_stdin = None # type: Optional[SendStream] + trio_stdout = None # type: Optional[ReceiveStream] + trio_stderr = None # type: Optional[ReceiveStream] + + if stdin == subprocess.PIPE: + trio_stdin, stdin = create_pipe_to_child_stdin() + if stdout == subprocess.PIPE: + trio_stdout, stdout = create_pipe_from_child_output() + if stderr == subprocess.STDOUT: + # If we created a pipe for stdout, pass the same pipe for + # stderr. If stdout was some non-pipe thing (DEVNULL or a + # given FD), pass the same thing. If stdout was passed as + # None, keep stderr as STDOUT to allow subprocess to dup + # our stdout. Regardless of which of these is applicable, + # don't create a new Trio stream for stderr -- if stdout + # is piped, stderr will be intermixed on the stdout stream. + if stdout is not None: + stderr = stdout + elif stderr == subprocess.PIPE: + trio_stderr, stderr = create_pipe_from_child_output() + + try: + popen = await trio.to_thread.run_sync( + partial( + subprocess.Popen, + command, + stdin=stdin, + stdout=stdout, + stderr=stderr, + **options + ) + ) + finally: + # Close the parent's handle for each child side of a pipe; + # we want the child to have the only copy, so that when + # it exits we can read EOF on our side. + if trio_stdin is not None: + os.close(stdin) + if trio_stdout is not None: + os.close(stdout) + if trio_stderr is not None: + os.close(stderr) + + return Process._create(popen, trio_stdin, trio_stdout, trio_stderr) async def run_process( diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 7fe64564c7..c522847c7d 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -58,14 +58,6 @@ async def test_basic(): ) -# Delete this test when we remove direct Process construction -async def test_deprecated_Process_init(): - with pytest.warns(TrioDeprecationWarning): - async with Process(EXIT_TRUE) as proc: - assert isinstance(proc, Process) - assert proc.returncode == 0 - - async def test_multi_wait(): async with await open_process(SLEEP(10)) as proc: # Check that wait (including multi-wait) tolerates being cancelled From b94e8084acd61f1881a4221ab8962c1a333b82c1 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 6 May 2020 00:27:28 -0700 Subject: [PATCH 0087/1498] Better error messages from Final and NoPublicConstructor This is a pretty trivial change, but I noticed it while working on gh-1496 so I figured I'd just fix it. --- trio/_core/tests/test_run.py | 26 ++++++-------------------- trio/_util.py | 8 ++++++-- trio/tests/test_util.py | 10 +++------- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index fa399d93b4..e705af5c22 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2155,11 +2155,7 @@ async def inner(): def test_Nursery_init(): - check_Nursery_error = pytest.raises( - TypeError, match='no public constructor available' - ) - - with check_Nursery_error: + with pytest.raises(TypeError): _core._run.Nursery(None, None) @@ -2170,23 +2166,17 @@ async def test_Nursery_private_init(): def test_Nursery_subclass(): - with pytest.raises( - TypeError, match='`Nursery` does not support subclassing' - ): + with pytest.raises(TypeError): class Subclass(_core._run.Nursery): pass def test_Cancelled_init(): - check_Cancelled_error = pytest.raises( - TypeError, match='no public constructor available' - ) - - with check_Cancelled_error: + with pytest.raises(TypeError): raise _core.Cancelled - with check_Cancelled_error: + with pytest.raises(TypeError): _core.Cancelled() # private constructor should not raise @@ -2199,18 +2189,14 @@ def test_Cancelled_str(): def test_Cancelled_subclass(): - with pytest.raises( - TypeError, match='`Cancelled` does not support subclassing' - ): + with pytest.raises(TypeError): class Subclass(_core.Cancelled): pass def test_CancelScope_subclass(): - with pytest.raises( - TypeError, match='`CancelScope` does not support subclassing' - ): + with pytest.raises(TypeError): class Subclass(_core.CancelScope): pass diff --git a/trio/_util.py b/trio/_util.py index b50a58036e..5940788859 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -216,7 +216,9 @@ def __new__(cls, name, bases, cls_namespace): for base in bases: if isinstance(base, Final): raise TypeError( - "`%s` does not support subclassing" % base.__name__ + "the {!r} class does not support subclassing".format( + base.__name__ + ) ) return super().__new__(cls, name, bases, cls_namespace) @@ -240,7 +242,9 @@ class SomeClass(metaclass=NoPublicConstructor): - TypeError if a sub class or an instance is created. """ def __call__(self, *args, **kwargs): - raise TypeError("no public constructor available") + raise TypeError( + "the {!r} class has no public constructor".format(self.__name__) + ) def _create(self, *args, **kwargs): return super().__call__(*args, **kwargs) diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 90aec096b1..d9e97e949b 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -100,9 +100,7 @@ def test_final_metaclass(): class FinalClass(metaclass=Final): pass - with pytest.raises( - TypeError, match="`FinalClass` does not support subclassing" - ): + with pytest.raises(TypeError): class SubClass(FinalClass): pass @@ -112,12 +110,10 @@ def test_no_public_constructor_metaclass(): class SpecialClass(metaclass=NoPublicConstructor): pass - with pytest.raises(TypeError, match="no public constructor available"): + with pytest.raises(TypeError): SpecialClass() - with pytest.raises( - TypeError, match="`SpecialClass` does not support subclassing" - ): + with pytest.raises(TypeError): class SubClass(SpecialClass): pass From 4abde75ae4e212aa561971682402b0c556da487d Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 6 May 2020 14:06:31 +0400 Subject: [PATCH 0088/1498] Remove deprecated `clear` method from `trio.Event` --- trio/_sync.py | 8 -------- trio/tests/test_sync.py | 10 ---------- 2 files changed, 18 deletions(-) diff --git a/trio/_sync.py b/trio/_sync.py index 7bb881f772..73854c3cc0 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -62,14 +62,6 @@ def set(self): self._flag = True self._lot.unpark_all() - @deprecated( - "0.12.0", - issue=637, - instead="multiple Event objects or other synchronization primitives" - ) - def clear(self): - self._flag = False - async def wait(self): """Block until the internal flag value becomes True. diff --git a/trio/tests/test_sync.py b/trio/tests/test_sync.py index ab5b26d06e..e4de476993 100644 --- a/trio/tests/test_sync.py +++ b/trio/tests/test_sync.py @@ -40,16 +40,6 @@ async def child(): assert record == ["sleeping", "sleeping", "woken", "woken"] -# When we remove clear() then this test can be removed too -def test_Event_clear(recwarn): - e = Event() - assert not e.is_set() - e.set() - assert e.is_set() - e.clear() - assert not e.is_set() - - async def test_CapacityLimiter(): with pytest.raises(TypeError): CapacityLimiter(1.0) From a6de20a6c8f877289715fe2c19c4077f5acc32ad Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 6 May 2020 14:09:24 +0400 Subject: [PATCH 0089/1498] Add newsfragment --- newsfragments/1498.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1498.removal.rst diff --git a/newsfragments/1498.removal.rst b/newsfragments/1498.removal.rst new file mode 100644 index 0000000000..fffb385ca1 --- /dev/null +++ b/newsfragments/1498.removal.rst @@ -0,0 +1 @@ +Remove ``clear`` method on `trio.Event`: it was deprecated in 0.12.0. From 4807db7db1a29dfbe6f35e5dc36e0511d14f444b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 14:31:43 -0700 Subject: [PATCH 0090/1498] Use fully-qualified names for even clearer error messages --- trio/_util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index 5940788859..4236d4f8cb 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -216,9 +216,7 @@ def __new__(cls, name, bases, cls_namespace): for base in bases: if isinstance(base, Final): raise TypeError( - "the {!r} class does not support subclassing".format( - base.__name__ - ) + f"{base.__module__}.{base.__qualname__} does not support subclassing" ) return super().__new__(cls, name, bases, cls_namespace) @@ -243,7 +241,7 @@ class SomeClass(metaclass=NoPublicConstructor): """ def __call__(self, *args, **kwargs): raise TypeError( - "the {!r} class has no public constructor".format(self.__name__) + f"{self.__module__}.{self.__qualname__} has no public constructor" ) def _create(self, *args, **kwargs): From 902b310f159cfaddb9499e97a967778356964339 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 14:40:59 -0700 Subject: [PATCH 0091/1498] Apply the NoPublicConstructor metaclass to Task and TrioToken These have never had a usable public constructor, so we might as well use the annotation so in case anyone tries they'll get a nice error message. --- trio/_core/_entry_queue.py | 3 ++- trio/_core/_run.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index a8fd8e32bd..869f616b5e 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -4,6 +4,7 @@ import attr from .. import _core +from .._util import NoPublicConstructor from ._wakeup_socketpair import WakeupSocketpair __all__ = ["TrioToken"] @@ -123,7 +124,7 @@ def run_sync_soon(self, sync_fn, *args, idempotent=False): self.wakeup.wakeup_thread_and_signal_safe() -class TrioToken: +class TrioToken(metaclass=NoPublicConstructor): """An opaque object representing a single call to :func:`trio.run`. It has no public constructor; instead, see :func:`current_trio_token`. diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 11d65750e6..5904e682fd 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -968,7 +968,7 @@ def __del__(self): @attr.s(eq=False, hash=False, repr=False) -class Task: +class Task(metaclass=NoPublicConstructor): _parent_nursery = attr.ib() coro = attr.ib() _runner = attr.ib() @@ -1353,7 +1353,7 @@ async def python_wrapper(orig_coro): LOCALS_KEY_KI_PROTECTION_ENABLED, system_task ) - task = Task( + task = Task._create( coro=coro, parent_nursery=nursery, runner=self, @@ -1489,7 +1489,7 @@ def current_trio_token(self): """ if self.trio_token is None: - self.trio_token = TrioToken(self.entry_queue) + self.trio_token = TrioToken._create(self.entry_queue) return self.trio_token ################ From d5929fc8dfa644b258b264912c49dcc2b3a17795 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 14:44:30 -0700 Subject: [PATCH 0092/1498] Add SubclassingDeprecatedIn_v0_15_0 metaclass For when we want to switch to Final, but with a deprecation period first. --- trio/_util.py | 16 ++++++++++++++++ trio/tests/test_util.py | 12 +++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/trio/_util.py b/trio/_util.py index b50a58036e..5cc5319d93 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -9,6 +9,8 @@ import typing as t import threading +from ._deprecate import warn_deprecated + # There's a dependency loop here... _core is allowed to use this file (in fact # it's the *only* file in the main trio/ package it's allowed to use), but # ConflictDetector needs checkpoint so it also has to import @@ -221,6 +223,20 @@ def __new__(cls, name, bases, cls_namespace): return super().__new__(cls, name, bases, cls_namespace) +class SubclassingDeprecatedIn_v0_15_0(BaseMeta): + def __new__(cls, name, bases, cls_namespace): + for base in bases: + if isinstance(base, SubclassingDeprecatedIn_v0_15_0): + warn_deprecated( + f"subclassing {base.__module__}.{base.__qualname__}", + "0.15.0", + issue=1044, + instead="composition or delegation" + ) + break + return super().__new__(cls, name, bases, cls_namespace) + + class NoPublicConstructor(Final): """Metaclass that enforces a class to be final (i.e., subclass not allowed) and ensures a private constructor. diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 90aec096b1..8369ea6da0 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -6,7 +6,7 @@ from .. import _core from .._util import ( signal_raise, ConflictDetector, is_main_thread, generic_function, Final, - NoPublicConstructor + NoPublicConstructor, SubclassingDeprecatedIn_v0_15_0 ) from ..testing import wait_all_tasks_blocked @@ -108,6 +108,16 @@ class SubClass(FinalClass): pass +def test_subclassing_deprecated_metaclass(): + class Blah(metaclass=SubclassingDeprecatedIn_v0_15_0): + pass + + with pytest.warns(trio.TrioDeprecationWarning): + + class Blah2(Blah): + pass + + def test_no_public_constructor_metaclass(): class SpecialClass(metaclass=NoPublicConstructor): pass From 3382551d6bb41090c31b3797329ea4b5bf85c36c Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 16:16:09 -0700 Subject: [PATCH 0093/1498] Deprecate subclassing for most of Trio's public classes --- newsfragments/1044.removal.rst | 17 ++++++++++++++ trio/_core/_local.py | 4 +++- trio/_core/_parking_lot.py | 3 ++- trio/_core/_unbounded_queue.py | 3 ++- trio/_highlevel_generic.py | 6 ++++- trio/_highlevel_socket.py | 10 +++++--- trio/_path.py | 4 ++-- trio/_ssl.py | 8 ++++--- trio/_sync.py | 41 ++++++++++++++++++--------------- trio/_unix_pipes.py | 4 ++-- trio/_windows_pipes.py | 6 ++--- trio/testing/_memory_streams.py | 8 +++++-- trio/testing/_mock_clock.py | 3 ++- trio/testing/_sequencer.py | 2 +- 14 files changed, 79 insertions(+), 40 deletions(-) create mode 100644 newsfragments/1044.removal.rst diff --git a/newsfragments/1044.removal.rst b/newsfragments/1044.removal.rst new file mode 100644 index 0000000000..a1f3868263 --- /dev/null +++ b/newsfragments/1044.removal.rst @@ -0,0 +1,17 @@ +Most of the public classes that Trio exports – like `trio.Lock`, +`trio.SocketStream`, and so on – weren't designed with subclassing in +mind. And we've noticed that some users were trying to subclass them +anyway, and ending up with fragile code that we're likely to +accidentally break in the future, or else be stuck unable to make +changes for fear of breaking subclasses. + +There are also some classes that were explicitly designed to be +subclassed, like the ones in `trio.abc`. Subclassing these is still +supported. However, for all other classes, attempts to subclass will +now raise a deprecation warning, and in the future will raise an +error. + +If this causes problems for you, feel free to drop by our `chat room +`__ or file a bug, to discuss +alternatives or make a case for why some particular class should be +designed to support subclassing. diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 7ff3757356..589f5f5a90 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -1,6 +1,8 @@ # Runvar implementations from . import _run +from .._util import SubclassingDeprecatedIn_v0_15_0 + __all__ = ["RunVar"] @@ -19,7 +21,7 @@ def __init__(self, var, value): self.redeemed = False -class RunVar: +class RunVar(metaclass=SubclassingDeprecatedIn_v0_15_0): """The run-local variant of a context variable. :class:`RunVar` objects are similar to context variable objects, diff --git a/trio/_core/_parking_lot.py b/trio/_core/_parking_lot.py index 696ff90917..0bfc12230b 100644 --- a/trio/_core/_parking_lot.py +++ b/trio/_core/_parking_lot.py @@ -75,6 +75,7 @@ from collections import OrderedDict from .. import _core +from .._util import SubclassingDeprecatedIn_v0_15_0 __all__ = ["ParkingLot"] @@ -87,7 +88,7 @@ class _ParkingLotStatistics: @attr.s(eq=False, hash=False) -class ParkingLot: +class ParkingLot(metaclass=SubclassingDeprecatedIn_v0_15_0): """A fair wait queue with cancellation and requeueing. This class encapsulates the tricky parts of implementing a wait diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index 642a4a25ac..2cb82d1fce 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -2,6 +2,7 @@ from .. import _core from .._deprecate import deprecated +from .._util import SubclassingDeprecatedIn_v0_15_0 __all__ = ["UnboundedQueue"] @@ -12,7 +13,7 @@ class _UnboundedQueueStats: tasks_waiting = attr.ib() -class UnboundedQueue: +class UnboundedQueue(metaclass=SubclassingDeprecatedIn_v0_15_0): """An unbounded queue suitable for certain unusual forms of inter-task communication. diff --git a/trio/_highlevel_generic.py b/trio/_highlevel_generic.py index 79a82d36d8..7cdc0c75d1 100644 --- a/trio/_highlevel_generic.py +++ b/trio/_highlevel_generic.py @@ -3,6 +3,8 @@ import trio from .abc import HalfCloseableStream +from trio._util import SubclassingDeprecatedIn_v0_15_0 + async def aclose_forcefully(resource): """Close an async resource or async generator immediately, without @@ -35,7 +37,9 @@ async def aclose_forcefully(resource): @attr.s(eq=False, hash=False) -class StapledStream(HalfCloseableStream): +class StapledStream( + HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0 +): """This class `staples `__ together two unidirectional streams to make single bidirectional stream. diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index 405375479c..133a9981a1 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -5,7 +5,7 @@ import trio from . import socket as tsocket -from ._util import ConflictDetector +from ._util import ConflictDetector, SubclassingDeprecatedIn_v0_15_0 from .abc import HalfCloseableStream, Listener __all__ = ["SocketStream", "SocketListener"] @@ -39,7 +39,9 @@ def _translate_socket_errors_to_stream_errors(): ) from exc -class SocketStream(HalfCloseableStream): +class SocketStream( + HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0 +): """An implementation of the :class:`trio.abc.HalfCloseableStream` interface based on a raw network socket. @@ -322,7 +324,9 @@ def getsockopt(self, level, option, buffersize=0): pass -class SocketListener(Listener[SocketStream]): +class SocketListener( + Listener[SocketStream], metaclass=SubclassingDeprecatedIn_v0_15_0 +): """A :class:`~trio.abc.Listener` that uses a listening socket to accept incoming connections as :class:`SocketStream` objects. diff --git a/trio/_path.py b/trio/_path.py index 2f34a38972..45fe0e4341 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -4,7 +4,7 @@ import pathlib import trio -from trio._util import async_wraps +from trio._util import async_wraps, SubclassingDeprecatedIn_v0_15_0 __all__ = ['Path'] @@ -77,7 +77,7 @@ async def wrapper(cls, *args, **kwargs): return wrapper -class AsyncAutoWrapperType(type): +class AsyncAutoWrapperType(SubclassingDeprecatedIn_v0_15_0): def __init__(cls, name, bases, attrs): super().__init__(name, bases, attrs) diff --git a/trio/_ssl.py b/trio/_ssl.py index c72e0fc3b0..12182c58af 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -158,7 +158,7 @@ from .abc import Stream, Listener from ._highlevel_generic import aclose_forcefully from . import _sync -from ._util import ConflictDetector +from ._util import ConflictDetector, SubclassingDeprecatedIn_v0_15_0 from ._deprecate import warn_deprecated ################################################################ @@ -224,7 +224,7 @@ def done(self): _State = _Enum("_State", ["OK", "BROKEN", "CLOSED"]) -class SSLStream(Stream): +class SSLStream(Stream, metaclass=SubclassingDeprecatedIn_v0_15_0): r"""Encrypted communication using SSL/TLS. :class:`SSLStream` wraps an arbitrary :class:`~trio.abc.Stream`, and @@ -882,7 +882,9 @@ async def wait_send_all_might_not_block(self): await self.transport_stream.wait_send_all_might_not_block() -class SSLListener(Listener[SSLStream]): +class SSLListener( + Listener[SSLStream], metaclass=SubclassingDeprecatedIn_v0_15_0 +): """A :class:`~trio.abc.Listener` for SSL/TLS-encrypted servers. :class:`SSLListener` wraps around another Listener, and converts diff --git a/trio/_sync.py b/trio/_sync.py index 73854c3cc0..d5a5d04372 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -7,6 +7,7 @@ from ._core import enable_ki_protection, ParkingLot from ._deprecate import deprecated +from ._util import SubclassingDeprecatedIn_v0_15_0 __all__ = [ "Event", @@ -19,7 +20,7 @@ @attr.s(repr=False, eq=False, hash=False) -class Event: +class Event(metaclass=SubclassingDeprecatedIn_v0_15_0): """A waitable boolean value useful for inter-task synchronization, inspired by :class:`threading.Event`. @@ -111,7 +112,7 @@ class _CapacityLimiterStatistics: @async_cm -class CapacityLimiter: +class CapacityLimiter(metaclass=SubclassingDeprecatedIn_v0_15_0): """An object for controlling access to a resource with limited capacity. Sometimes you need to put a limit on how many tasks can do something at @@ -363,7 +364,7 @@ def statistics(self): @async_cm -class Semaphore: +class Semaphore(metaclass=SubclassingDeprecatedIn_v0_15_0): """A `semaphore `__. A semaphore holds an integer value, which can be incremented by @@ -499,19 +500,7 @@ class _LockStatistics: @async_cm @attr.s(eq=False, hash=False, repr=False) -class Lock: - """A classic `mutex - `__. - - This is a non-reentrant, single-owner lock. Unlike - :class:`threading.Lock`, only the owner of the lock is allowed to release - it. - - A :class:`Lock` object can be used as an async context manager; it - blocks on entry but not on exit. - - """ - +class _LockImpl: _lot = attr.ib(factory=ParkingLot, init=False) _owner = attr.ib(default=None, init=False) @@ -606,7 +595,21 @@ def statistics(self): ) -class StrictFIFOLock(Lock): +class Lock(_LockImpl, metaclass=SubclassingDeprecatedIn_v0_15_0): + """A classic `mutex + `__. + + This is a non-reentrant, single-owner lock. Unlike + :class:`threading.Lock`, only the owner of the lock is allowed to release + it. + + A :class:`Lock` object can be used as an async context manager; it + blocks on entry but not on exit. + + """ + + +class StrictFIFOLock(_LockImpl, metaclass=SubclassingDeprecatedIn_v0_15_0): r"""A variant of :class:`Lock` where tasks are guaranteed to acquire the lock in strict first-come-first-served order. @@ -658,7 +661,7 @@ class StrictFIFOLock(Lock): :class:`StrictFIFOLock` guarantees that each task will send its data in the same order that the state machine generated it. - Currently, :class:`StrictFIFOLock` is simply an alias for :class:`Lock`, + Currently, :class:`StrictFIFOLock` is identical to :class:`Lock`, but (a) this may not always be true in the future, especially if Trio ever implements `more sophisticated scheduling policies `__, and (b) the above code @@ -676,7 +679,7 @@ class _ConditionStatistics: @async_cm -class Condition: +class Condition(metaclass=SubclassingDeprecatedIn_v0_15_0): """A classic `condition variable `__, similar to :class:`threading.Condition`. diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index 6a961d027d..f2afe8d2b1 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -2,7 +2,7 @@ import errno from ._abc import Stream -from ._util import ConflictDetector +from ._util import ConflictDetector, SubclassingDeprecatedIn_v0_15_0 import trio @@ -74,7 +74,7 @@ async def aclose(self): await trio.lowlevel.checkpoint() -class FdStream(Stream): +class FdStream(Stream, metaclass=SubclassingDeprecatedIn_v0_15_0): """ Represents a stream given the file descriptor to a pipe, TTY, etc. diff --git a/trio/_windows_pipes.py b/trio/_windows_pipes.py index 04bcdc7100..6af69f364a 100644 --- a/trio/_windows_pipes.py +++ b/trio/_windows_pipes.py @@ -1,6 +1,6 @@ from . import _core from ._abc import SendStream, ReceiveStream -from ._util import ConflictDetector +from ._util import ConflictDetector, Final from ._core._windows_cffi import _handle, raise_winerror, kernel32, ffi # XX TODO: don't just make this up based on nothing. @@ -37,7 +37,7 @@ def __del__(self): self._close() -class PipeSendStream(SendStream): +class PipeSendStream(SendStream, metaclass=Final): """Represents a send stream over a Windows named pipe that has been opened in OVERLAPPED mode. """ @@ -79,7 +79,7 @@ async def aclose(self): await self._handle_holder.aclose() -class PipeReceiveStream(ReceiveStream): +class PipeReceiveStream(ReceiveStream, metaclass=Final): """Represents a receive stream over an os.pipe object.""" def __init__(self, handle: int) -> None: self._handle_holder = _HandleHolder(handle) diff --git a/trio/testing/_memory_streams.py b/trio/testing/_memory_streams.py index d86e301888..184626d58b 100644 --- a/trio/testing/_memory_streams.py +++ b/trio/testing/_memory_streams.py @@ -83,7 +83,9 @@ async def get(self, max_bytes=None): return self._get_impl(max_bytes) -class MemorySendStream(SendStream): +class MemorySendStream( + SendStream, metaclass=_util.SubclassingDeprecatedIn_v0_15_0 +): """An in-memory :class:`~trio.abc.SendStream`. Args: @@ -198,7 +200,9 @@ def get_data_nowait(self, max_bytes=None): return self._outgoing.get_nowait(max_bytes) -class MemoryReceiveStream(ReceiveStream): +class MemoryReceiveStream( + ReceiveStream, metaclass=_util.SubclassingDeprecatedIn_v0_15_0 +): """An in-memory :class:`~trio.abc.ReceiveStream`. Args: diff --git a/trio/testing/_mock_clock.py b/trio/testing/_mock_clock.py index c3cf863392..2e0667ab94 100644 --- a/trio/testing/_mock_clock.py +++ b/trio/testing/_mock_clock.py @@ -3,6 +3,7 @@ from .. import _core from .._abc import Clock +from .._util import SubclassingDeprecatedIn_v0_15_0 __all__ = ["MockClock"] @@ -14,7 +15,7 @@ # Prior art: # https://twistedmatrix.com/documents/current/api/twisted.internet.task.Clock.html # https://github.com/ztellman/manifold/issues/57 -class MockClock(Clock): +class MockClock(Clock, metaclass=SubclassingDeprecatedIn_v0_15_0): """A user-controllable clock suitable for writing tests. Args: diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index 05b7560eec..1ff190e985 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -14,7 +14,7 @@ @attr.s(eq=False, hash=False) -class Sequencer: +class Sequencer(metaclass=_util.SubclassingDeprecatedIn_v0_15_0): """A convenience class for forcing code in different tasks to run in an explicit linear order. From db179cbfa44f9c95e590b430ddc17ead350344b0 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 17:54:25 -0700 Subject: [PATCH 0094/1498] Add a test to confirm that all public classes are Final, or have a good excuse --- trio/tests/test_exports.py | 56 +++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 6085182a0e..af34070267 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -1,6 +1,8 @@ import sys import importlib import types +import inspect +import enum import pytest @@ -8,6 +10,7 @@ import trio.testing from .. import _core +from .. import _util def test_core_is_properly_reexported(): @@ -28,25 +31,26 @@ def test_core_is_properly_reexported(): assert found == 1 -def public_namespaces(module): - yield module.__name__ - for name, value in module.__dict__.items(): +def public_modules(module): + yield module + for name, class_ in module.__dict__.items(): if name.startswith("_"): continue - if not isinstance(value, types.ModuleType): + if not isinstance(class_, types.ModuleType): continue - if not value.__name__.startswith(module.__name__): + if not class_.__name__.startswith(module.__name__): continue - if value is module: + if class_ is module: continue # We should rename the trio.tests module (#274), but until then we use # a special-case hack: - if value.__name__ == "trio.tests": + if class_.__name__ == "trio.tests": continue - yield from public_namespaces(value) + yield from public_modules(class_) -NAMESPACES = list(public_namespaces(trio)) +PUBLIC_MODULES = list(public_modules(trio)) +PUBLIC_MODULE_NAMES = [m.__name__ for m in PUBLIC_MODULES] # It doesn't make sense for downstream redistributors to run this test, since @@ -64,7 +68,7 @@ def public_namespaces(module): # https://github.com/PyCQA/astroid/issues/681 "ignore:the imp module is deprecated.*:DeprecationWarning" ) -@pytest.mark.parametrize("modname", NAMESPACES) +@pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) @pytest.mark.parametrize("tool", ["pylint", "jedi"]) def test_static_tool_sees_all_symbols(tool, modname): module = importlib.import_module(modname) @@ -106,3 +110,35 @@ def no_underscores(symbols): for name in sorted(missing_names): print(" {}".format(name)) assert False + + +def test_classes_are_final(): + for module in PUBLIC_MODULES: + for name, class_ in module.__dict__.items(): + if not isinstance(class_, type): + continue + if name.startswith("_"): + continue + + # Abstract classes can be subclassed, because that's the whole + # point of ABCs + if inspect.isabstract(class_): + continue + # Exceptions are allowed to be subclassed, because exception + # subclassing isn't used to inherit behavior. + if issubclass(class_, BaseException): + continue + # These are classes that are conceptually abstract, but + # inspect.isabstract returns False for boring reasons. + if class_ in {trio.abc.Instrument, trio.socket.SocketType}: + continue + # Enums have their own metaclass, so we can't use our metaclasses. + # And I don't think there's a lot of risk from people subclassing + # enums... + if issubclass(class_, enum.Enum): + continue + # ... insert other special cases here ... + + assert isinstance( + class_, (_util.Final, _util.SubclassingDeprecatedIn_v0_15_0) + ) From 7e10cd547aab92e4f650e563375b75c734f6055d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 21:57:42 -0700 Subject: [PATCH 0095/1498] Don't try to link to trio.abc --- newsfragments/1044.removal.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/1044.removal.rst b/newsfragments/1044.removal.rst index a1f3868263..7939eafc00 100644 --- a/newsfragments/1044.removal.rst +++ b/newsfragments/1044.removal.rst @@ -6,7 +6,7 @@ accidentally break in the future, or else be stuck unable to make changes for fear of breaking subclasses. There are also some classes that were explicitly designed to be -subclassed, like the ones in `trio.abc`. Subclassing these is still +subclassed, like the ones in ``trio.abc``. Subclassing these is still supported. However, for all other classes, attempts to subclass will now raise a deprecation warning, and in the future will raise an error. From d535bb527717d7573de38ebed47dae522e195f88 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 22:01:52 -0700 Subject: [PATCH 0096/1498] Fix trio.Lock docs --- docs/source/reference-core.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index d868f9b1e7..b5f6d7c4d8 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1444,8 +1444,14 @@ don't have any special access to Trio's internals.) .. autoclass:: Semaphore :members: +.. We have to use :inherited-members: here because all the actual lock + methods are stashed in _LockImpl. Weird side-effect of having both + Lock and StrictFIFOLock, but wanting both to be marked Final so + neither can inherit from the other. + .. autoclass:: Lock :members: + :inherited-members: .. autoclass:: StrictFIFOLock :members: From 03f2601503d5d49987c280e0749accbd50a2bb9b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 22:20:49 -0700 Subject: [PATCH 0097/1498] Add CI for Alpine Also deletes some old dead code in ci.sh --- .github/workflows/ci.yml | 24 ++++++++++++++++++++--- ci.sh | 41 +++++----------------------------------- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7d43ccdc5..04a96c495c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,8 @@ jobs: # Should match 'name:' up above JOB_NAME: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' - Linux: - name: 'Linux (${{ matrix.python }}${{ matrix.extra_name }})' + Ubuntu: + name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' timeout-minutes: 10 runs-on: 'ubuntu-latest' strategy: @@ -74,7 +74,7 @@ jobs: CHECK_DOCS: '${{ matrix.check_docs }}' CHECK_FORMATTING: '${{ matrix.check_formatting }}' # Should match 'name:' up above - JOB_NAME: 'Linux (${{ matrix.python }}${{ matrix.extra_name }})' + JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' macOS: name: 'macOS (${{ matrix.python }})' @@ -96,3 +96,21 @@ jobs: env: # Should match 'name:' up above JOB_NAME: 'macOS (${{ matrix.python }})' + + Alpine: + name: 'Alpine' + timeout-minutes: 10 + runs-on: 'ubuntu-latest' + container: 'alpine' + strategy: + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install bash + run: apk add --no-cache bash + - name: Run tests + run: ./ci.sh + env: + # Should match 'name:' up above + JOB_NAME: 'Alpine' diff --git a/ci.sh b/ci.sh index 23d10210c9..52a1d79efd 100755 --- a/ci.sh +++ b/ci.sh @@ -34,43 +34,12 @@ function curl-harder() { # Bootstrap python environment, if necessary ################################################################ -### Azure pipelines + Windows ### - -# On azure pipeline's windows VMs, to get reasonable performance, we need to -# jump through hoops to avoid touching the C:\ drive as much as possible. -if [ "$AGENT_OS" = "Windows_NT" ]; then - # By default temp and cache directories are on C:\. Fix that. - export TEMP="${AGENT_TEMPDIRECTORY}" - export TMP="${AGENT_TEMPDIRECTORY}" - export TMPDIR="${AGENT_TEMPDIRECTORY}" - export PIP_CACHE_DIR="${AGENT_TEMPDIRECTORY}\\pip-cache" - - # Download and install Python from scratch onto D:\, instead of using the - # pre-installed versions that azure pipelines provides on C:\. - # Also use -DirectDownload to stop nuget from caching things on C:\. - nuget install "${PYTHON_PKG}" -Version "${PYTHON_VERSION}" \ - -OutputDirectory "$PWD/pyinstall" -ExcludeVersion \ - -Source "https://api.nuget.org/v3/index.json" \ - -Verbosity detailed -DirectDownload -NonInteractive - - pydir="$PWD/pyinstall/${PYTHON_PKG}" - export PATH="${pydir}/tools:${pydir}/tools/scripts:$PATH" -fi +### Alpine ### -### Travis + macOS ### - -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - JOB_NAME="osx_${MACPYTHON}" - curl-harder -o macpython.pkg https://www.python.org/ftp/python/${MACPYTHON}/python-${MACPYTHON}-macosx10.6.pkg - sudo installer -pkg macpython.pkg -target / - ls /Library/Frameworks/Python.framework/Versions/*/bin/ - PYTHON_EXE=/Library/Frameworks/Python.framework/Versions/*/bin/python3 - # The pip in older MacPython releases doesn't support a new enough TLS - curl-harder -o get-pip.py https://bootstrap.pypa.io/get-pip.py - sudo $PYTHON_EXE get-pip.py - sudo $PYTHON_EXE -m pip install virtualenv - $PYTHON_EXE -m virtualenv testenv - source testenv/bin/activate +if [ -e /etc/alpine-release ]; then + apk add --no-cache gcc musl-dev libffi-dev openssl-dev python3-dev curl + python3 -m venv venv + source venv/bin/activate fi ### PyPy nightly (currently on Travis) ### From e95fd32a1f7b45d112289cf115c524e79f44f72c Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 23:14:15 -0700 Subject: [PATCH 0098/1498] Fix spurious test failure on Alpine --- newsfragments/1499.bugfix.rst | 2 ++ trio/tests/test_socket.py | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 newsfragments/1499.bugfix.rst diff --git a/newsfragments/1499.bugfix.rst b/newsfragments/1499.bugfix.rst new file mode 100644 index 0000000000..2b6a61f603 --- /dev/null +++ b/newsfragments/1499.bugfix.rst @@ -0,0 +1,2 @@ +Fixed an over-strict test that caused failures on Alpine Linux. +Started testing against Alpine in CI. diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index c03c8bb8ff..47ba558d6b 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -101,11 +101,19 @@ def test_socket_has_some_reexports(): async def test_getaddrinfo(monkeygai): def check(got, expected): # win32 returns 0 for the proto field - def without_proto(gai_tup): - return gai_tup[:2] + (0,) + gai_tup[3:] - - expected2 = [without_proto(gt) for gt in expected] - assert got == expected or got == expected2 + # alpine and glibc have inconsistent handling of the canonical name + # field (https://github.com/python-trio/trio/issues/1499) + # Neither gets used much and there isn't much opportunity for us to + # mess them up, so we don't bother checking them + def interesting_fields(gai_tup): + # (family, type, proto, canonname, sockaddr) + family, type, proto, canonname, sockaddr = gai_tup + return (family, type, sockaddr) + + def filtered(gai_list): + return [interesting_fields(gai_tup) for gai_tup in gai_list] + + assert filtered(got) == filtered(expected) # Simple non-blocking non-error cases, ipv4 and ipv6: with assert_checkpoints(): From 68261db43fe79c3a8b5a22659de159fe40329daf Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 23:31:24 -0700 Subject: [PATCH 0099/1498] Another small test tweak to work around an Alpine quirk --- trio/tests/test_socket.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 47ba558d6b..8be7b7ea46 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -151,8 +151,10 @@ def filtered(gai_list): with assert_checkpoints(): with pytest.raises(tsocket.gaierror) as excinfo: await tsocket.getaddrinfo("::1", "12345", type=-1) - # Linux, Windows + # Linux + glibc, Windows expected_errnos = {tsocket.EAI_SOCKTYPE} + # Linux + musl + expected_errnos.add(tsocket.EAI_SERVICE) # macOS if hasattr(tsocket, "EAI_BADHINTS"): expected_errnos.add(tsocket.EAI_BADHINTS) From cdbbd2d84823006309b2aed9b8a86cd86b03670a Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 7 May 2020 23:38:15 -0700 Subject: [PATCH 0100/1498] Slightly more accurate comment --- trio/tests/test_socket.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 8be7b7ea46..4e76711d34 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -101,10 +101,10 @@ def test_socket_has_some_reexports(): async def test_getaddrinfo(monkeygai): def check(got, expected): # win32 returns 0 for the proto field - # alpine and glibc have inconsistent handling of the canonical name + # musl and glibc have inconsistent handling of the canonical name # field (https://github.com/python-trio/trio/issues/1499) - # Neither gets used much and there isn't much opportunity for us to - # mess them up, so we don't bother checking them + # Neither field gets used much and there isn't much opportunity for us + # to mess them up, so we don't bother checking them here def interesting_fields(gai_tup): # (family, type, proto, canonname, sockaddr) family, type, proto, canonname, sockaddr = gai_tup From bd9313c93c537c004be7d1d2c61fc7907cbd0971 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 10 May 2020 00:24:00 -0700 Subject: [PATCH 0101/1498] Add FreeBSD-specific socket constants --- trio/socket.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/socket.py b/trio/socket.py index 3a0a61b4bf..fa51f6dbf1 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -106,7 +106,8 @@ CAN_BCM_TX_ANNOUNCE, CAN_BCM_TX_COUNTEVT, CAN_BCM_TX_CP_CAN_ID, CAN_BCM_TX_RESET_MULTI_IDX, IPPROTO_CBT, IPPROTO_ICLFXBM, IPPROTO_IGP, IPPROTO_L2TP, IPPROTO_PGM, IPPROTO_RDP, IPPROTO_ST, AF_QIPCRTR, - CAN_BCM_CAN_FD_FRAME + CAN_BCM_CAN_FD_FRAME, IPPROTO_MOBILE, IPV6_USE_MIN_MTU, + MSG_NOTIFICATION, SO_SETFIB ) except ImportError: pass From 4d6cd20c942d0d97b16472e3948c376d1fb4bd5b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 10 May 2020 00:24:19 -0700 Subject: [PATCH 0102/1498] FreeBSD gives a different errno in this case than macOS --- trio/_core/_io_kqueue.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index e3989152ea..40851ca468 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -3,6 +3,7 @@ import outcome from contextlib import contextmanager import attr +import errno from .. import _core from ._run import _public @@ -122,16 +123,21 @@ def abort(_): event = select.kevent(fd, filter, select.KQ_EV_DELETE) try: self._kqueue.control([event], 0) - except FileNotFoundError: + except OSError as exc: # kqueue tracks individual fds (*not* the underlying file # object, see _io_epoll.py for a long discussion of why this # distinction matters), and automatically deregisters an event # if the fd is closed. So if kqueue.control says that it # doesn't know about this event, then probably it's because - # the fd was closed behind our backs. (Too bad it doesn't tell - # us that this happened... oh well, you can't have - # everything.) - pass + # the fd was closed behind our backs. (Too bad we can't ask it + # to wake us up when this happens, versus discovering it after + # the fact... oh well, you can't have everything.) + # + # FreeBSD reports this using EBADF. macOS uses ENOENT. + if exc.errno in (errno.EBADF, errno.ENOENT): + pass + else: + raise return _core.Abort.SUCCEEDED await self.wait_kevent(fd, filter, abort) From b02656b6d338c00cdd62d0d7a64e46823e90c187 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 10 May 2020 00:24:34 -0700 Subject: [PATCH 0103/1498] Adjust measure_backlog to work on FreeBSD --- trio/tests/test_highlevel_open_tcp_listeners.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index 2bee358ee5..baf06c380e 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -74,8 +74,16 @@ async def measure_backlog(listener, limit): # the SYN gets dropped, and the client retries after 1 second. So # we assume that any connect() call to localhost that takes >0.5 # seconds indicates a dropped SYN. + # + # Exception: on FreeBSD when the backlog is exhausted, connect + # raises ConnectionResetError. with trio.move_on_after(0.5) as cancel_scope: - client_stream = await open_stream_to_socket_listener(listener) + try: + client_stream = await open_stream_to_socket_listener( + listener + ) + except ConnectionResetError: + break client_streams.append(client_stream) if cancel_scope.cancelled_caught: break From 0bc4e05ff8091667e63bcb1fd94e1ff3cb23fc3d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 10 May 2020 12:57:04 -0700 Subject: [PATCH 0104/1498] Skip a test that can't be implemented on FreeBSD --- trio/tests/test_unix_pipes.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/trio/tests/test_unix_pipes.py b/trio/tests/test_unix_pipes.py index 9349139e23..395e54e838 100644 --- a/trio/tests/test_unix_pipes.py +++ b/trio/tests/test_unix_pipes.py @@ -2,6 +2,7 @@ import select import os import tempfile +import sys import pytest @@ -236,6 +237,14 @@ async def patched_wait_writable(*args, **kwargs): await r.receive_some(10000) +# On FreeBSD, directories are readable, and we haven't found any other trick +# for making an unreadable fd, so there's no way to run this test. Fortunately +# the logic this is testing doesn't depend on the platform, so testing on +# other platforms is probably good enough. +@pytest.mark.skipif( + sys.platform.startswith("freebsd"), + reason="no way to make read() return a bizarro error on FreeBSD" +) async def test_bizarro_OSError_from_receive(): # Make sure that if the read syscall returns some bizarro error, then we # get a BrokenResourceError. This is incredibly unlikely; there's almost From 1ff9c50a704ea1fc49d621053eb092a10af63bb4 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 10 May 2020 13:41:35 -0700 Subject: [PATCH 0105/1498] Document upstream FreeBSD bug with kqueue/pipe handling, and skip two affected tests --- notes-to-self/fbsd-pipe-close-notify.py | 38 +++++++++++++++++++++++++ trio/tests/test_subprocess.py | 5 ++++ trio/tests/test_unix_pipes.py | 5 ++++ 3 files changed, 48 insertions(+) create mode 100644 notes-to-self/fbsd-pipe-close-notify.py diff --git a/notes-to-self/fbsd-pipe-close-notify.py b/notes-to-self/fbsd-pipe-close-notify.py new file mode 100644 index 0000000000..7b18f65d6f --- /dev/null +++ b/notes-to-self/fbsd-pipe-close-notify.py @@ -0,0 +1,38 @@ +# This script completes correctly on macOS and FreeBSD 13.0-CURRENT, but hangs +# on FreeBSD 12.1. I'm told the fix will be backported to 12.2 (which is due +# out in October 2020). +# +# Upstream bug: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 + +import select +import os +import threading + +r, w = os.pipe() + +os.set_blocking(w, False) + +print("filling pipe buffer") +while True: + try: + os.write(w, b"x") + except BlockingIOError: + break + +_, wfds, _ = select.select([], [w], [], 0) +print("select() says the write pipe is", "writable" if w in wfds else "NOT writable") + +kq = select.kqueue() +event = select.kevent(w, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) +kq.control([event], 0) + +print("closing read end of pipe") +os.close(r) + +_, wfds, _ = select.select([], [w], [], 0) +print("select() says the write pipe is", "writable" if w in wfds else "NOT writable") + +print("waiting for kqueue to report the write end is writable") +got = kq.control([], 1) +print("done!") +print(got) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index c522847c7d..b9d94d15d0 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -254,6 +254,11 @@ async def test_run_check(): assert result.returncode == 1 +# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 +@pytest.mark.skipif( + os.uname().sysname == "FreeBSD" and os.uname().release[:4] < "12.2", + reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350" +) async def test_run_with_broken_pipe(): result = await run_process( [sys.executable, "-c", "import sys; sys.stdin.close()"], diff --git a/trio/tests/test_unix_pipes.py b/trio/tests/test_unix_pipes.py index 395e54e838..e9a07b8e65 100644 --- a/trio/tests/test_unix_pipes.py +++ b/trio/tests/test_unix_pipes.py @@ -264,5 +264,10 @@ async def test_bizarro_OSError_from_receive(): os.close(dir_fd) +# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 +@pytest.mark.skipif( + os.uname().sysname == "FreeBSD" and os.uname().release[:4] < "12.2", + reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350" +) async def test_pipe_fully(): await check_one_way_stream(make_pipe, make_clogged_pipe) From 656dd44e6662795be5216e0ef72a2c328f5b6394 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 11 May 2020 06:46:56 +0000 Subject: [PATCH 0106/1498] Bump pytest from 5.4.1 to 5.4.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.1 to 5.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.1...5.4.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 15d90350fa..ef1092850c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ pylint==2.5.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in -pytest==5.4.1 # via -r test-requirements.in, pytest-cov +pytest==5.4.2 # via -r test-requirements.in, pytest-cov six==1.14.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in From ef553e80e29747c45cccda93e8037f7e938d5ee5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 10 May 2020 18:03:34 -0700 Subject: [PATCH 0107/1498] Run CI on FreeBSD --- .travis.yml | 16 +++- ci.sh | 216 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 226 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 576f0b9ae5..5b75911edf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,19 @@ jobs: - language: generic env: PYPY_NIGHTLY_BRANCH=py3.6 # Qemu tests are also slow + # FreeBSD: + - language: generic + dist: bionic + env: + - "JOB_NAME='FreeBSD 12.1-RELEASE, full VM'" + - "FREEBSD_INSTALLER_ISO_XZ=https://download.freebsd.org/ftp/releases/amd64/amd64/ISO-IMAGES/12.1/FreeBSD-12.1-RELEASE-amd64-disc1.iso.xz" + # Increment this each time you change the image build code in + # ci.sh and want to intentionally bust the cache. + - "CACHE_GEN=2" + cache: + directories: + - travis-cache + # More recent Ubuntu: # The unique thing this provides is testing on the given distro's # kernel, which is important when we use new kernel features. This # is also good for testing the latest openssl etc., and getting @@ -20,7 +33,8 @@ jobs: dist: bionic env: - "JOB_NAME='Ubuntu 19.10, full VM'" - - "VM_IMAGE=https://cloud-images.ubuntu.com/eoan/current/eoan-server-cloudimg-amd64.img" + - "LINUX_VM_IMAGE=https://cloud-images.ubuntu.com/eoan/current/eoan-server-cloudimg-amd64.img" + - python: 3.6.1 # earliest 3.6 version available on Travis - python: 3.6-dev - python: 3.7-dev diff --git a/ci.sh b/ci.sh index 23d10210c9..4ede5ee36d 100755 --- a/ci.sh +++ b/ci.sh @@ -103,9 +103,215 @@ if [ "$PYPY_NIGHTLY_BRANCH" != "" ]; then source testenv/bin/activate fi -### Qemu virtual-machine inception, on Travis +### FreeBSD-in-Qemu virtual-machine inception, on Travis + +# This is complex, because there aren't pre-made images available. So we hack +# up an install CD, run it to build an image, and use that. But that's slow +# and we don't want to do it every run, so we try to get Travis to cache the +# image for us. +# +# Additional subtlety: we actually re-use the same image in-place, and let +# Travis re-cache it every time. The point of this is that it saves our system +# packages + pip cache, so we don't have to recreate them from scratch every +# time. (Especially useful since there are no FreeBSD wheels on PyPI, and we +# don't want to have to build packages like cryptography from source every +# time.) +# +# You can run this locally for testing. But some things to watch out for: +# +# - It'll modify /etc/exports on your host machine. You might want to clean +# it up after. +# - It'll create a symlink at /host-files on your host machine. You might want +# to clean it up after. +# - If you don't want to keep downloading the installer ISO over and over, +# then drop an unpacked copy at ./local-freebsd-installer.iso + +if [ "$FREEBSD_INSTALLER_ISO_XZ" != "" ]; then + sudo apt update + sudo apt install qemu-system-x86 qemu-utils nfs-kernel-server + + if [ ! -e travis-cache/image.qcow2 ]; then + echo "--- No cached FreeBSD VM image; recreating from scratch ---" + sudo apt install growisofs genisoimage + rm -rf scratch + mkdir scratch + qemu-img create -f qcow2 scratch/image.qcow2 10G + if [ -e local-freebsd-installer.iso ]; then + cp local-freebsd-installer.iso scratch/installer.iso + else + curl-harder "$FREEBSD_INSTALLER_ISO_XZ" -o scratch/installer.iso.xz + unxz scratch/installer.iso.xz + fi + + # Files that we want to add to the ISO, to convert it into an + # unattended-installer: + mkdir -p scratch/overlay/etc + mkdir -p scratch/overlay/boot + # Use serial console, and disable the normal 10 second pause before + # booting + cat >scratch/overlay/boot/loader.conf.local <scratch/overlay/etc/rc.local <scratch/overlay/etc/installerconfig <> /etc/rc.conf +/bin/cp /usr/share/zoneinfo/UTC /etc/localtime + +echo 'autoboot_delay=0' >> /boot/loader.conf.local +echo 'console=comconsole' >> /boot/loader.conf.local + +dhclient em0 +export ASSUME_ALWAYS_YES=true +pkg install bash curl python38 py38-sqlite3 + +mkdir /host-files +cat >>/etc/rc.local <freebsd-wrapper.sh < Date: Mon, 11 May 2020 02:13:24 -0700 Subject: [PATCH 0108/1498] On FreeBSD VM, use tmpfs for pytest working dir Coverage.py uses the working dir to store its data, in an sqlite database. Apparently, doing that over NFS is incredibly slow. Without this, tests involving subprocesses in particular keep failing, because spawning a trivial Python process could take 10 seconds (!) before it started running any code. With this, it becomes super-quick again. --- ci.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 4ede5ee36d..80d64cbd05 100755 --- a/ci.sh +++ b/ci.sh @@ -274,6 +274,11 @@ set +u source /venv/bin/activate set -u +# And put a tmpfs on the empty dir as well, because coverage uses it for +# storage, and this makes it *massively* faster than if it's on NFS +mkdir empty +mount -t tmpfs tmpfs ./empty + # And then we re-invoke ourselves! bash ./ci.sh @@ -462,7 +467,10 @@ else netsh winsock show catalog fi - mkdir empty + # We run the tests from inside an empty directory, to make sure Python + # doesn't pick up any .py files from our working dir. Might have been + # pre-created by some of the code above. + mkdir empty || true cd empty INSTALLDIR=$(python -c "import os, trio; print(os.path.dirname(trio.__file__))") From ec413bb4b42ce013e8ffcdf974982c936d2acbbe Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 02:57:04 -0700 Subject: [PATCH 0109/1498] Update docs to note FreeBSD support. --- README.rst | 10 +++++----- docs/source/index.rst | 9 ++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index b1dae85404..d85cd64173 100644 --- a/README.rst +++ b/README.rst @@ -103,11 +103,11 @@ older library versus Trio. **Cool, but will it work on my system?** Probably! As long as you have some kind of Python 3.6-or-better (CPython or the latest PyPy3 are -both fine), and are using Linux, macOS, or Windows, then Trio should -absolutely work. *BSD and illumos likely work too, but we don't have -testing infrastructure for them. And all of our dependencies are pure -Python, except for CFFI on Windows, and that has wheels available, so -installation should be easy. +both fine), and are using Linux, macOS, Windows, or FreeBSD, then Trio +should definitely work. Other environments might work too, but those +are the ones we test on. And all of our dependencies are pure Python, +except for CFFI on Windows, and that has wheels available, so +installation should be easy (no C compiler needed). **I tried it but it's not working.** Sorry to hear that! You can try asking for help in our `chat room diff --git a/docs/source/index.rst b/docs/source/index.rst index 610c6b07de..e3adddaee8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -43,9 +43,12 @@ chance to give feedback about any compatibility-breaking changes. Vital statistics: -* Supported environments: Linux, macOS, or Windows running some kind of Python - 3.6-or-better (either CPython or PyPy3 is fine). \*BSD and - illumos likely work too, but are untested. +* Supported environments: We test on + + - Python: 3.6+ (CPython and PyPy) + - Windows, macOS, Linux (glibc and musl), FreeBSD + + Other environments might also work; give it a try and see. * Install: ``python3 -m pip install -U trio`` (or on Windows, maybe ``py -3 -m pip install -U trio``). No compiler needed. From da681c254dc9958f38b1761e151551ced6e5839b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 02:58:49 -0700 Subject: [PATCH 0110/1498] Add newsfragment --- newsfragments/1118.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1118.feature.rst diff --git a/newsfragments/1118.feature.rst b/newsfragments/1118.feature.rst new file mode 100644 index 0000000000..b54208af8a --- /dev/null +++ b/newsfragments/1118.feature.rst @@ -0,0 +1 @@ +We've added FreeBSD to the list of platforms we support and test on. From e6490f81ebd2a6d7cd23f4e24112b8ddb2b3c59e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 03:07:08 -0700 Subject: [PATCH 0111/1498] Don't try to call os.uname unless it exists --- trio/_core/tests/tutil.py | 9 +++++++++ trio/tests/test_subprocess.py | 8 ++------ trio/tests/test_unix_pipes.py | 8 ++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index d5d22d525e..dac53b81fd 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -1,5 +1,6 @@ # Utilities for testing import socket as stdlib_socket +import os import pytest @@ -61,3 +62,11 @@ def check_sequence_matches(seq, template): got = set(seq[i:i + len(pattern)]) assert got == pattern i += len(got) + + +# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 +skip_if_fbsd_pipes_broken = pytest.mark.skipif( + hasattr(os, "uname") and os.uname().sysname == "FreeBSD" + and os.uname().release[:4] < "12.2", + reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350" +) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index b9d94d15d0..b9290f0401 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -9,7 +9,7 @@ _core, move_on_after, fail_after, sleep, sleep_forever, Process, open_process, run_process, TrioDeprecationWarning ) -from .._core.tests.tutil import slow +from .._core.tests.tutil import slow, skip_if_fbsd_pipes_broken from ..testing import wait_all_tasks_blocked posix = os.name == "posix" @@ -254,11 +254,7 @@ async def test_run_check(): assert result.returncode == 1 -# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 -@pytest.mark.skipif( - os.uname().sysname == "FreeBSD" and os.uname().release[:4] < "12.2", - reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350" -) +@skip_if_fbsd_pipes_broken async def test_run_with_broken_pipe(): result = await run_process( [sys.executable, "-c", "import sys; sys.stdin.close()"], diff --git a/trio/tests/test_unix_pipes.py b/trio/tests/test_unix_pipes.py index e9a07b8e65..5b85716b75 100644 --- a/trio/tests/test_unix_pipes.py +++ b/trio/tests/test_unix_pipes.py @@ -6,7 +6,7 @@ import pytest -from .._core.tests.tutil import gc_collect_harder +from .._core.tests.tutil import gc_collect_harder, skip_if_fbsd_pipes_broken from .. import _core, move_on_after from ..testing import wait_all_tasks_blocked, check_one_way_stream @@ -264,10 +264,6 @@ async def test_bizarro_OSError_from_receive(): os.close(dir_fd) -# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 -@pytest.mark.skipif( - os.uname().sysname == "FreeBSD" and os.uname().release[:4] < "12.2", - reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350" -) +@skip_if_fbsd_pipes_broken async def test_pipe_fully(): await check_one_way_stream(make_pipe, make_clogged_pipe) From bb0171dfe5b5a802016f4a724e2db1a36aaabadb Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 03:23:27 -0700 Subject: [PATCH 0112/1498] Apparently FreeBSD doesn't always raise ConnectionResetError here Judging from coverage. But I saw it locally, so, :shrug:, doesn't hurt anything to leave the catch in. --- trio/tests/test_highlevel_open_tcp_listeners.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index baf06c380e..d11c1c6e6d 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -76,13 +76,13 @@ async def measure_backlog(listener, limit): # seconds indicates a dropped SYN. # # Exception: on FreeBSD when the backlog is exhausted, connect - # raises ConnectionResetError. + # has been observed to sometimes raise ConnectionResetError. with trio.move_on_after(0.5) as cancel_scope: try: client_stream = await open_stream_to_socket_listener( listener ) - except ConnectionResetError: + except ConnectionResetError: # pragma: no cover break client_streams.append(client_stream) if cancel_scope.cancelled_caught: From 4405d3125c29299df76f1c596a38dd3216e7a058 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 03:25:37 -0700 Subject: [PATCH 0113/1498] Add some coverage pragmas --- trio/_core/_io_kqueue.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index 40851ca468..95339167e3 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -134,9 +134,10 @@ def abort(_): # the fact... oh well, you can't have everything.) # # FreeBSD reports this using EBADF. macOS uses ENOENT. - if exc.errno in (errno.EBADF, errno.ENOENT): + if exc.errno in (errno.EBADF, errno.ENOENT): # pragma: no branch pass - else: + else: # pragma: no cover + # As far as we know, this branch can't happen. raise return _core.Abort.SUCCEEDED From 0a7cfdf6639480a38c22a48a7eec2a6772a93dc3 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 03:39:30 -0700 Subject: [PATCH 0114/1498] Improved comment --- ci.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ci.sh b/ci.sh index 80d64cbd05..8bb3e2bec2 100755 --- a/ci.sh +++ b/ci.sh @@ -105,17 +105,18 @@ fi ### FreeBSD-in-Qemu virtual-machine inception, on Travis -# This is complex, because there aren't pre-made images available. So we hack -# up an install CD, run it to build an image, and use that. But that's slow -# and we don't want to do it every run, so we try to get Travis to cache the -# image for us. +# This is complex, because none of the pre-made images are set up to be +# controlled by an automatic process – making them do anything requires a +# human to look at the screen and type stuff. So we hack up an install CD, run +# it to build our own custom image, and use that. But that's slow and we don't +# want to do it every run, so we try to get Travis to cache the image for us. # # Additional subtlety: we actually re-use the same image in-place, and let # Travis re-cache it every time. The point of this is that it saves our system -# packages + pip cache, so we don't have to recreate them from scratch every -# time. (Especially useful since there are no FreeBSD wheels on PyPI, and we -# don't want to have to build packages like cryptography from source every -# time.) +# packages + pip cache, so we don't have to re-fetch them from scratch every +# time. (In particular, this means that the first time we install a given +# version of cryptography, we have to build it from source, but on subsequent +# runs we'll have a pre-built wheel sitting in our disk image.) # # You can run this locally for testing. But some things to watch out for: # From d52658b77ad6b006551f4e3a0dc2d6ffef8adb5f Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 03:41:24 -0700 Subject: [PATCH 0115/1498] yapf --- trio/_core/_io_kqueue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index 95339167e3..8f0f492c69 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -134,7 +134,9 @@ def abort(_): # the fact... oh well, you can't have everything.) # # FreeBSD reports this using EBADF. macOS uses ENOENT. - if exc.errno in (errno.EBADF, errno.ENOENT): # pragma: no branch + if exc.errno in ( + errno.EBADF, errno.ENOENT + ): # pragma: no branch pass else: # pragma: no cover # As far as we know, this branch can't happen. From 4ef2c33f7182daa978907a01e819d52c04269b22 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 11 May 2020 11:39:42 +0400 Subject: [PATCH 0116/1498] Test Fedora 32 VM to get Linux 5.6 --- .travis.yml | 4 ++-- ci.sh | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b75911edf..256580ffce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,8 +32,8 @@ jobs: # 'bionic' systems have nested KVM enabled. dist: bionic env: - - "JOB_NAME='Ubuntu 19.10, full VM'" - - "LINUX_VM_IMAGE=https://cloud-images.ubuntu.com/eoan/current/eoan-server-cloudimg-amd64.img" + - "JOB_NAME='Fedora 32, full VM'" + - "LINUX_VM_IMAGE=https://download.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-1.6.x86_64.qcow2" - python: 3.6.1 # earliest 3.6 version available on Travis - python: 3.6-dev diff --git a/ci.sh b/ci.sh index f6bf99dcc1..13c5b73f43 100755 --- a/ci.sh +++ b/ci.sh @@ -319,7 +319,7 @@ trap "poweroff" exit uname -a echo \$PWD id -cat /etc/lsb-release +cat /etc/fedora-release cat /proc/cpuinfo # Pass-through JOB_NAME + the env vars that codecov-bash looks at @@ -340,15 +340,18 @@ env | sort mkdir /host-files mount -t 9p -o trans=virtio,version=9p2000.L host-files /host-files -# Install and set up the system Python (assumes Debian/Ubuntu) -apt update -apt install -y python3-dev python3-virtualenv git build-essential curl -python3 -m virtualenv -p python3 /venv +# Set up the system Python (Fedora preinstalls Python 3) +python3 -m venv /venv # Uses unbound shell variable PS1, so have to allow that temporarily set +u source /venv/bin/activate set -u +# We put a tmpfs on the empty dir because coverage uses it for +# storage, and this makes it *massively* faster than if it's on NFS +mkdir /host-files/empty +mount -t tmpfs tmpfs /host-files/empty + # And then we re-invoke ourselves! cd /host-files ./ci.sh From b3ffa2129ab0d0757c9e3430d97085a38ecd5d5a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 11 May 2020 10:09:24 +0400 Subject: [PATCH 0117/1498] Use Ubuntu 20.04 in Travis CI --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 256580ffce..d87147f8e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,17 @@ os: linux language: python -dist: bionic +dist: focal jobs: include: # The pypy tests are slow, so we list them first - python: pypy3.6-7.2.0 + dist: bionic - language: generic env: PYPY_NIGHTLY_BRANCH=py3.6 # Qemu tests are also slow # FreeBSD: - language: generic - dist: bionic env: - "JOB_NAME='FreeBSD 12.1-RELEASE, full VM'" - "FREEBSD_INSTALLER_ISO_XZ=https://download.freebsd.org/ftp/releases/amd64/amd64/ISO-IMAGES/12.1/FreeBSD-12.1-RELEASE-amd64-disc1.iso.xz" @@ -28,14 +28,12 @@ jobs: # early warning of any issues that might happen in the next Ubuntu # LTS. - language: generic - # We use bionic for the host, b/c rumor says that Travis's - # 'bionic' systems have nested KVM enabled. - dist: bionic env: - "JOB_NAME='Fedora 32, full VM'" - "LINUX_VM_IMAGE=https://download.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-1.6.x86_64.qcow2" - python: 3.6.1 # earliest 3.6 version available on Travis + dist: bionic - python: 3.6-dev - python: 3.7-dev - python: 3.8-dev From 28c79770aa39668a09ebf2f332c769f77472d8a5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 16:48:14 -0700 Subject: [PATCH 0118/1498] Small correction to comment text --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 13c5b73f43..318420e1ea 100755 --- a/ci.sh +++ b/ci.sh @@ -245,7 +245,7 @@ source /venv/bin/activate set -u # And put a tmpfs on the empty dir as well, because coverage uses it for -# storage, and this makes it *massively* faster than if it's on NFS +# storage, and this is much faster than if coverage has to go through virtfs. mkdir empty mount -t tmpfs tmpfs ./empty From 8f2e2a06bde78b0563b1e9cff1c1367fbd4ef603 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 11 May 2020 20:01:20 -0700 Subject: [PATCH 0119/1498] Fix filesystem CI comments Pointed out by Quentin here: https://github.com/python-trio/trio/pull/1510#issuecomment-627069882 --- ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci.sh b/ci.sh index 318420e1ea..ee428da097 100755 --- a/ci.sh +++ b/ci.sh @@ -245,7 +245,7 @@ source /venv/bin/activate set -u # And put a tmpfs on the empty dir as well, because coverage uses it for -# storage, and this is much faster than if coverage has to go through virtfs. +# storage, and this makes it *massively* faster than if it's on NFS mkdir empty mount -t tmpfs tmpfs ./empty @@ -348,7 +348,7 @@ source /venv/bin/activate set -u # We put a tmpfs on the empty dir because coverage uses it for -# storage, and this makes it *massively* faster than if it's on NFS +# storage, and this is much faster than if coverage has to go through virtfs. mkdir /host-files/empty mount -t tmpfs tmpfs /host-files/empty From 8be6af56102c6dcfe1f8ae361e4e5568d1a88cd2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 12 May 2020 06:21:30 +0000 Subject: [PATCH 0120/1498] Bump flake8 from 3.7.9 to 3.8.1 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.7.9 to 3.8.1. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.7.9...3.8.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index ef1092850c..e59a652dc2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,8 +13,7 @@ cffi==1.14.0 # via cryptography coverage==5.1 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets -entrypoints==0.3 # via flake8 -flake8==3.7.9 # via -r test-requirements.in +flake8==3.8.1 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme immutables==0.12 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets @@ -33,9 +32,9 @@ pluggy==0.13.1 # via pytest prompt-toolkit==3.0.5 # via ipython ptyprocess==0.6.0 # via pexpect py==1.8.1 # via pytest -pycodestyle==2.5.0 # via flake8 +pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi -pyflakes==2.1.1 # via flake8 +pyflakes==2.2.0 # via flake8 pygments==2.6.1 # via ipython pylint==2.5.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in From 96cdeb256b60b13ec6c3f12aefa9e6a616aa5fcc Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Tue, 12 May 2020 09:10:16 -0300 Subject: [PATCH 0121/1498] Added helpful error message in the case the user passed incorrect type of function to both "from_thread_run" functions. --- trio/_threads.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index 811bc526a0..f9d8664bb8 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -5,6 +5,8 @@ import attr import outcome +import inspect + import trio from ._sync import CapacityLimiter @@ -364,7 +366,7 @@ def from_thread_run(afn, *args, trio_token=None): RuntimeError: if you try calling this from inside the Trio thread, which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer - one from context. + one from context. Also if ``afn`` is not an async function. **Locating a Trio Token**: There are two ways to specify which `trio.run` loop to reenter: @@ -377,6 +379,9 @@ def from_thread_run(afn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ + if not inspect.iscoroutinefunction(afn): + raise AttributeError("afn must be an asynchronous function") + def callback(q, afn, args): @disable_ki_protection async def unprotected_afn(): @@ -409,7 +414,7 @@ def from_thread_run_sync(fn, *args, trio_token=None): RuntimeError: if you try calling this from inside the Trio thread, which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer - one from context. + one from context. Also if ``fn`` is not a sync function. **Locating a Trio Token**: There are two ways to specify which `trio.run` loop to reenter: @@ -422,6 +427,9 @@ def from_thread_run_sync(fn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ + if not (inspect.isfunction(fn) and not inspect.iscoroutinefunction(fn)): + raise AttributeError("fn must be a synchronous function") + def callback(q, fn, args): @disable_ki_protection def unprotected_fn(): From 9599348f9b668feafd0a05db012e8806cda8389f Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 12 May 2020 12:48:23 -0700 Subject: [PATCH 0122/1498] make trio.Process.returncode automatically update itself as needed --- newsfragments/1315.feature.rst | 4 +++ trio/_subprocess.py | 47 +++++++++++++++++++--------------- trio/tests/test_subprocess.py | 11 ++++++++ 3 files changed, 42 insertions(+), 20 deletions(-) create mode 100644 newsfragments/1315.feature.rst diff --git a/newsfragments/1315.feature.rst b/newsfragments/1315.feature.rst new file mode 100644 index 0000000000..824a793e74 --- /dev/null +++ b/newsfragments/1315.feature.rst @@ -0,0 +1,4 @@ +The `trio.Process.returncode` attribute is now automatically updated +as needed, instead of only when you call `~trio.Process.poll` or +`~trio.Process.wait`. Also, ``repr(process_object)`` now always +contains up-to-date information about the process status. diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 6983c839c7..ab6ab76a7e 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -133,19 +133,20 @@ def __init__(self, popen, stdin, stdout, stderr): self.pid = self._proc.pid def __repr__(self): - if self.returncode is None: + returncode = self.returncode + if returncode is None: status = "running with PID {}".format(self.pid) else: - if self.returncode < 0: - status = "exited with signal {}".format(-self.returncode) + if returncode < 0: + status = "exited with signal {}".format(-returncode) else: - status = "exited with status {}".format(self.returncode) + status = "exited with status {}".format(returncode) return "".format(self.args, status) @property def returncode(self): - """The exit status of the process (an integer), or ``None`` if it is - not yet known to have exited. + """The exit status of the process (an integer), or ``None`` if it's + still running. By convention, a return code of zero indicates success. On UNIX, negative values indicate termination due to a signal, @@ -153,10 +154,16 @@ def returncode(self): Windows, a process that exits due to a call to :meth:`Process.terminate` will have an exit status of 1. - Accessing this attribute does not check for termination; - use :meth:`poll` or :meth:`wait` for that. + Unlike the standard library `subprocess.Popen.returncode`, you don't + have to call `poll` or `wait` to update this attribute; it's + automatically updated as needed, and will always give you the latest + information. + """ - return self._proc.returncode + result = self._proc.poll() + if result is not None: + self._close_pidfd() + return result async def aclose(self): """Close any pipes we have to the process (both input and output) @@ -175,7 +182,7 @@ async def aclose(self): try: await self.wait() finally: - if self.returncode is None: + if self._proc.returncode is None: self.kill() with trio.CancelScope(shield=True): await self.wait() @@ -205,20 +212,20 @@ async def wait(self): # actually block for a tiny fraction of a second. self._proc.wait() self._close_pidfd() - assert self.returncode is not None - return self.returncode + assert self._proc.returncode is not None + return self._proc.returncode def poll(self): - """Check if the process has exited yet. + """Returns the exit status of the process (an integer), or ``None`` if + it's still running. + + Note that on Trio (unlike the standard library `subprocess.Popen`), + ``process.poll()`` and ``process.returncode`` always give the same + result. See `returncode` for more details. This method is only + included to make it easier to port code from `subprocess`. - Returns: - The exit status of the process, or ``None`` if it is still - running; see :attr:`returncode`. """ - result = self._proc.poll() - if result is not None: - self._close_pidfd() - return result + return self.returncode def send_signal(self, sig): """Send signal ``sig`` to the process. diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index b9290f0401..ebc838b0ed 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -47,6 +47,7 @@ async def test_basic(): assert repr(proc) == repr_template.format( "running with PID {}".format(proc.pid) ) + assert proc._pidfd is None assert proc.returncode == 0 assert repr(proc) == repr_template.format("exited with status 0") @@ -58,6 +59,16 @@ async def test_basic(): ) +async def test_auto_update_returncode(): + p = await open_process(SLEEP(9999)) + assert p.returncode is None + p.kill() + p._proc.wait() + assert p.returncode is not None + assert p._pidfd is None + assert p.returncode is not None + + async def test_multi_wait(): async with await open_process(SLEEP(10)) as proc: # Check that wait (including multi-wait) tolerates being cancelled From 5b5f7876f28fd289121937628525f3967916fef1 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 12 May 2020 18:07:20 -0700 Subject: [PATCH 0123/1498] Delete a bit of stray code+comments that are no longer meaningful --- ci.sh | 7 +------ trio/_util.py | 6 ------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/ci.sh b/ci.sh index ee428da097..aed1974ddd 100755 --- a/ci.sh +++ b/ci.sh @@ -6,12 +6,7 @@ set -ex -o pipefail env | sort if [ "$JOB_NAME" = "" ]; then - if [ "$SYSTEM_JOBIDENTIFIER" != "" ]; then - # azure pipelines - JOB_NAME="$SYSTEM_JOBDISPLAYNAME" - else - JOB_NAME="${TRAVIS_OS_NAME}-${TRAVIS_PYTHON_VERSION:-unknown}" - fi + JOB_NAME="${TRAVIS_OS_NAME}-${TRAVIS_PYTHON_VERSION:-unknown}" fi # Curl's built-in retry system is not very robust; it gives up on lots of diff --git a/trio/_util.py b/trio/_util.py index b331c8b48f..4d9b46ec7b 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -11,12 +11,6 @@ from ._deprecate import warn_deprecated -# There's a dependency loop here... _core is allowed to use this file (in fact -# it's the *only* file in the main trio/ package it's allowed to use), but -# ConflictDetector needs checkpoint so it also has to import -# _core. Possibly we should split this file into two: one for true generic -# low-level utility code, and one for higher level helpers? - import trio # Equivalent to the C function raise(), which Python doesn't wrap From bdbf143f0a321e6b4c077d0fd1a588eb2adba637 Mon Sep 17 00:00:00 2001 From: kAldown Date: Wed, 13 May 2020 04:33:41 +0300 Subject: [PATCH 0124/1498] [#1500]: Remove unused __all__ variables from Trio modules --- .gitignore | 1 + trio/_core/_entry_queue.py | 2 -- trio/_core/_ki.py | 6 ------ trio/_core/_local.py | 2 -- trio/_core/_multierror.py | 2 -- trio/_core/_parking_lot.py | 2 -- trio/_core/_unbounded_queue.py | 2 -- trio/_file_io.py | 2 -- trio/_highlevel_open_tcp_listeners.py | 2 -- trio/_highlevel_open_tcp_stream.py | 2 -- trio/_highlevel_open_unix_stream.py | 2 -- trio/_highlevel_serve_listeners.py | 2 -- trio/_highlevel_socket.py | 2 -- trio/_highlevel_ssl_helpers.py | 5 ----- trio/_path.py | 2 -- trio/_signals.py | 2 -- trio/_socket.py | 2 -- trio/_sync.py | 9 --------- trio/_timeouts.py | 11 ----------- trio/_wait_for_object.py | 2 -- trio/testing/_check_streams.py | 6 ------ trio/testing/_checkpoints.py | 2 -- trio/testing/_memory_streams.py | 10 ---------- trio/testing/_mock_clock.py | 2 -- trio/testing/_network.py | 4 +--- trio/testing/_sequencer.py | 2 -- trio/testing/_trio_test.py | 2 -- 27 files changed, 2 insertions(+), 88 deletions(-) diff --git a/.gitignore b/.gitignore index a50c10b8a3..057e28568e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ pip-log.txt htmlcov/ .tox/ .venv/ +pyvenv.cfg .coverage .coverage.* .cache diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index 869f616b5e..0d29e393b0 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -7,8 +7,6 @@ from .._util import NoPublicConstructor from ._wakeup_socketpair import WakeupSocketpair -__all__ = ["TrioToken"] - @attr.s(slots=True) class EntryQueue: diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index a3f64c5dca..9406ebb7e1 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -12,12 +12,6 @@ from typing import Any, TypeVar, Callable F = TypeVar('F', bound=Callable[..., Any]) -__all__ = [ - "enable_ki_protection", - "disable_ki_protection", - "currently_ki_protected", -] - # In ordinary single-threaded Python code, when you hit control-C, it raises # an exception and automatically does all the regular unwinding stuff. # diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 589f5f5a90..aaf5f6d2f7 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -3,8 +3,6 @@ from .._util import SubclassingDeprecatedIn_v0_15_0 -__all__ = ["RunVar"] - class _RunVarToken: _no_value = object() diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index a29f19f60d..00d44e30ed 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -5,8 +5,6 @@ import attr -__all__ = ["MultiError"] - # python traceback.TracebackException < 3.6.4 does not support unhashable exceptions # see https://github.com/python/cpython/pull/4014 for details if sys.version_info < (3, 6, 4): diff --git a/trio/_core/_parking_lot.py b/trio/_core/_parking_lot.py index 0bfc12230b..c55d298133 100644 --- a/trio/_core/_parking_lot.py +++ b/trio/_core/_parking_lot.py @@ -77,8 +77,6 @@ from .. import _core from .._util import SubclassingDeprecatedIn_v0_15_0 -__all__ = ["ParkingLot"] - _counter = count() diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index 2cb82d1fce..efa0a8d1b3 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -4,8 +4,6 @@ from .._deprecate import deprecated from .._util import SubclassingDeprecatedIn_v0_15_0 -__all__ = ["UnboundedQueue"] - @attr.s(frozen=True) class _UnboundedQueueStats: diff --git a/trio/_file_io.py b/trio/_file_io.py index 159e08be59..1d3508875e 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -6,8 +6,6 @@ import trio -__all__ = ['open_file', 'wrap_file'] - # This list is also in the docs, make sure to keep them in sync _FILE_SYNC_ATTRS = { 'closed', diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index 2625238803..6ac44db43b 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -5,8 +5,6 @@ import trio from . import socket as tsocket -__all__ = ["open_tcp_listeners", "serve_tcp"] - # Default backlog size: # diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 71418b12d3..11a1e1846f 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -3,8 +3,6 @@ import trio from trio.socket import getaddrinfo, SOCK_STREAM, socket -__all__ = ["open_tcp_stream"] - # Implementation of RFC 6555 "Happy eyeballs" # https://tools.ietf.org/html/rfc6555 # diff --git a/trio/_highlevel_open_unix_stream.py b/trio/_highlevel_open_unix_stream.py index 08eef0c135..87462c1232 100644 --- a/trio/_highlevel_open_unix_stream.py +++ b/trio/_highlevel_open_unix_stream.py @@ -10,8 +10,6 @@ except ImportError: has_unix = False -__all__ = ["open_unix_socket"] - @contextmanager def close_on_error(obj): diff --git a/trio/_highlevel_serve_listeners.py b/trio/_highlevel_serve_listeners.py index 958624a6f4..0a46be780a 100644 --- a/trio/_highlevel_serve_listeners.py +++ b/trio/_highlevel_serve_listeners.py @@ -4,8 +4,6 @@ import trio -__all__ = ["serve_listeners"] - # Errors that accept(2) can return, and which indicate that the system is # overloaded ACCEPT_CAPACITY_ERRNOS = { diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index 133a9981a1..abcf951028 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -8,8 +8,6 @@ from ._util import ConflictDetector, SubclassingDeprecatedIn_v0_15_0 from .abc import HalfCloseableStream, Listener -__all__ = ["SocketStream", "SocketListener"] - # XX TODO: this number was picked arbitrarily. We should do experiments to # tune it. (Or make it dynamic -- one idea is to start small and increase it # if we observe single reads filling up the whole buffer, at least within some diff --git a/trio/_highlevel_ssl_helpers.py b/trio/_highlevel_ssl_helpers.py index 0df1665e5c..fc604d9286 100644 --- a/trio/_highlevel_ssl_helpers.py +++ b/trio/_highlevel_ssl_helpers.py @@ -3,11 +3,6 @@ from ._highlevel_open_tcp_stream import DEFAULT_DELAY -__all__ = [ - "open_ssl_over_tcp_stream", "open_ssl_over_tcp_listeners", - "serve_ssl_over_tcp" -] - # It might have been nice to take a ssl_protocols= argument here to set up # NPN/ALPN, but to do this we have to mutate the context object, which is OK diff --git a/trio/_path.py b/trio/_path.py index 45fe0e4341..75f1fab615 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -6,8 +6,6 @@ import trio from trio._util import async_wraps, SubclassingDeprecatedIn_v0_15_0 -__all__ = ['Path'] - # re-wrap return value from methods that return new instances of pathlib.Path def rewrap_path(value): diff --git a/trio/_signals.py b/trio/_signals.py index 6392eebd0f..aca23dda63 100644 --- a/trio/_signals.py +++ b/trio/_signals.py @@ -5,8 +5,6 @@ import trio from ._util import signal_raise, is_main_thread, ConflictDetector -__all__ = ["open_signal_receiver"] - # Discussion of signal handling strategies: # # - On Windows signals barely exist. There are no options; signal handlers are diff --git a/trio/_socket.py b/trio/_socket.py index 843b540ad9..5518335b44 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -324,8 +324,6 @@ def _sniff_sockopts_for_fileno(family, type, proto, fileno): ) -# Note that this is *not* in __all__. -# # This function will modify the given socket to match the behavior in python # 3.7. This will become unecessary and can be removed when support for versions # older than 3.7 is dropped. diff --git a/trio/_sync.py b/trio/_sync.py index d5a5d04372..068a5684f4 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -9,15 +9,6 @@ from ._deprecate import deprecated from ._util import SubclassingDeprecatedIn_v0_15_0 -__all__ = [ - "Event", - "CapacityLimiter", - "Semaphore", - "Lock", - "StrictFIFOLock", - "Condition", -] - @attr.s(repr=False, eq=False, hash=False) class Event(metaclass=SubclassingDeprecatedIn_v0_15_0): diff --git a/trio/_timeouts.py b/trio/_timeouts.py index eabc6befbb..9bfefe4b03 100644 --- a/trio/_timeouts.py +++ b/trio/_timeouts.py @@ -2,17 +2,6 @@ import trio -__all__ = [ - "move_on_at", - "move_on_after", - "sleep_forever", - "sleep_until", - "sleep", - "fail_at", - "fail_after", - "TooSlowError", -] - def move_on_at(deadline): """Use as a context manager to create a cancel scope with the given diff --git a/trio/_wait_for_object.py b/trio/_wait_for_object.py index dfbf47d8a3..1b209ddb26 100644 --- a/trio/_wait_for_object.py +++ b/trio/_wait_for_object.py @@ -3,8 +3,6 @@ import trio from ._core._windows_cffi import ffi, kernel32, ErrorCodes, raise_winerror, _handle -__all__ = ["WaitForSingleObject"] - async def WaitForSingleObject(obj): """Async and cancellable variant of WaitForSingleObject. Windows only. diff --git a/trio/testing/_check_streams.py b/trio/testing/_check_streams.py index 2216692df4..48880c13c4 100644 --- a/trio/testing/_check_streams.py +++ b/trio/testing/_check_streams.py @@ -8,12 +8,6 @@ from .._abc import SendStream, ReceiveStream, Stream, HalfCloseableStream from ._checkpoints import assert_checkpoints -__all__ = [ - "check_one_way_stream", - "check_two_way_stream", - "check_half_closeable_stream", -] - class _ForceCloseBoth: def __init__(self, both): diff --git a/trio/testing/_checkpoints.py b/trio/testing/_checkpoints.py index 716260893b..27fd4d187d 100644 --- a/trio/testing/_checkpoints.py +++ b/trio/testing/_checkpoints.py @@ -2,8 +2,6 @@ from .. import _core -__all__ = ["assert_checkpoints", "assert_no_checkpoints"] - @contextmanager def _assert_yields_or_not(expected): diff --git a/trio/testing/_memory_streams.py b/trio/testing/_memory_streams.py index 184626d58b..62d63b73d9 100644 --- a/trio/testing/_memory_streams.py +++ b/trio/testing/_memory_streams.py @@ -5,16 +5,6 @@ from .. import _util from ..abc import SendStream, ReceiveStream -__all__ = [ - "MemorySendStream", - "MemoryReceiveStream", - "memory_stream_pump", - "memory_stream_one_way_pair", - "memory_stream_pair", - "lockstep_stream_one_way_pair", - "lockstep_stream_pair", -] - ################################################################ # In-memory streams - Unbounded buffer version ################################################################ diff --git a/trio/testing/_mock_clock.py b/trio/testing/_mock_clock.py index 2e0667ab94..997e701f39 100644 --- a/trio/testing/_mock_clock.py +++ b/trio/testing/_mock_clock.py @@ -5,8 +5,6 @@ from .._abc import Clock from .._util import SubclassingDeprecatedIn_v0_15_0 -__all__ = ["MockClock"] - ################################################################ # The glorious MockClock ################################################################ diff --git a/trio/testing/_network.py b/trio/testing/_network.py index e3844847d5..615ce2effb 100644 --- a/trio/testing/_network.py +++ b/trio/testing/_network.py @@ -1,7 +1,5 @@ from .. import socket as tsocket -from .._highlevel_socket import SocketListener, SocketStream - -__all__ = ["open_stream_to_socket_listener"] +from .._highlevel_socket import SocketStream async def open_stream_to_socket_listener(socket_listener): diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index 1ff190e985..21fc492dff 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -10,8 +10,6 @@ if False: from typing import DefaultDict, Set -__all__ = ["Sequencer"] - @attr.s(eq=False, hash=False) class Sequencer(metaclass=_util.SubclassingDeprecatedIn_v0_15_0): diff --git a/trio/testing/_trio_test.py b/trio/testing/_trio_test.py index 9215084955..87caa5881a 100644 --- a/trio/testing/_trio_test.py +++ b/trio/testing/_trio_test.py @@ -3,8 +3,6 @@ from .. import _core from ..abc import Clock, Instrument -__all__ = ["trio_test"] - # Use: # From 015b220ad0e7e5f766a04b52705709d611ffb5b0 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Tue, 12 May 2020 23:54:58 -0300 Subject: [PATCH 0125/1498] Extracted "coroutine or TypeError" logic from Runner.spawn_impl, moved it to trio._util. Used it in trio.from_thread_run. --- trio/_core/_run.py | 85 ++------------------------------------------ trio/_threads.py | 12 +++---- trio/_util.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 90 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 5904e682fd..26da1ed9f0 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -17,7 +17,6 @@ from sniffio import current_async_library_cvar import attr -from async_generator import isasyncgen from sortedcontainers import SortedDict from outcome import Error, Value, capture @@ -36,7 +35,7 @@ ) from .. import _core from .._deprecate import deprecated -from .._util import Final, NoPublicConstructor +from .._util import Final, NoPublicConstructor, coroutine_or_error _NO_SEND = object() @@ -1243,86 +1242,8 @@ def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): if nursery is None: assert self.init_task is None - ###### - # Call the function and get the coroutine object, while giving helpful - # errors for common mistakes. - ###### - - def _return_value_looks_like_wrong_library(value): - # Returned by legacy @asyncio.coroutine functions, which includes - # a surprising proportion of asyncio builtins. - if isinstance(value, collections.abc.Generator): - return True - # The protocol for detecting an asyncio Future-like object - if getattr(value, "_asyncio_future_blocking", None) is not None: - return True - # This janky check catches tornado Futures and twisted Deferreds. - # By the time we're calling this function, we already know - # something has gone wrong, so a heuristic is pretty safe. - if value.__class__.__name__ in ("Future", "Deferred"): - return True - return False - - try: - coro = async_fn(*args) - except TypeError: - # Give good error for: nursery.start_soon(trio.sleep(1)) - if isinstance(async_fn, collections.abc.Coroutine): - raise TypeError( - "Trio was expecting an async function, but instead it got " - "a coroutine object {async_fn!r}\n" - "\n" - "Probably you did something like:\n" - "\n" - " trio.run({async_fn.__name__}(...)) # incorrect!\n" - " nursery.start_soon({async_fn.__name__}(...)) # incorrect!\n" - "\n" - "Instead, you want (notice the parentheses!):\n" - "\n" - " trio.run({async_fn.__name__}, ...) # correct!\n" - " nursery.start_soon({async_fn.__name__}, ...) # correct!" - .format(async_fn=async_fn) - ) from None - - # Give good error for: nursery.start_soon(future) - if _return_value_looks_like_wrong_library(async_fn): - raise TypeError( - "Trio was expecting an async function, but instead it got " - "{!r} – are you trying to use a library written for " - "asyncio/twisted/tornado or similar? That won't work " - "without some sort of compatibility shim." - .format(async_fn) - ) from None - - raise - - # We can't check iscoroutinefunction(async_fn), because that will fail - # for things like functools.partial objects wrapping an async - # function. So we have to just call it and then check whether the - # return value is a coroutine object. - if not isinstance(coro, collections.abc.Coroutine): - # Give good error for: nursery.start_soon(func_returning_future) - if _return_value_looks_like_wrong_library(coro): - raise TypeError( - "start_soon got unexpected {!r} – are you trying to use a " - "library written for asyncio/twisted/tornado or similar? " - "That won't work without some sort of compatibility shim." - .format(coro) - ) - - if isasyncgen(coro): - raise TypeError( - "start_soon expected an async function but got an async " - "generator {!r}".format(coro) - ) - - # Give good error for: nursery.start_soon(some_sync_fn) - raise TypeError( - "Trio expected an async function, but {!r} appears to be " - "synchronous".format( - getattr(async_fn, "__qualname__", async_fn) - ) - ) + # Check if async_fn is callable coroutine + coro = coroutine_or_error(async_fn, args) ###### # Set up the Task object diff --git a/trio/_threads.py b/trio/_threads.py index f9d8664bb8..5c2041776c 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -5,12 +5,11 @@ import attr import outcome -import inspect - import trio from ._sync import CapacityLimiter from ._core import enable_ki_protection, disable_ki_protection, RunVar, TrioToken +from ._util import coroutine_or_error # Global due to Threading API, thread local storage for trio token TOKEN_LOCAL = threading.local() @@ -366,7 +365,7 @@ def from_thread_run(afn, *args, trio_token=None): RuntimeError: if you try calling this from inside the Trio thread, which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer - one from context. Also if ``afn`` is not an async function. + one from context. **Locating a Trio Token**: There are two ways to specify which `trio.run` loop to reenter: @@ -379,13 +378,12 @@ def from_thread_run(afn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ - if not inspect.iscoroutinefunction(afn): - raise AttributeError("afn must be an asynchronous function") def callback(q, afn, args): @disable_ki_protection async def unprotected_afn(): - return await afn(*args) + coro = coroutine_or_error(afn, args) + return await coro async def await_in_trio_thread_task(): q.put_nowait(await outcome.acapture(unprotected_afn)) @@ -427,8 +425,6 @@ def from_thread_run_sync(fn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ - if not (inspect.isfunction(fn) and not inspect.iscoroutinefunction(fn)): - raise AttributeError("fn must be a synchronous function") def callback(q, fn, args): @disable_ki_protection diff --git a/trio/_util.py b/trio/_util.py index b331c8b48f..0ba286418f 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -8,6 +8,9 @@ from functools import wraps, update_wrapper import typing as t import threading +import collections + +from async_generator import isasyncgen from ._deprecate import warn_deprecated @@ -85,6 +88,91 @@ def is_main_thread(): return False +###### +# Call the function and get the coroutine object, while giving helpful +# errors for common mistakes. Returns coroutine object. +###### +def coroutine_or_error(async_fn, args): + + def _return_value_looks_like_wrong_library(value): + # Returned by legacy @asyncio.coroutine functions, which includes + # a surprising proportion of asyncio builtins. + if isinstance(value, collections.abc.Generator): + return True + # The protocol for detecting an asyncio Future-like object + if getattr(value, "_asyncio_future_blocking", None) is not None: + return True + # This janky check catches tornado Futures and twisted Deferreds. + # By the time we're calling this function, we already know + # something has gone wrong, so a heuristic is pretty safe. + if value.__class__.__name__ in ("Future", "Deferred"): + return True + return False + + try: + coro = async_fn(*args) + except TypeError: + # Give good error for: nursery.start_soon(trio.sleep(1)) + if isinstance(async_fn, collections.abc.Coroutine): + raise TypeError( + "Trio was expecting an async function, but instead it got " + "a coroutine object {async_fn!r}\n" + "\n" + "Probably you did something like:\n" + "\n" + " trio.run({async_fn.__name__}(...)) # incorrect!\n" + " nursery.start_soon({async_fn.__name__}(...)) # incorrect!\n" + "\n" + "Instead, you want (notice the parentheses!):\n" + "\n" + " trio.run({async_fn.__name__}, ...) # correct!\n" + " nursery.start_soon({async_fn.__name__}, ...) # correct!" + .format(async_fn=async_fn) + ) from None + + # Give good error for: nursery.start_soon(future) + if _return_value_looks_like_wrong_library(async_fn): + raise TypeError( + "Trio was expecting an async function, but instead it got " + "{!r} – are you trying to use a library written for " + "asyncio/twisted/tornado or similar? That won't work " + "without some sort of compatibility shim." + .format(async_fn) + ) from None + + raise + + # We can't check iscoroutinefunction(async_fn), because that will fail + # for things like functools.partial objects wrapping an async + # function. So we have to just call it and then check whether the + # return value is a coroutine object. + if not isinstance(coro, collections.abc.Coroutine): + # Give good error for: nursery.start_soon(func_returning_future) + if _return_value_looks_like_wrong_library(coro): + raise TypeError( + "start_soon got unexpected {!r} – are you trying to use a " + "library written for asyncio/twisted/tornado or similar? " + "That won't work without some sort of compatibility shim." + .format(coro) + ) + + if isasyncgen(coro): + raise TypeError( + "start_soon expected an async function but got an async " + "generator {!r}".format(coro) + ) + + # Give good error for: nursery.start_soon(some_sync_fn) + raise TypeError( + "Trio expected an async function, but {!r} appears to be " + "synchronous".format( + getattr(async_fn, "__qualname__", async_fn) + ) + ) + + return coro + + class ConflictDetector: """Detect when two tasks are about to perform operations that would conflict. From 126f7f4e93117f03b1d8db0d41bebb3453c6b009 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 13 May 2020 08:37:42 -0300 Subject: [PATCH 0126/1498] Added sync check --- trio/_threads.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/trio/_threads.py b/trio/_threads.py index 5c2041776c..67d879f495 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -3,6 +3,7 @@ from itertools import count import attr +import inspect import outcome import trio @@ -407,12 +408,13 @@ def from_thread_run_sync(fn, *args, trio_token=None): RunFinishedError: if the corresponding call to `trio.run` has already completed. Cancelled: if the corresponding call to `trio.run` completes - while ``afn(*args)`` is running, then ``afn`` is likely to raise + while ``fn(*args)`` is running, then ``fn`` is likely to raise :exc:`trio.Cancelled`, and this will propagate out into RuntimeError: if you try calling this from inside the Trio thread, which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer one from context. Also if ``fn`` is not a sync function. + TypeError: if ``fn`` is not callable or is an async function **Locating a Trio Token**: There are two ways to specify which `trio.run` loop to reenter: @@ -426,6 +428,14 @@ def from_thread_run_sync(fn, *args, trio_token=None): to enter Trio. """ + if not callable(fn) or inspect.iscoroutinefunction(fn): + raise TypeError( + "Trio expected a sync function, but {!r} appears to not be " + "callable or asynchronous".format( + getattr(fn, "__qualname__", fn) + ) + ) + def callback(q, fn, args): @disable_ki_protection def unprotected_fn(): From 417ba5b93782d4fa56b97f5ed7531247ba7d1464 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 13 May 2020 08:42:29 -0300 Subject: [PATCH 0127/1498] Ran code formatter and added comments. --- trio/_threads.py | 8 +++----- trio/_util.py | 8 ++------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index 67d879f495..b93dc3b5d8 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -367,6 +367,7 @@ def from_thread_run(afn, *args, trio_token=None): which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer one from context. + TypeError: if ``afn`` is not an asynchronous function. **Locating a Trio Token**: There are two ways to specify which `trio.run` loop to reenter: @@ -379,7 +380,6 @@ def from_thread_run(afn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ - def callback(q, afn, args): @disable_ki_protection async def unprotected_afn(): @@ -414,7 +414,7 @@ def from_thread_run_sync(fn, *args, trio_token=None): which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer one from context. Also if ``fn`` is not a sync function. - TypeError: if ``fn`` is not callable or is an async function + TypeError: if ``fn`` is not callable or is an async function. **Locating a Trio Token**: There are two ways to specify which `trio.run` loop to reenter: @@ -431,9 +431,7 @@ def from_thread_run_sync(fn, *args, trio_token=None): if not callable(fn) or inspect.iscoroutinefunction(fn): raise TypeError( "Trio expected a sync function, but {!r} appears to not be " - "callable or asynchronous".format( - getattr(fn, "__qualname__", fn) - ) + "callable or asynchronous".format(getattr(fn, "__qualname__", fn)) ) def callback(q, fn, args): diff --git a/trio/_util.py b/trio/_util.py index 0ba286418f..f5733ad85e 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -93,7 +93,6 @@ def is_main_thread(): # errors for common mistakes. Returns coroutine object. ###### def coroutine_or_error(async_fn, args): - def _return_value_looks_like_wrong_library(value): # Returned by legacy @asyncio.coroutine functions, which includes # a surprising proportion of asyncio builtins. @@ -136,8 +135,7 @@ def _return_value_looks_like_wrong_library(value): "Trio was expecting an async function, but instead it got " "{!r} – are you trying to use a library written for " "asyncio/twisted/tornado or similar? That won't work " - "without some sort of compatibility shim." - .format(async_fn) + "without some sort of compatibility shim.".format(async_fn) ) from None raise @@ -165,9 +163,7 @@ def _return_value_looks_like_wrong_library(value): # Give good error for: nursery.start_soon(some_sync_fn) raise TypeError( "Trio expected an async function, but {!r} appears to be " - "synchronous".format( - getattr(async_fn, "__qualname__", async_fn) - ) + "synchronous".format(getattr(async_fn, "__qualname__", async_fn)) ) return coro From dc06cad0ad2d7ae45f388713530af7f6cd048836 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 13 May 2020 16:17:44 -0300 Subject: [PATCH 0128/1498] Added coroutine_or_error test. --- trio/tests/test_util.py | 67 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index d57b1997df..8e93e7c34d 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -1,12 +1,13 @@ import signal - +import warnings import pytest import trio from .. import _core from .._util import ( - signal_raise, ConflictDetector, is_main_thread, generic_function, Final, - NoPublicConstructor, SubclassingDeprecatedIn_v0_15_0 + signal_raise, ConflictDetector, is_main_thread, coroutine_or_error, + generic_function, Final, NoPublicConstructor, + SubclassingDeprecatedIn_v0_15_0 ) from ..testing import wait_all_tasks_blocked @@ -82,6 +83,66 @@ def not_main_thread(): await trio.to_thread.run_sync(not_main_thread) +async def test_coroutine_or_error(): + + # error for: nursery.start_soon(trio.sleep(1)) + warnings.filterwarnings("error") + + async def test_afunc(): + pass + + with pytest.raises(TypeError): + try: + coroutine_or_error(test_afunc(), []) + except RuntimeWarning: + pass + + # error for: nursery.start_soon(future) + + # legacy @asyncio.coroutine functions + def test_generator(): + yield None + + with pytest.raises(TypeError): + coroutine_or_error(test_generator, []) + + # asyncio Future-like object + class AsycioFutureLike: + def __init__(self): + self._asyncio_future_blocking = "im a value!" + + with pytest.raises(TypeError): + coroutine_or_error(AsycioFutureLike(), []) + + # tornado Futures + class Future: + pass + + with pytest.raises(TypeError): + coroutine_or_error(Future(), []) + + # twisted Deferreds + class Deferreds: + pass + + with pytest.raises(TypeError): + coroutine_or_error(Deferreds(), []) + + # async generator + async def test_agenerator(): + yield None + + with pytest.raises(TypeError): + coroutine_or_error(test_agenerator, []) + + # synchronous function + def test_fn(): + pass + + with pytest.raises(TypeError): + coroutine_or_error(test_fn, []) + + def test_generic_function(): @generic_function def test_func(arg): From dc1e5b8f7220097f1d5f2f87346d7e9796075b32 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 13 May 2020 16:46:41 -0300 Subject: [PATCH 0129/1498] Added TypeError tests to from_thread_run and from_thread_run_sync --- trio/tests/test_threads.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 29d44adc4a..e937cb66e1 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -471,6 +471,13 @@ def thread_fn(): trio_time = await to_thread_run_sync(thread_fn) assert isinstance(trio_time, float) + # Test correct error when passed async function + async def async_fn(): + pass + + with pytest.raises(TypeError): + from_thread_run_sync(async_fn) + async def test_trio_from_thread_run(): # Test that to_thread_run_sync correctly "hands off" the trio token to @@ -488,6 +495,16 @@ def thread_fn(): await to_thread_run_sync(thread_fn) assert record == ["in thread", "back in trio"] + # Test correct error when passed sync function + def sync_fn(): + pass + + def thread_fn(): + from_thread_run(sync_fn) + + with pytest.raises(TypeError): + await to_thread_run_sync(thread_fn) + async def test_trio_from_thread_token(): # Test that to_thread_run_sync and spawned trio.from_thread.run_sync() From 5e4145decca7edaf29ce344ecfd7bbd6635413d4 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 13 May 2020 17:39:47 -0300 Subject: [PATCH 0130/1498] Moved comprehensive test of callable from test_run.py to test_util.py --- trio/_core/tests/test_run.py | 68 ------------------------ trio/_util.py | 2 +- trio/tests/test_util.py | 100 +++++++++++++++++++++-------------- 3 files changed, 60 insertions(+), 110 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index e705af5c22..880b89369e 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1696,74 +1696,6 @@ async def test_current_effective_deadline(mock_clock): assert _core.current_effective_deadline() == inf -# @coroutine is deprecated since python 3.8, which is fine with us. -@pytest.mark.filterwarnings("ignore:.*@coroutine.*:DeprecationWarning") -def test_nice_error_on_bad_calls_to_run_or_spawn(): - def bad_call_run(*args): - _core.run(*args) - - def bad_call_spawn(*args): - async def main(): - async with _core.open_nursery() as nursery: - nursery.start_soon(*args) - - _core.run(main) - - class Deferred: - "Just kidding" - - with ignore_coroutine_never_awaited_warnings(): - for bad_call in bad_call_run, bad_call_spawn: - - async def f(): # pragma: no cover - pass - - with pytest.raises(TypeError) as excinfo: - bad_call(f()) - assert "expecting an async function" in str(excinfo.value) - - import asyncio - - @asyncio.coroutine - def generator_based_coro(): # pragma: no cover - yield from asyncio.sleep(1) - - with pytest.raises(TypeError) as excinfo: - bad_call(generator_based_coro()) - assert "asyncio" in str(excinfo.value) - - with pytest.raises(TypeError) as excinfo: - bad_call(asyncio.Future()) - assert "asyncio" in str(excinfo.value) - - with pytest.raises(TypeError) as excinfo: - bad_call(lambda: asyncio.Future()) - assert "asyncio" in str(excinfo.value) - - with pytest.raises(TypeError) as excinfo: - bad_call(Deferred()) - assert "twisted" in str(excinfo.value) - - with pytest.raises(TypeError) as excinfo: - bad_call(lambda: Deferred()) - assert "twisted" in str(excinfo.value) - - with pytest.raises(TypeError) as excinfo: - bad_call(len, [1, 2, 3]) - assert "appears to be synchronous" in str(excinfo.value) - - async def async_gen(arg): # pragma: no cover - yield - - with pytest.raises(TypeError) as excinfo: - bad_call(async_gen, 0) - msg = "expected an async function but got an async generator" - assert msg in str(excinfo.value) - - # Make sure no references are kept around to keep anything alive - del excinfo - - def test_calling_asyncio_function_gives_nice_error(): async def child_xyzzy(): import asyncio diff --git a/trio/_util.py b/trio/_util.py index f5733ad85e..ac1a87e41b 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -92,7 +92,7 @@ def is_main_thread(): # Call the function and get the coroutine object, while giving helpful # errors for common mistakes. Returns coroutine object. ###### -def coroutine_or_error(async_fn, args): +def coroutine_or_error(async_fn, args=[]): def _return_value_looks_like_wrong_library(value): # Returned by legacy @asyncio.coroutine functions, which includes # a surprising proportion of asyncio builtins. diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 8e93e7c34d..c09060292e 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -1,9 +1,11 @@ import signal import warnings import pytest +from contextlib import contextmanager import trio from .. import _core +from .._core.tests.tutil import gc_collect_harder from .._util import ( signal_raise, ConflictDetector, is_main_thread, coroutine_or_error, generic_function, Final, NoPublicConstructor, @@ -83,64 +85,80 @@ def not_main_thread(): await trio.to_thread.run_sync(not_main_thread) -async def test_coroutine_or_error(): +# Some of our tests need to leak coroutines, and thus trigger the +# "RuntimeWarning: coroutine '...' was never awaited" message. This context +# manager should be used anywhere this happens to hide those messages, because +# when expected they're clutter. +@contextmanager +def ignore_coroutine_never_awaited_warnings(): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message="coroutine '.*' was never awaited" + ) + try: + yield + finally: + # Make sure to trigger any coroutine __del__ methods now, before + # we leave the context manager. + gc_collect_harder() - # error for: nursery.start_soon(trio.sleep(1)) - warnings.filterwarnings("error") - async def test_afunc(): - pass +# @coroutine is deprecated since python 3.8, which is fine with us. +@pytest.mark.filterwarnings("ignore:.*@coroutine.*:DeprecationWarning") +def test_coroutine_or_error(): + class Deferred: + "Just kidding" - with pytest.raises(TypeError): - try: - coroutine_or_error(test_afunc(), []) - except RuntimeWarning: + with ignore_coroutine_never_awaited_warnings(): + + async def f(): # pragma: no cover pass - # error for: nursery.start_soon(future) + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(f()) + assert "expecting an async function" in str(excinfo.value) - # legacy @asyncio.coroutine functions - def test_generator(): - yield None + import asyncio - with pytest.raises(TypeError): - coroutine_or_error(test_generator, []) + @asyncio.coroutine + def generator_based_coro(): # pragma: no cover + yield from asyncio.sleep(1) - # asyncio Future-like object - class AsycioFutureLike: - def __init__(self): - self._asyncio_future_blocking = "im a value!" + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(generator_based_coro()) + assert "asyncio" in str(excinfo.value) - with pytest.raises(TypeError): - coroutine_or_error(AsycioFutureLike(), []) + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(asyncio.Future()) + assert "asyncio" in str(excinfo.value) - # tornado Futures - class Future: - pass + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(lambda: asyncio.Future()) + assert "asyncio" in str(excinfo.value) - with pytest.raises(TypeError): - coroutine_or_error(Future(), []) + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(Deferred()) + assert "twisted" in str(excinfo.value) - # twisted Deferreds - class Deferreds: - pass + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(lambda: Deferred()) + assert "twisted" in str(excinfo.value) - with pytest.raises(TypeError): - coroutine_or_error(Deferreds(), []) + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(len, [[1, 2, 3]]) - # async generator - async def test_agenerator(): - yield None + assert "appears to be synchronous" in str(excinfo.value) - with pytest.raises(TypeError): - coroutine_or_error(test_agenerator, []) + async def async_gen(arg): # pragma: no cover + yield - # synchronous function - def test_fn(): - pass + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(async_gen, [0]) + msg = "expected an async function but got an async generator" + assert msg in str(excinfo.value) - with pytest.raises(TypeError): - coroutine_or_error(test_fn, []) + # Make sure no references are kept around to keep anything alive + del excinfo def test_generic_function(): From c74a829e8c38795f89cb2bbc472e2f8880e69663 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 13 May 2020 21:15:12 -0300 Subject: [PATCH 0131/1498] Added error message check --- trio/tests/test_threads.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index e937cb66e1..ed53a13e01 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -475,9 +475,11 @@ def thread_fn(): async def async_fn(): pass - with pytest.raises(TypeError): + with pytest.raises(TypeError) as excinfo: from_thread_run_sync(async_fn) + assert "expected a sync function" in str(excinfo.value) + async def test_trio_from_thread_run(): # Test that to_thread_run_sync correctly "hands off" the trio token to @@ -502,9 +504,11 @@ def sync_fn(): def thread_fn(): from_thread_run(sync_fn) - with pytest.raises(TypeError): + with pytest.raises(TypeError) as excinfo: await to_thread_run_sync(thread_fn) + assert "appears to be synchronous" in str(excinfo.value) + async def test_trio_from_thread_token(): # Test that to_thread_run_sync and spawned trio.from_thread.run_sync() From ad2733845cbaad79d294f57dab21d9a1183fcb08 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 13 May 2020 22:30:11 -0300 Subject: [PATCH 0132/1498] Marked some function stubs with #pragma no cover --- trio/tests/test_threads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index ed53a13e01..1c9c5e59bc 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -472,7 +472,7 @@ def thread_fn(): assert isinstance(trio_time, float) # Test correct error when passed async function - async def async_fn(): + async def async_fn(): # pragma: no cover pass with pytest.raises(TypeError) as excinfo: @@ -498,7 +498,7 @@ def thread_fn(): assert record == ["in thread", "back in trio"] # Test correct error when passed sync function - def sync_fn(): + def sync_fn(): # pragma: no cover pass def thread_fn(): From fcaac494bc58f2ad6634b85a1cc33956a2fdaf25 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 14 May 2020 06:33:29 +0000 Subject: [PATCH 0133/1498] Bump immutables from 0.12 to 0.13 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.12 to 0.13. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.12...v0.13) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index dcb85af4f3..5b9286e901 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ click==7.1.2 # via towncrier docutils==0.16 # via sphinx idna==2.9 # via -r docs-requirements.in, requests imagesize==1.2.0 # via sphinx -immutables==0.12 # via -r docs-requirements.in +immutables==0.13 # via -r docs-requirements.in incremental==17.5.0 # via towncrier jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 diff --git a/test-requirements.txt b/test-requirements.txt index e59a652dc2..b264195b39 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,7 +15,7 @@ cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.1 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme -immutables==0.12 # via -r test-requirements.in +immutables==0.13 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.14.0 # via -r test-requirements.in isort==4.3.21 # via pylint From 22d95e67e4a9846e509028e0676deec0d1b52cf3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 14 May 2020 06:34:07 +0000 Subject: [PATCH 0134/1498] Bump toml from 0.10.0 to 0.10.1 Bumps [toml](https://github.com/uiri/toml) from 0.10.0 to 0.10.1. - [Release notes](https://github.com/uiri/toml/releases) - [Changelog](https://github.com/uiri/toml/blob/master/RELEASE.rst) - [Commits](https://github.com/uiri/toml/compare/0.10.0...0.10.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index dcb85af4f3..152cf2373b 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -37,6 +37,6 @@ sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -toml==0.10.0 # via towncrier +toml==0.10.1 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in urllib3==1.25.9 # via requests diff --git a/test-requirements.txt b/test-requirements.txt index e59a652dc2..619078d6c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -44,7 +44,7 @@ pytest==5.4.2 # via -r test-requirements.in, pytest-cov six==1.14.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in -toml==0.10.0 # via pylint +toml==0.10.1 # via pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in wcwidth==0.1.9 # via prompt-toolkit, pytest From 57c10809f8275ed02c478305d72bb42f45f5d582 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Fri, 15 May 2020 17:48:49 -0300 Subject: [PATCH 0135/1498] Fixed comment on spawn_impl. Moved ignore_coroutine_never_awaited_warnings to tutil. Added raised error test to nursery.start_soon & _core.run. coroutine_or_error now takes variable arguments. Added RuntimeWarning protection to from_thread_run_sync & coroutine_or_error. Fixed typos in comments. --- trio/_core/_run.py | 7 ++--- trio/_core/tests/test_run.py | 56 ++++++++++++++++++++++++------------ trio/_core/tests/tutil.py | 20 +++++++++++++ trio/_threads.py | 23 +++++++-------- trio/_util.py | 9 ++++-- trio/tests/test_threads.py | 16 +++++------ trio/tests/test_util.py | 22 +------------- 7 files changed, 86 insertions(+), 67 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 26da1ed9f0..02ee8621ab 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1242,12 +1242,11 @@ def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): if nursery is None: assert self.init_task is None - # Check if async_fn is callable coroutine - coro = coroutine_or_error(async_fn, args) - ###### - # Set up the Task object + # Call the function and get the coroutine object, while giving helpful + # errors for common mistakes. ###### + coro = coroutine_or_error(async_fn, *args) if name is None: name = async_fn diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 880b89369e..118b046fb5 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -15,7 +15,11 @@ import sniffio import pytest -from .tutil import slow, check_sequence_matches, gc_collect_harder +from .tutil import ( + slow, check_sequence_matches, gc_collect_harder, + ignore_coroutine_never_awaited_warnings +) + from ... import _core from ..._threads import to_thread_run_sync from ..._timeouts import sleep, fail_after @@ -33,24 +37,6 @@ async def sleep_forever(): return await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) -# Some of our tests need to leak coroutines, and thus trigger the -# "RuntimeWarning: coroutine '...' was never awaited" message. This context -# manager should be used anywhere this happens to hide those messages, because -# when expected they're clutter. -@contextmanager -def ignore_coroutine_never_awaited_warnings(): - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", message="coroutine '.*' was never awaited" - ) - try: - yield - finally: - # Make sure to trigger any coroutine __del__ methods now, before - # we leave the context manager. - gc_collect_harder() - - def test_basic(): async def trivial(x): return x @@ -1696,6 +1682,38 @@ async def test_current_effective_deadline(mock_clock): assert _core.current_effective_deadline() == inf +def test_nice_error_on_bad_calls_to_run_or_spawn(): + def bad_call_run(*args): + _core.run(*args) + + def bad_call_spawn(*args): + async def main(): + async with _core.open_nursery() as nursery: + nursery.start_soon(*args) + + _core.run(main) + + for bad_call in bad_call_run, bad_call_spawn: + + async def f(): # pragma: no cover + pass + + with ignore_coroutine_never_awaited_warnings(): + with pytest.raises(TypeError, match="expecting an async function"): + + bad_call(f()) + + async def async_gen(arg): # pragma: no cover + yield arg + + with pytest.raises( + TypeError, + match="expected an async function but got an async generator" + ): + + bad_call(async_gen, 0) + + def test_calling_asyncio_function_gives_nice_error(): async def child_xyzzy(): import asyncio diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index dac53b81fd..ac090cb8de 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -3,6 +3,8 @@ import os import pytest +import warnings +from contextlib import contextmanager import gc @@ -52,6 +54,24 @@ def gc_collect_harder(): gc.collect() +# Some of our tests need to leak coroutines, and thus trigger the +# "RuntimeWarning: coroutine '...' was never awaited" message. This context +# manager should be used anywhere this happens to hide those messages, because +# when expected they're clutter. +@contextmanager +def ignore_coroutine_never_awaited_warnings(): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message="coroutine '.*' was never awaited" + ) + try: + yield + finally: + # Make sure to trigger any coroutine __del__ methods now, before + # we leave the context manager. + gc_collect_harder() + + # template is like: # [1, {2.1, 2.2}, 3] -> matches [1, 2.1, 2.2, 3] or [1, 2.2, 2.1, 3] def check_sequence_matches(seq, template): diff --git a/trio/_threads.py b/trio/_threads.py index b93dc3b5d8..142d7005ac 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -383,7 +383,7 @@ def from_thread_run(afn, *args, trio_token=None): def callback(q, afn, args): @disable_ki_protection async def unprotected_afn(): - coro = coroutine_or_error(afn, args) + coro = coroutine_or_error(afn, *args) return await coro async def await_in_trio_thread_task(): @@ -407,9 +407,6 @@ def from_thread_run_sync(fn, *args, trio_token=None): Raises: RunFinishedError: if the corresponding call to `trio.run` has already completed. - Cancelled: if the corresponding call to `trio.run` completes - while ``fn(*args)`` is running, then ``fn`` is likely to raise - :exc:`trio.Cancelled`, and this will propagate out into RuntimeError: if you try calling this from inside the Trio thread, which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer @@ -427,17 +424,19 @@ def from_thread_run_sync(fn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ - - if not callable(fn) or inspect.iscoroutinefunction(fn): - raise TypeError( - "Trio expected a sync function, but {!r} appears to not be " - "callable or asynchronous".format(getattr(fn, "__qualname__", fn)) - ) - def callback(q, fn, args): @disable_ki_protection def unprotected_fn(): - return fn(*args) + call = fn(*args) + + if inspect.iscoroutine(call): + call.close() + raise TypeError( + "Trio expected a sync function, but {!r} appears to be " + "asynchronous".format(getattr(fn, "__qualname__", fn)) + ) + + return call res = outcome.capture(unprotected_fn) q.put_nowait(res) diff --git a/trio/_util.py b/trio/_util.py index ac1a87e41b..5cc9737d22 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -92,7 +92,7 @@ def is_main_thread(): # Call the function and get the coroutine object, while giving helpful # errors for common mistakes. Returns coroutine object. ###### -def coroutine_or_error(async_fn, args=[]): +def coroutine_or_error(async_fn, *args): def _return_value_looks_like_wrong_library(value): # Returned by legacy @asyncio.coroutine functions, which includes # a surprising proportion of asyncio builtins. @@ -110,9 +110,14 @@ def _return_value_looks_like_wrong_library(value): try: coro = async_fn(*args) + except TypeError: # Give good error for: nursery.start_soon(trio.sleep(1)) if isinstance(async_fn, collections.abc.Coroutine): + + # explicitly close coroutine to avoid RuntimeWarning + async_fn.close() + raise TypeError( "Trio was expecting an async function, but instead it got " "a coroutine object {async_fn!r}\n" @@ -148,7 +153,7 @@ def _return_value_looks_like_wrong_library(value): # Give good error for: nursery.start_soon(func_returning_future) if _return_value_looks_like_wrong_library(coro): raise TypeError( - "start_soon got unexpected {!r} – are you trying to use a " + "Trio got unexpected {!r} – are you trying to use a " "library written for asyncio/twisted/tornado or similar? " "That won't work without some sort of compatibility shim." .format(coro) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 1c9c5e59bc..8f92d94d65 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -13,7 +13,7 @@ ) from .._core.tests.test_ki import ki_self -from .._core.tests.tutil import slow +from .._core.tests.tutil import ignore_coroutine_never_awaited_warnings async def test_do_in_trio_thread(): @@ -475,10 +475,12 @@ def thread_fn(): async def async_fn(): # pragma: no cover pass - with pytest.raises(TypeError) as excinfo: + def thread_fn(): from_thread_run_sync(async_fn) - assert "expected a sync function" in str(excinfo.value) + with pytest.raises(TypeError, match="expected a sync function"): + + await to_thread_run_sync(thread_fn) async def test_trio_from_thread_run(): @@ -501,13 +503,9 @@ def thread_fn(): def sync_fn(): # pragma: no cover pass - def thread_fn(): - from_thread_run(sync_fn) - - with pytest.raises(TypeError) as excinfo: - await to_thread_run_sync(thread_fn) + with pytest.raises(TypeError, match="appears to be synchronous"): - assert "appears to be synchronous" in str(excinfo.value) + await to_thread_run_sync(from_thread_run, sync_fn) async def test_trio_from_thread_token(): diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index c09060292e..009f9fa8f7 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -1,11 +1,9 @@ import signal -import warnings import pytest -from contextlib import contextmanager import trio from .. import _core -from .._core.tests.tutil import gc_collect_harder +from .._core.tests.tutil import ignore_coroutine_never_awaited_warnings from .._util import ( signal_raise, ConflictDetector, is_main_thread, coroutine_or_error, generic_function, Final, NoPublicConstructor, @@ -85,24 +83,6 @@ def not_main_thread(): await trio.to_thread.run_sync(not_main_thread) -# Some of our tests need to leak coroutines, and thus trigger the -# "RuntimeWarning: coroutine '...' was never awaited" message. This context -# manager should be used anywhere this happens to hide those messages, because -# when expected they're clutter. -@contextmanager -def ignore_coroutine_never_awaited_warnings(): - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", message="coroutine '.*' was never awaited" - ) - try: - yield - finally: - # Make sure to trigger any coroutine __del__ methods now, before - # we leave the context manager. - gc_collect_harder() - - # @coroutine is deprecated since python 3.8, which is fine with us. @pytest.mark.filterwarnings("ignore:.*@coroutine.*:DeprecationWarning") def test_coroutine_or_error(): From 3038ab75a3337e70112b991ac7e15704e4d5b837 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 15 May 2020 21:29:28 -0700 Subject: [PATCH 0136/1498] Automatically run towncrier on RTD for in-development versions This has several advantages: - On RTD, folks using the 'latest' tag to look at the git version of the docs will be able to see what's changed in git. - On pull requests, newsfragments will be included in the RTD CI build, so we have the option of previewing them in rendered form before merging. This commit also configures RTD sphinx to fail on warnings, in hopes that this will make RTD CI work as a replacement for our current doc CI job. --- .readthedocs.yml | 3 +++ docs/source/conf.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 909ccf1bb3..2a32d6c9b5 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,3 +9,6 @@ python: version: 3.7 install: - requirements: docs-requirements.txt + +sphinx: + fail_on_warning: true diff --git a/docs/source/conf.py b/docs/source/conf.py index 9ec78d6ea9..4492638dea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,6 +24,18 @@ # For trio itself sys.path.insert(0, os.path.abspath('../..')) +# https://docs.readthedocs.io/en/stable/builds.html#build-environment +if "READTHEDOCS" in os.environ: + import glob + if glob.glob("../../newsfragments/*.*.rst"): + print("-- Found newsfragments; running towncrier --", flush=True) + import subprocess + subprocess.run( + ["towncrier", "--yes", "--date", "not released yet"], + cwd="../..", + check=True, + ) + # Warn about all references to unknown targets nitpicky = True # Except for these ones, which we expect to point to unknown targets: From 5ea127c8db62de4855af2b0fc39fb681d1a770b9 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 16 May 2020 01:38:27 -0700 Subject: [PATCH 0137/1498] Allow customization of run_process cancellation behavior And also tweak the default to something a bit nicer on Unix. --- newsfragments/1104.feature.rst | 7 +++ trio/_subprocess.py | 87 +++++++++++++++++++++++++++++++++- trio/tests/test_subprocess.py | 54 +++++++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 newsfragments/1104.feature.rst diff --git a/newsfragments/1104.feature.rst b/newsfragments/1104.feature.rst new file mode 100644 index 0000000000..034fad25eb --- /dev/null +++ b/newsfragments/1104.feature.rst @@ -0,0 +1,7 @@ +Previously, when `trio.run_process` was cancelled, it always killed +the subprocess immediately. Now, on Unix, it first gives the process a +chance to clean up by sending ``SIGTERM``, and only escalates to +``SIGKILL`` if the process is still running after 5 seconds. But if +you prefer the old behavior, or want to adjust the timeout, then don't +worry: you can now pass a custom ``deliver_cancel=`` argument to +define your own process killing policy. diff --git a/trio/_subprocess.py b/trio/_subprocess.py index ab6ab76a7e..0b656c712d 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -3,6 +3,7 @@ import sys from typing import Optional from functools import partial +import warnings from ._abc import AsyncResource, SendStream, ReceiveStream from ._highlevel_generic import StapledStream @@ -377,6 +378,34 @@ async def open_process( return Process._create(popen, trio_stdin, trio_stdout, trio_stderr) +async def _windows_deliver_cancel(p): + try: + p.terminate() + except OSError as exc: + warnings.warn( + RuntimeWarning(f"TerminateProcess on {p!r} failed with: {exc!r}") + ) + + +async def _posix_deliver_cancel(p): + try: + p.terminate() + await trio.sleep(5) + warnings.warn( + RuntimeWarning( + f"process {p!r} ignored SIGTERM for 5 seconds. " + f"(Maybe you should custom deliver_cancel?) Trying SIGKILL." + ) + ) + p.kill() + except OSError as exc: + warnings.warn( + RuntimeWarning( + f"tried to kill process {p!r}, but failed with: {exc!r}" + ) + ) + + async def run_process( command, *, @@ -384,6 +413,7 @@ async def run_process( capture_stdout=False, capture_stderr=False, check=True, + deliver_cancel=None, **options ): """Run ``command`` in a subprocess, wait for it to complete, and @@ -441,6 +471,7 @@ async def run_process( ``**options``, or on Windows, ``command`` may alternatively be a string, which will be parsed following platform-dependent :ref:`quoting rules `. + stdin (:obj:`bytes`, file descriptor, or None): The bytes to provide to the subprocess on its standard input stream, or ``None`` if the subprocess's standard input should come from the same place as @@ -449,18 +480,52 @@ async def run_process( file descriptor or an object with a ``fileno()`` method, in which case the subprocess's standard input will come from that file. + capture_stdout (bool): If true, capture the bytes that the subprocess writes to its standard output stream and return them in the :attr:`~subprocess.CompletedProcess.stdout` attribute of the returned :class:`~subprocess.CompletedProcess` object. + capture_stderr (bool): If true, capture the bytes that the subprocess writes to its standard error stream and return them in the :attr:`~subprocess.CompletedProcess.stderr` attribute of the returned :class:`~subprocess.CompletedProcess` object. + check (bool): If false, don't validate that the subprocess exits successfully. You should be sure to check the ``returncode`` attribute of the returned object if you pass ``check=False``, so that errors don't pass silently. + + deliver_cancel (async function or None): If `run_process` is cancelled, + then it needs to kill the child process. There are multiple ways to + do this, so we let you customize it. + + If you pass None (the default), then the behavior depends on the + platform: + + - On Windows, Trio calls ``TerminateProcess``, which should kill the + process immediately. + + - On Unix-likes, the default behavior is to send a ``SIGTERM``, wait + 5 seconds, and send a ``SIGKILL``. + + Alternatively, you can customize this behavior by passing in an + arbitrary async function, which will be called with the `Process` + object as an argument. For example, the default Unix behavior could + be implemented like this:: + + async def my_deliver_cancel(process): + process.send_signal(signal.SIGTERM) + await trio.sleep(5) + process.send_signal(signal.SIGKILL) + + When the process actually exits, the ``deliver_cancel`` function + will automatically be cancelled – so if the process exits after + ``SIGTERM``, then we'll never reach the ``SIGKILL``. + + In any case, `run_process` will always wait for the child process to + exit before raising `Cancelled`. + **options: :func:`run_process` also accepts any :ref:`general subprocess options ` and passes them on to the :class:`~trio.Process` constructor. This includes the @@ -518,6 +583,13 @@ async def run_process( raise ValueError("can't specify both stderr and capture_stderr") options["stderr"] = subprocess.PIPE + if deliver_cancel is None: + if os.name == "nt": + deliver_cancel = _windows_deliver_cancel + else: + assert os.name == "posix" + deliver_cancel = _posix_deliver_cancel + stdout_chunks = [] stderr_chunks = [] @@ -542,7 +614,20 @@ async def read_output(stream, chunks): nursery.start_soon(read_output, proc.stdout, stdout_chunks) if proc.stderr is not None: nursery.start_soon(read_output, proc.stderr, stderr_chunks) - await proc.wait() + try: + await proc.wait() + except trio.Cancelled: + with trio.CancelScope(shield=True): + killer_cscope = trio.CancelScope(shield=True) + + async def killer(): + with killer_cscope: + await deliver_cancel(proc) + + nursery.start_soon(killer) + await proc.wait() + killer_cscope.cancel() + raise stdout = b"".join(stdout_chunks) if proc.stdout is not None else None stderr = b"".join(stderr_chunks) if proc.stderr is not None else None diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index ebc838b0ed..c5be0e393e 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -4,6 +4,7 @@ import sys import pytest import random +from functools import partial from .. import ( _core, move_on_after, fail_after, sleep, sleep_forever, Process, @@ -425,3 +426,56 @@ def on_alarm(sig, frame): sleeper.kill() sleeper.wait() signal.signal(signal.SIGALRM, old_sigalrm) + + +async def test_custom_deliver_cancel(): + custom_deliver_cancel_called = False + + async def custom_deliver_cancel(proc): + nonlocal custom_deliver_cancel_called + custom_deliver_cancel_called = True + proc.terminate() + # Make sure this does get cancelled when the process exits, and that + # the process really exited. + try: + await sleep_forever() + finally: + assert proc.returncode is not None + + async with _core.open_nursery() as nursery: + nursery.start_soon( + partial( + run_process, SLEEP(9999), deliver_cancel=custom_deliver_cancel + ) + ) + await wait_all_tasks_blocked() + nursery.cancel_scope.cancel() + + assert custom_deliver_cancel_called + + +async def test_warn_on_failed_cancel_terminate(monkeypatch): + original_terminate = Process.terminate + + def broken_terminate(self): + original_terminate(self) + raise OSError("whoops") + + monkeypatch.setattr(Process, "terminate", broken_terminate) + + with pytest.warns(RuntimeWarning, match=".*whoops.*"): + async with _core.open_nursery() as nursery: + nursery.start_soon(run_process, SLEEP(9999)) + await wait_all_tasks_blocked() + nursery.cancel_scope.cancel() + + +@pytest.mark.skipif(os.name != "posix", reason="posix only") +async def test_warn_on_cancel_SIGKILL_escalation(autojump_clock, monkeypatch): + monkeypatch.setattr(Process, "terminate", lambda *args: None) + + with pytest.warns(RuntimeWarning, match=".*ignored SIGTERM.*"): + async with _core.open_nursery() as nursery: + nursery.start_soon(run_process, SLEEP(9999)) + await wait_all_tasks_blocked() + nursery.cancel_scope.cancel() From 1ec39a7465c1aa6744c1ffd019fe7db13f9fb8ac Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 16 May 2020 01:43:47 -0700 Subject: [PATCH 0138/1498] Tell Sphinx to allow "async function" as a valid argument type --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 4492638dea..49afad6cfd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -49,6 +49,7 @@ ("py:mod", "trio.abc"), ("py:class", "math.inf"), ("py:exc", "Anything else"), + ("py:class", "async function"), ] autodoc_inherit_docstrings = False default_role = "obj" From ec960c7c1672773823417082d1fa001576bbd35e Mon Sep 17 00:00:00 2001 From: kAldown Date: Sat, 16 May 2020 15:00:02 +0300 Subject: [PATCH 0139/1498] [#1526]: Preventing the problem of catching an empty list of signals. Newsfragment bugfix were added. --- newsfragments/1526.bugfix.rst | 1 + trio/_signals.py | 5 +++++ trio/tests/test_signals.py | 6 ++++++ 3 files changed, 12 insertions(+) create mode 100644 newsfragments/1526.bugfix.rst diff --git a/newsfragments/1526.bugfix.rst b/newsfragments/1526.bugfix.rst new file mode 100644 index 0000000000..02e1d37c4a --- /dev/null +++ b/newsfragments/1526.bugfix.rst @@ -0,0 +1 @@ +Calling `open_signal_receiver` with no arguments used to succeed without listening for any signals. This was confusing, so now it raises TypeError instead. diff --git a/trio/_signals.py b/trio/_signals.py index aca23dda63..cee3b7db53 100644 --- a/trio/_signals.py +++ b/trio/_signals.py @@ -129,6 +129,8 @@ def open_signal_receiver(*signals): signals: the signals to listen for. Raises: + TypeError: if no signals were provided. + RuntimeError: if you try to use this anywhere except Python's main thread. (This is a Python limitation.) @@ -144,6 +146,9 @@ def open_signal_receiver(*signals): reload_configuration() """ + if not signals: + raise TypeError("No signals were provided") + if not is_main_thread(): raise RuntimeError( "Sorry, open_signal_receiver is only possible when running in " diff --git a/trio/tests/test_signals.py b/trio/tests/test_signals.py index 7ae930403c..20821b40f2 100644 --- a/trio/tests/test_signals.py +++ b/trio/tests/test_signals.py @@ -41,6 +41,12 @@ async def test_open_signal_receiver_restore_handler_after_one_bad_signal(): assert signal.getsignal(signal.SIGILL) is orig +async def test_open_signal_receiver_empty_fail(): + with pytest.raises(TypeError, match="No signals were provided"): + with open_signal_receiver(): + pass + + async def test_open_signal_receiver_restore_handler_after_duplicate_signal(): orig = signal.getsignal(signal.SIGILL) with open_signal_receiver(signal.SIGILL, signal.SIGILL): From f4c11e9dacabbc62588fbf0ef1fe71d7a0cf630b Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Sat, 16 May 2020 09:02:30 -0300 Subject: [PATCH 0140/1498] Fixed/added some comments. Removed unnecessary ignore_coroutine_never_awaited_warnings. Better naming of return value in from_thread_run_sync. --- trio/_core/_run.py | 3 +++ trio/_core/tests/test_run.py | 6 ++---- trio/_threads.py | 12 +++++++----- trio/tests/test_threads.py | 1 - 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 02ee8621ab..f77860d2f2 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1273,6 +1273,9 @@ async def python_wrapper(orig_coro): LOCALS_KEY_KI_PROTECTION_ENABLED, system_task ) + ###### + # Set up the Task object + ###### task = Task._create( coro=coro, parent_nursery=nursery, diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 118b046fb5..6c08c685f3 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1698,10 +1698,9 @@ async def main(): async def f(): # pragma: no cover pass - with ignore_coroutine_never_awaited_warnings(): - with pytest.raises(TypeError, match="expecting an async function"): + with pytest.raises(TypeError, match="expecting an async function"): - bad_call(f()) + bad_call(f()) async def async_gen(arg): # pragma: no cover yield arg @@ -1710,7 +1709,6 @@ async def async_gen(arg): # pragma: no cover TypeError, match="expected an async function but got an async generator" ): - bad_call(async_gen, 0) diff --git a/trio/_threads.py b/trio/_threads.py index 142d7005ac..ffd85812cf 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -411,7 +411,7 @@ def from_thread_run_sync(fn, *args, trio_token=None): which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer one from context. Also if ``fn`` is not a sync function. - TypeError: if ``fn`` is not callable or is an async function. + TypeError: if ``fn`` is an async function. **Locating a Trio Token**: There are two ways to specify which `trio.run` loop to reenter: @@ -427,16 +427,18 @@ def from_thread_run_sync(fn, *args, trio_token=None): def callback(q, fn, args): @disable_ki_protection def unprotected_fn(): - call = fn(*args) + ret = fn(*args) - if inspect.iscoroutine(call): - call.close() + if inspect.iscoroutine(ret): + + # Manually close coroutine to avoid RuntimeWarnings + ret.close() raise TypeError( "Trio expected a sync function, but {!r} appears to be " "asynchronous".format(getattr(fn, "__qualname__", fn)) ) - return call + return ret res = outcome.capture(unprotected_fn) q.put_nowait(res) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 8f92d94d65..545986a795 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -13,7 +13,6 @@ ) from .._core.tests.test_ki import ki_self -from .._core.tests.tutil import ignore_coroutine_never_awaited_warnings async def test_do_in_trio_thread(): From 0e56e28a9cece675db3fa6badfca21922dab5a19 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 16 May 2020 17:07:37 -0700 Subject: [PATCH 0141/1498] Make Trio's excepthook play nicely with Ubuntu's excepthook Fixes gh-1065 --- ci.sh | 5 ++ newsfragments/1065.bugfix.rst | 8 +++ trio/_core/_multierror.py | 53 ++++++++++++++----- trio/_core/tests/test_multierror.py | 23 ++++++++ .../apport_excepthook.py | 11 ++++ 5 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 newsfragments/1065.bugfix.rst create mode 100644 trio/_core/tests/test_multierror_scripts/apport_excepthook.py diff --git a/ci.sh b/ci.sh index aed1974ddd..73a3d8d0a9 100755 --- a/ci.sh +++ b/ci.sh @@ -407,6 +407,11 @@ else # Actual tests python -m pip install -r test-requirements.txt + # So we can run the test for our apport/excepthook interaction working + if [ -e /etc/lsb-release ] && grep -q Ubuntu /etc/lsb-release; then + sudo apt install -q python3-apport + fi + # If we're testing with a LSP installed, then it might break network # stuff, so wait until after we've finished setting everything else # up. diff --git a/newsfragments/1065.bugfix.rst b/newsfragments/1065.bugfix.rst new file mode 100644 index 0000000000..737680bd4c --- /dev/null +++ b/newsfragments/1065.bugfix.rst @@ -0,0 +1,8 @@ +On Ubuntu systems, the system Python includes a custom +unhandled-exception hook to perform `crash reporting +`__. Unfortunately, Trio wants to use +the same hook to print nice `MultiError` tracebacks, causing a +conflict. Previously, Trio would detect the conflict, print a warning, +and you just wouldn't get nice `MultiError` tracebacks. Now, Trio has +gotten clever enough to integrate its hook with Ubuntu's, so the two +systems should Just Work together. diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 00d44e30ed..f98540344d 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -434,8 +434,8 @@ def trio_excepthook(etype, value, tb): sys.stderr.write(chunk) -IPython_handler_installed = False -warning_given = False +monkeypatched_or_warned = False + if "IPython" in sys.modules: import IPython ip = IPython.get_ipython() @@ -448,7 +448,7 @@ def trio_excepthook(etype, value, tb): "tracebacks.", category=RuntimeWarning ) - warning_given = True + monkeypatched_or_warned = True else: def trio_show_traceback(self, etype, value, tb, tb_offset=None): @@ -457,15 +457,44 @@ def trio_show_traceback(self, etype, value, tb, tb_offset=None): trio_excepthook(etype, value, tb) ip.set_custom_exc((MultiError,), trio_show_traceback) - IPython_handler_installed = True + monkeypatched_or_warned = True if sys.excepthook is sys.__excepthook__: sys.excepthook = trio_excepthook -else: - if not IPython_handler_installed and not warning_given: - warnings.warn( - "You seem to already have a custom sys.excepthook handler " - "installed. I'll skip installing Trio's custom handler, but this " - "means MultiErrors will not show full tracebacks.", - category=RuntimeWarning - ) + monkeypatched_or_warned = True + +# Ubuntu's system Python has a sitecustomize.py file that import +# apport_python_hook and replaces sys.excepthook. +# +# The custom hook captures the error for crash reporting, and then calls +# sys.__excepthook__ to actually print the error. +# +# We don't mind it capturing the error for crash reporting, but we want to +# take over printing the error. So we monkeypatch the apport_python_hook +# module so that instead of calling sys.__excepthook__, it calls our custom +# hook. +# +# More details: https://github.com/python-trio/trio/issues/1065 +if sys.excepthook.__name__ == "apport_excepthook": + import apport_python_hook + assert sys.excepthook is apport_python_hook.apport_excepthook + + # Give it a descriptive name as a hint for anyone who's stuck trying to + # debug this mess later. + class TrioFakeSysModuleForApport: + pass + + fake_sys = TrioFakeSysModuleForApport() + fake_sys.__dict__.update(sys.__dict__) + fake_sys.__excepthook__ = trio_excepthook + apport_python_hook.sys = fake_sys + + monkeypatched_or_warned = True + +if not monkeypatched_or_warned: + warnings.warn( + "You seem to already have a custom sys.excepthook handler " + "installed. I'll skip installing Trio's custom handler, but this " + "means MultiErrors will not show full tracebacks.", + category=RuntimeWarning + ) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 83894f358d..6debf4d45c 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -709,3 +709,26 @@ def test_ipython_custom_exc_handler(): ) # Make sure our other warning doesn't show up assert "custom sys.excepthook" not in completed.stdout.decode("utf-8") + + +@slow +@pytest.mark.skipif( + not Path("/usr/lib/python3/dist-packages/apport_python_hook.py").exists(), + reason="need Ubuntu with python3-apport installed" +) +def test_apport_excepthook_monkeypatch_interaction(): + completed = run_script("apport_excepthook.py") + stdout = completed.stdout.decode("utf-8") + + # No warning + assert "custom sys.excepthook" not in stdout + + # Proper traceback + assert_match_in_seq( + [ + "Details of embedded", + "KeyError", + "Details of embedded", + "ValueError", + ], stdout + ) diff --git a/trio/_core/tests/test_multierror_scripts/apport_excepthook.py b/trio/_core/tests/test_multierror_scripts/apport_excepthook.py new file mode 100644 index 0000000000..ac8110f36e --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/apport_excepthook.py @@ -0,0 +1,11 @@ +# The apport_python_hook package is only installed as part of Ubuntu's system +# python, and not available in venvs. So before we can import it we have to +# make sure it's on sys.path. +import sys +sys.path.append("/usr/lib/python3/dist-packages") +import apport_python_hook +apport_python_hook.install() + +import trio + +raise trio.MultiError([KeyError("key_error"), ValueError("value_error")]) From 8174f4502db8b933a94be22c50b06a45ededc6d7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 06:37:20 +0000 Subject: [PATCH 0142/1498] Bump immutables from 0.13 to 0.14 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.13 to 0.14. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.13...v0.14) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c751af7825..1b4da0c00a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ click==7.1.2 # via towncrier docutils==0.16 # via sphinx idna==2.9 # via -r docs-requirements.in, requests imagesize==1.2.0 # via sphinx -immutables==0.13 # via -r docs-requirements.in +immutables==0.14 # via -r docs-requirements.in incremental==17.5.0 # via towncrier jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 diff --git a/test-requirements.txt b/test-requirements.txt index 6845634d4a..768584add6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,7 +15,7 @@ cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.1 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme -immutables==0.13 # via -r test-requirements.in +immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.14.0 # via -r test-requirements.in isort==4.3.21 # via pylint From eae6ff256bd105de5b15bc387b3eedf0bf9c0469 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 06:37:52 +0000 Subject: [PATCH 0143/1498] Bump more-itertools from 8.2.0 to 8.3.0 Bumps [more-itertools](https://github.com/more-itertools/more-itertools) from 8.2.0 to 8.3.0. - [Release notes](https://github.com/more-itertools/more-itertools/releases) - [Commits](https://github.com/more-itertools/more-itertools/compare/v8.2.0...v8.3.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6845634d4a..c8106a04c6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ isort==4.3.21 # via pylint jedi==0.17.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint -more-itertools==8.2.0 # via pytest +more-itertools==8.3.0 # via pytest outcome==1.0.1 # via -r test-requirements.in packaging==20.3 # via pytest parso==0.7.0 # via jedi From 28b08d1a40b83d3a478d9ac9315b7e3bc5af4f8d Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Mon, 18 May 2020 09:17:49 -0300 Subject: [PATCH 0144/1498] Fixed a comment and other formatting issues. --- trio/_core/tests/test_run.py | 1 - trio/_threads.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 6c08c685f3..53446a601f 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1699,7 +1699,6 @@ async def f(): # pragma: no cover pass with pytest.raises(TypeError, match="expecting an async function"): - bad_call(f()) async def async_gen(arg): # pragma: no cover diff --git a/trio/_threads.py b/trio/_threads.py index ffd85812cf..c03a353789 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -410,7 +410,7 @@ def from_thread_run_sync(fn, *args, trio_token=None): RuntimeError: if you try calling this from inside the Trio thread, which would otherwise cause a deadlock. AttributeError: if no ``trio_token`` was provided, and we can't infer - one from context. Also if ``fn`` is not a sync function. + one from context. TypeError: if ``fn`` is an async function. **Locating a Trio Token**: There are two ways to specify which @@ -430,7 +430,6 @@ def unprotected_fn(): ret = fn(*args) if inspect.iscoroutine(ret): - # Manually close coroutine to avoid RuntimeWarnings ret.close() raise TypeError( From 4c68a41f591cab8cd2f5541399cc7472a17566c6 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 18 May 2020 11:56:19 -0700 Subject: [PATCH 0145/1498] Fix grammar in deliver_cancel warning --- trio/_subprocess.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 0b656c712d..16a869f8ef 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -394,7 +394,8 @@ async def _posix_deliver_cancel(p): warnings.warn( RuntimeWarning( f"process {p!r} ignored SIGTERM for 5 seconds. " - f"(Maybe you should custom deliver_cancel?) Trying SIGKILL." + f"(Maybe you should pass a custom deliver_cancel?) " + f"Trying SIGKILL." ) ) p.kill() From 84d550d21ea59e54b0ae60676e53b9f1ff92fc8e Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 19 May 2020 09:02:06 +0400 Subject: [PATCH 0146/1498] Bump version to 0.15.0 --- docs/source/history.rst | 73 ++++++++++++++++++++++++++++++++++ newsfragments/1044.removal.rst | 17 -------- newsfragments/1065.bugfix.rst | 8 ---- newsfragments/1104.feature.rst | 7 ---- newsfragments/1109.feature.rst | 3 -- newsfragments/1109.removal.rst | 3 -- newsfragments/1118.feature.rst | 1 - newsfragments/1241.feature.rst | 4 -- newsfragments/1315.feature.rst | 4 -- newsfragments/1498.removal.rst | 1 - newsfragments/1499.bugfix.rst | 2 - newsfragments/1526.bugfix.rst | 1 - newsfragments/476.removal.rst | 3 -- newsfragments/75.removal.rst | 1 - trio/_version.py | 2 +- 15 files changed, 74 insertions(+), 56 deletions(-) delete mode 100644 newsfragments/1044.removal.rst delete mode 100644 newsfragments/1065.bugfix.rst delete mode 100644 newsfragments/1104.feature.rst delete mode 100644 newsfragments/1109.feature.rst delete mode 100644 newsfragments/1109.removal.rst delete mode 100644 newsfragments/1118.feature.rst delete mode 100644 newsfragments/1241.feature.rst delete mode 100644 newsfragments/1315.feature.rst delete mode 100644 newsfragments/1498.removal.rst delete mode 100644 newsfragments/1499.bugfix.rst delete mode 100644 newsfragments/1526.bugfix.rst delete mode 100644 newsfragments/476.removal.rst delete mode 100644 newsfragments/75.removal.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 320ecbab9a..133b615d85 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,79 @@ Release history .. towncrier release notes start +Trio 0.15.0 (2020-05-19) +------------------------ + +Features +~~~~~~~~ + +- Previously, when `trio.run_process` was cancelled, it always killed + the subprocess immediately. Now, on Unix, it first gives the process a + chance to clean up by sending ``SIGTERM``, and only escalates to + ``SIGKILL`` if the process is still running after 5 seconds. But if + you prefer the old behavior, or want to adjust the timeout, then don't + worry: you can now pass a custom ``deliver_cancel=`` argument to + define your own process killing policy. (`#1104 `__) +- It turns out that creating a subprocess can block the parent process + for a surprisingly long time. So `trio.open_process` now uses a worker + thread to avoid blocking the event loop. (`#1109 `__) +- We've added FreeBSD to the list of platforms we support and test on. (`#1118 `__) +- On Linux kernels v5.3 or newer, `trio.Process.wait` now uses `the + pidfd API `__ to track child + processes. This shouldn't have any user-visible change, but it makes + working with subprocesses faster and use less memory. (`#1241 `__) +- The `trio.Process.returncode` attribute is now automatically updated + as needed, instead of only when you call `~trio.Process.poll` or + `~trio.Process.wait`. Also, ``repr(process_object)`` now always + contains up-to-date information about the process status. (`#1315 `__) + + +Bugfixes +~~~~~~~~ + +- On Ubuntu systems, the system Python includes a custom + unhandled-exception hook to perform `crash reporting + `__. Unfortunately, Trio wants to use + the same hook to print nice `MultiError` tracebacks, causing a + conflict. Previously, Trio would detect the conflict, print a warning, + and you just wouldn't get nice `MultiError` tracebacks. Now, Trio has + gotten clever enough to integrate its hook with Ubuntu's, so the two + systems should Just Work together. (`#1065 `__) +- Fixed an over-strict test that caused failures on Alpine Linux. + Started testing against Alpine in CI. (`#1499 `__) +- Calling `open_signal_receiver` with no arguments used to succeed without listening for any signals. This was confusing, so now it raises TypeError instead. (`#1526 `__) + + +Deprecations and Removals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Remove support for Python 3.5. (`#75 `__) +- It turns out that everyone got confused by the name ``trio.hazmat``. + So that name has been deprecated, and the new name is + :mod:`trio.lowlevel`. (`#476 `__) +- Most of the public classes that Trio exports – like `trio.Lock`, + `trio.SocketStream`, and so on – weren't designed with subclassing in + mind. And we've noticed that some users were trying to subclass them + anyway, and ending up with fragile code that we're likely to + accidentally break in the future, or else be stuck unable to make + changes for fear of breaking subclasses. + + There are also some classes that were explicitly designed to be + subclassed, like the ones in ``trio.abc``. Subclassing these is still + supported. However, for all other classes, attempts to subclass will + now raise a deprecation warning, and in the future will raise an + error. + + If this causes problems for you, feel free to drop by our `chat room + `__ or file a bug, to discuss + alternatives or make a case for why some particular class should be + designed to support subclassing. (`#1044 `__) +- If you want to create a `trio.Process` object, you now have to call + `trio.open_process`; calling ``trio.Process()`` directly was + deprecated in v0.12.0 and has now been removed. (`#1109 `__) +- Remove ``clear`` method on `trio.Event`: it was deprecated in 0.12.0. (`#1498 `__) + + Trio 0.14.0 (2020-04-27) ------------------------ diff --git a/newsfragments/1044.removal.rst b/newsfragments/1044.removal.rst deleted file mode 100644 index 7939eafc00..0000000000 --- a/newsfragments/1044.removal.rst +++ /dev/null @@ -1,17 +0,0 @@ -Most of the public classes that Trio exports – like `trio.Lock`, -`trio.SocketStream`, and so on – weren't designed with subclassing in -mind. And we've noticed that some users were trying to subclass them -anyway, and ending up with fragile code that we're likely to -accidentally break in the future, or else be stuck unable to make -changes for fear of breaking subclasses. - -There are also some classes that were explicitly designed to be -subclassed, like the ones in ``trio.abc``. Subclassing these is still -supported. However, for all other classes, attempts to subclass will -now raise a deprecation warning, and in the future will raise an -error. - -If this causes problems for you, feel free to drop by our `chat room -`__ or file a bug, to discuss -alternatives or make a case for why some particular class should be -designed to support subclassing. diff --git a/newsfragments/1065.bugfix.rst b/newsfragments/1065.bugfix.rst deleted file mode 100644 index 737680bd4c..0000000000 --- a/newsfragments/1065.bugfix.rst +++ /dev/null @@ -1,8 +0,0 @@ -On Ubuntu systems, the system Python includes a custom -unhandled-exception hook to perform `crash reporting -`__. Unfortunately, Trio wants to use -the same hook to print nice `MultiError` tracebacks, causing a -conflict. Previously, Trio would detect the conflict, print a warning, -and you just wouldn't get nice `MultiError` tracebacks. Now, Trio has -gotten clever enough to integrate its hook with Ubuntu's, so the two -systems should Just Work together. diff --git a/newsfragments/1104.feature.rst b/newsfragments/1104.feature.rst deleted file mode 100644 index 034fad25eb..0000000000 --- a/newsfragments/1104.feature.rst +++ /dev/null @@ -1,7 +0,0 @@ -Previously, when `trio.run_process` was cancelled, it always killed -the subprocess immediately. Now, on Unix, it first gives the process a -chance to clean up by sending ``SIGTERM``, and only escalates to -``SIGKILL`` if the process is still running after 5 seconds. But if -you prefer the old behavior, or want to adjust the timeout, then don't -worry: you can now pass a custom ``deliver_cancel=`` argument to -define your own process killing policy. diff --git a/newsfragments/1109.feature.rst b/newsfragments/1109.feature.rst deleted file mode 100644 index cbbef01ba3..0000000000 --- a/newsfragments/1109.feature.rst +++ /dev/null @@ -1,3 +0,0 @@ -It turns out that creating a subprocess can block the parent process -for a surprisingly long time. So `trio.open_process` now uses a worker -thread to avoid blocking the event loop. diff --git a/newsfragments/1109.removal.rst b/newsfragments/1109.removal.rst deleted file mode 100644 index 65344f64b8..0000000000 --- a/newsfragments/1109.removal.rst +++ /dev/null @@ -1,3 +0,0 @@ -If you want to create a `trio.Process` object, you now have to call -`trio.open_process`; calling ``trio.Process()`` directly was -deprecated in v0.12.0 and has now been removed. diff --git a/newsfragments/1118.feature.rst b/newsfragments/1118.feature.rst deleted file mode 100644 index b54208af8a..0000000000 --- a/newsfragments/1118.feature.rst +++ /dev/null @@ -1 +0,0 @@ -We've added FreeBSD to the list of platforms we support and test on. diff --git a/newsfragments/1241.feature.rst b/newsfragments/1241.feature.rst deleted file mode 100644 index 8accb4235e..0000000000 --- a/newsfragments/1241.feature.rst +++ /dev/null @@ -1,4 +0,0 @@ -On Linux kernels v5.3 or newer, `trio.Process.wait` now uses `the -pidfd API `__ to track child -processes. This shouldn't have any user-visible change, but it makes -working with subprocesses faster and use less memory. diff --git a/newsfragments/1315.feature.rst b/newsfragments/1315.feature.rst deleted file mode 100644 index 824a793e74..0000000000 --- a/newsfragments/1315.feature.rst +++ /dev/null @@ -1,4 +0,0 @@ -The `trio.Process.returncode` attribute is now automatically updated -as needed, instead of only when you call `~trio.Process.poll` or -`~trio.Process.wait`. Also, ``repr(process_object)`` now always -contains up-to-date information about the process status. diff --git a/newsfragments/1498.removal.rst b/newsfragments/1498.removal.rst deleted file mode 100644 index fffb385ca1..0000000000 --- a/newsfragments/1498.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``clear`` method on `trio.Event`: it was deprecated in 0.12.0. diff --git a/newsfragments/1499.bugfix.rst b/newsfragments/1499.bugfix.rst deleted file mode 100644 index 2b6a61f603..0000000000 --- a/newsfragments/1499.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed an over-strict test that caused failures on Alpine Linux. -Started testing against Alpine in CI. diff --git a/newsfragments/1526.bugfix.rst b/newsfragments/1526.bugfix.rst deleted file mode 100644 index 02e1d37c4a..0000000000 --- a/newsfragments/1526.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Calling `open_signal_receiver` with no arguments used to succeed without listening for any signals. This was confusing, so now it raises TypeError instead. diff --git a/newsfragments/476.removal.rst b/newsfragments/476.removal.rst deleted file mode 100644 index c4f8331f8c..0000000000 --- a/newsfragments/476.removal.rst +++ /dev/null @@ -1,3 +0,0 @@ -It turns out that everyone got confused by the name ``trio.hazmat``. -So that name has been deprecated, and the new name is -:mod:`trio.lowlevel`. diff --git a/newsfragments/75.removal.rst b/newsfragments/75.removal.rst deleted file mode 100644 index b239384710..0000000000 --- a/newsfragments/75.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Remove support for Python 3.5. diff --git a/trio/_version.py b/trio/_version.py index 7aff89c31d..7529894457 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.14.0+dev" +__version__ = "0.15.0" From 183ec18aa7e0d537031116707aa33612e4481365 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 19 May 2020 09:22:53 +0400 Subject: [PATCH 0147/1498] Bump version to v0.15.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index 7529894457..4a06ac8f47 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.15.0" +__version__ = "0.15.0+dev" From 0520ffad09bfbe2bbb6f0d8092a95353dad4d477 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 19 May 2020 09:34:06 +0400 Subject: [PATCH 0148/1498] Simplify releasing notes --- docs/source/releasing.rst | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/docs/source/releasing.rst b/docs/source/releasing.rst index 0ebad58a80..98b53406f2 100644 --- a/docs/source/releasing.rst +++ b/docs/source/releasing.rst @@ -19,7 +19,7 @@ Things to do for releasing: * Do the actual release changeset - + update version number + + bump version number - increment as per Semantic Versioning rules @@ -37,31 +37,18 @@ Things to do for releasing: * create pull request to ``python-trio/trio``'s "master" branch -* announce PR on gitter - - + wait for feedback - - + fix problems, if any - * verify that all checks succeeded -* acknowledge the release PR - - + or rather, somebody else should do that - -* tag with vVERSION +* tag with vVERSION, push tag * push to PyPI + ``python3 setup.py sdist bdist_wheel upload`` -* announce on gitter - -* update version number +* update version number in the same pull request + add ``+dev`` tag to the end -* prepare another pull request to "master" - - + acknowledge it +* approve and merge the release pull request +* announce on gitter From 84a1a18d1361291164926edc9e6f44f99d2debb4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 19 May 2020 06:55:44 +0000 Subject: [PATCH 0149/1498] Bump packaging from 20.3 to 20.4 Bumps [packaging](https://github.com/pypa/packaging) from 20.3 to 20.4. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/20.3...20.4) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 1b4da0c00a..340141008e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -19,7 +19,7 @@ incremental==17.5.0 # via towncrier jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.0.1 # via -r docs-requirements.in -packaging==20.3 # via sphinx +packaging==20.4 # via sphinx pygments==2.6.1 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.1 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index d561b263c2..25b1d7d3e9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -24,7 +24,7 @@ lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint more-itertools==8.3.0 # via pytest outcome==1.0.1 # via -r test-requirements.in -packaging==20.3 # via pytest +packaging==20.4 # via pytest parso==0.7.0 # via jedi pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython From 5cec84eb5b4bcc66c8cc4855656f0acf92cb8aad Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 19 May 2020 14:04:19 +0400 Subject: [PATCH 0150/1498] docs: improve releasing docs There's no need to approve your own PR, and it's safer to use git clean and twine. --- docs/source/releasing.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/releasing.rst b/docs/source/releasing.rst index 98b53406f2..f39aaf274f 100644 --- a/docs/source/releasing.rst +++ b/docs/source/releasing.rst @@ -41,14 +41,16 @@ Things to do for releasing: * tag with vVERSION, push tag -* push to PyPI +* push to PyPI:: - + ``python3 setup.py sdist bdist_wheel upload`` + git clean -xdf # maybe run 'git clean -xdn' first to see what it will delete + python3 setup.py sdist bdist_wheel + twine upload dist/* * update version number in the same pull request + add ``+dev`` tag to the end -* approve and merge the release pull request +* merge the release pull request * announce on gitter From 0f809d0ccfdfb5a94a8ee0311e91737a54eff9d8 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 19 May 2020 17:17:53 -0700 Subject: [PATCH 0151/1498] Remove another extra blank line --- trio/_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trio/_util.py b/trio/_util.py index 5cc9737d22..06c3a29a19 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -114,7 +114,6 @@ def _return_value_looks_like_wrong_library(value): except TypeError: # Give good error for: nursery.start_soon(trio.sleep(1)) if isinstance(async_fn, collections.abc.Coroutine): - # explicitly close coroutine to avoid RuntimeWarning async_fn.close() From 7a71a851e3cc8209dc85100d3e5548e2ba7dd268 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 19 May 2020 17:18:50 -0700 Subject: [PATCH 0152/1498] Remove a couple more of the extra added blank lines --- trio/tests/test_threads.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 545986a795..6f5d2b6229 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -478,7 +478,6 @@ def thread_fn(): from_thread_run_sync(async_fn) with pytest.raises(TypeError, match="expected a sync function"): - await to_thread_run_sync(thread_fn) @@ -503,7 +502,6 @@ def sync_fn(): # pragma: no cover pass with pytest.raises(TypeError, match="appears to be synchronous"): - await to_thread_run_sync(from_thread_run, sync_fn) From 7de4c84188abf80c463f367504306292b584a7a3 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Tue, 19 May 2020 23:21:07 -0300 Subject: [PATCH 0153/1498] Added newsfragment --- newsfragments/1244.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1244.bugfix.rst diff --git a/newsfragments/1244.bugfix.rst b/newsfragments/1244.bugfix.rst new file mode 100644 index 0000000000..d38868f8d9 --- /dev/null +++ b/newsfragments/1244.bugfix.rst @@ -0,0 +1 @@ +Added helpful error message in the case the user passed incorrect type of function to "from_thread_run" & "from_thread_run_sync" functions. From f77c2009e5a5e7a1c0e692609316273bef89f931 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 20 May 2020 08:00:11 -0300 Subject: [PATCH 0154/1498] Update newsfragments/1244.bugfix.rst Co-authored-by: Joshua Oreman --- newsfragments/1244.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/1244.bugfix.rst b/newsfragments/1244.bugfix.rst index d38868f8d9..6245199a2b 100644 --- a/newsfragments/1244.bugfix.rst +++ b/newsfragments/1244.bugfix.rst @@ -1 +1 @@ -Added helpful error message in the case the user passed incorrect type of function to "from_thread_run" & "from_thread_run_sync" functions. +Added a helpful error message if an async function is passed to `trio.from_thread.run_sync` or a sync function to `trio.from_thread.run`. From 630ff02b4634ddf5bf959b5015625959c74aa0d4 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 09:29:33 +0400 Subject: [PATCH 0155/1498] Add new socket symbols to fix nightly build --- trio/socket.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/trio/socket.py b/trio/socket.py index fa51f6dbf1..5951b5b099 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -107,7 +107,15 @@ CAN_BCM_TX_RESET_MULTI_IDX, IPPROTO_CBT, IPPROTO_ICLFXBM, IPPROTO_IGP, IPPROTO_L2TP, IPPROTO_PGM, IPPROTO_RDP, IPPROTO_ST, AF_QIPCRTR, CAN_BCM_CAN_FD_FRAME, IPPROTO_MOBILE, IPV6_USE_MIN_MTU, - MSG_NOTIFICATION, SO_SETFIB + MSG_NOTIFICATION, SO_SETFIB, CAN_J1939, CAN_RAW_JOIN_FILTERS, + IPPROTO_UDPLITE, J1939_EE_INFO_NONE, J1939_EE_INFO_TX_ABORT, + J1939_FILTER_MAX, J1939_IDLE_ADDR, J1939_MAX_UNICAST_ADDR, + J1939_NLA_BYTES_ACKED, J1939_NLA_PAD, J1939_NO_ADDR, J1939_NO_NAME, + J1939_NO_PGN, J1939_PGN_ADDRESS_CLAIMED, J1939_PGN_ADDRESS_COMMANDED, + J1939_PGN_MAX, J1939_PGN_PDU1_MAX, J1939_PGN_REQUEST, + SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, + SCM_J1939_PRIO, SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, + SO_J1939_SEND_PRIO, UDPLITE_RECV_CSCOV, UDPLITE_SEND_CSCOV ) except ImportError: pass From 5fda1f5c3520e0838d77f50d921aafcb56050380 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 22 May 2020 06:20:09 +0000 Subject: [PATCH 0156/1498] Bump six from 1.14.0 to 1.15.0 Bumps [six](https://github.com/benjaminp/six) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/benjaminp/six/releases) - [Changelog](https://github.com/benjaminp/six/blob/master/CHANGES) - [Commits](https://github.com/benjaminp/six/compare/1.14.0...1.15.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 340141008e..0811d9121c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -24,7 +24,7 @@ pygments==2.6.1 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.1 # via babel requests==2.23.0 # via sphinx -six==1.14.0 # via packaging +six==1.15.0 # via packaging sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.1.0 # via -r docs-requirements.in diff --git a/test-requirements.txt b/test-requirements.txt index 25b1d7d3e9..f32a41c163 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in pytest==5.4.2 # via -r test-requirements.in, pytest-cov -six==1.14.0 # via astroid, cryptography, packaging, pyopenssl, traitlets +six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in toml==0.10.1 # via pylint From a3bc1f90077f4b80f3893438b447645690262ec2 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 22 May 2020 00:55:19 -0700 Subject: [PATCH 0157/1498] Docs: suppress epub warnings for unknown file types (#1541) This looks like it will fix our RTD build. --- docs/source/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 49afad6cfd..9dddc93400 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -145,6 +145,10 @@ def setup(app): # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False +# This avoids a warning by the epub builder that it can't figure out +# the MIME type for our favicon. +suppress_warnings = ["epub.unknown_project_files"] + # -- Options for HTML output ---------------------------------------------- From 9f4b571141c15a41860e09570865fd7a041a4e5d Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 22 May 2020 08:03:00 +0000 Subject: [PATCH 0158/1498] Bump version and run towncrier for 0.15.1 release --- docs/source/history.rst | 12 ++++++++++++ newsfragments/1244.bugfix.rst | 1 - trio/_version.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) delete mode 100644 newsfragments/1244.bugfix.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 133b615d85..d6967217c0 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,18 @@ Release history .. towncrier release notes start +Trio 0.15.1 (2020-05-22) +------------------------ + +Bugfixes +~~~~~~~~ + +- Fix documentation build. (This must be a new release tag to get readthedocs + "stable" to include the changes from 0.15.0.) + +- Added a helpful error message if an async function is passed to `trio.from_thread.run_sync` or a sync function to `trio.from_thread.run`. (`#1244 `__) + + Trio 0.15.0 (2020-05-19) ------------------------ diff --git a/newsfragments/1244.bugfix.rst b/newsfragments/1244.bugfix.rst deleted file mode 100644 index 6245199a2b..0000000000 --- a/newsfragments/1244.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Added a helpful error message if an async function is passed to `trio.from_thread.run_sync` or a sync function to `trio.from_thread.run`. diff --git a/trio/_version.py b/trio/_version.py index 4a06ac8f47..f137b94817 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.15.0+dev" +__version__ = "0.15.1" From 9916aa8fa6aa4013e4117559e48f3377d3318cdd Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 22 May 2020 08:27:26 +0000 Subject: [PATCH 0159/1498] Bump version to 0.15.1+dev post release --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index f137b94817..4d8550b19f 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.15.1" +__version__ = "0.15.1+dev" From 4294673674b96a8d73a8d2e9a753d36714b7c6cf Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 10:33:04 +0400 Subject: [PATCH 0160/1498] Prepare switch to black --- .style.yapf | 185 ----------------------- check.sh | 4 +- docs/source/contributing.rst | 25 ++- pyproject.toml | 4 + test-requirements.in | 2 +- test-requirements.txt | 17 ++- trio/_core/tests/test_run.py | 2 +- trio/socket.py | 2 + trio/tests/test_deprecate.py | 6 +- trio/tests/test_highlevel_ssl_helpers.py | 2 +- trio/tests/test_socket.py | 6 +- 11 files changed, 41 insertions(+), 214 deletions(-) delete mode 100644 .style.yapf diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 745ec88d70..0000000000 --- a/.style.yapf +++ /dev/null @@ -1,185 +0,0 @@ -[style] -# Align closing bracket with visual indentation. -align_closing_bracket_with_visual_indent=True - -# Allow dictionary keys to exist on multiple lines. For example: -# -# x = { -# ('this is the first element of a tuple', -# 'this is the second element of a tuple'): -# value, -# } -allow_multiline_dictionary_keys=False - -# Allow lambdas to be formatted on more than one line. -allow_multiline_lambdas=False - -# Insert a blank line before a class-level docstring. -blank_line_before_class_docstring=False - -# Insert a blank line before a 'def' or 'class' immediately nested -# within another 'def' or 'class'. For example: -# -# class Foo: -# # <------ this blank line -# def method(): -# ... -blank_line_before_nested_class_or_def=False - -# Do not split consecutive brackets. Only relevant when -# dedent_closing_brackets is set. For example: -# -# call_func_that_takes_a_dict( -# { -# 'key1': 'value1', -# 'key2': 'value2', -# } -# ) -# -# would reformat to: -# -# call_func_that_takes_a_dict({ -# 'key1': 'value1', -# 'key2': 'value2', -# }) -coalesce_brackets=False - -# The column limit. -column_limit=79 - -# Indent width used for line continuations. -continuation_indent_width=4 - -# Put closing brackets on a separate line, dedented, if the bracketed -# expression can't fit in a single line. Applies to all kinds of brackets, -# including function definitions and calls. For example: -# -# config = { -# 'key1': 'value1', -# 'key2': 'value2', -# } # <--- this bracket is dedented and on a separate line -# -# time_series = self.remote_client.query_entity_counters( -# entity='dev3246.region1', -# key='dns.query_latency_tcp', -# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), -# start_ts=now()-timedelta(days=3), -# end_ts=now(), -# ) # <--- this bracket is dedented and on a separate line -dedent_closing_brackets=True - -# Place each dictionary entry onto its own line. -each_dict_entry_on_separate_line=True - -# The regex for an i18n comment. The presence of this comment stops -# reformatting of that line, because the comments are required to be -# next to the string they translate. -i18n_comment= - -# The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved -# away from the i18n comment. -i18n_function_call= - -# Indent the dictionary value if it cannot fit on the same line as the -# dictionary key. For example: -# -# config = { -# 'key1': -# 'value1', -# 'key2': value1 + -# value2, -# } -indent_dictionary_value=True - -# The number of columns to use for indentation. -indent_width=4 - -# Join short lines into one line. E.g., single line 'if' statements. -join_multiple_lines=False - -# Use spaces around default or named assigns. -spaces_around_default_or_named_assign=False - -# Use spaces around the power operator. -spaces_around_power_operator=False - -# The number of spaces required before a trailing comment. -spaces_before_comment=2 - -# Insert a space between the ending comma and closing bracket of a list, -# etc. -space_between_ending_comma_and_closing_bracket=False - -# Split before arguments if the argument list is terminated by a -# comma. -split_arguments_when_comma_terminated=True - -# Set to True to prefer splitting before '&', '|' or '^' rather than -# after. -split_before_bitwise_operator=True - -# Split before a dictionary or set generator (comp_for). For example, note -# the split before the 'for': -# -# foo = { -# variable: 'Hello world, have a nice day!' -# for variable in bar if variable != 42 -# } -split_before_dict_set_generator=True - -# If an argument / parameter list is going to be split, then split before -# the first argument. -split_before_first_argument=False - -# Set to True to prefer splitting before 'and' or 'or' rather than -# after. -split_before_logical_operator=True - -# Split named assignments onto individual lines. -split_before_named_assigns=True - -# The penalty for splitting right after the opening bracket. -split_penalty_after_opening_bracket=30 - -# The penalty for splitting the line after a unary operator. -split_penalty_after_unary_operator=10000 - -# The penalty for splitting right before an if expression. -split_penalty_before_if_expr=0 - -# The penalty of splitting the line around the '&', '|', and '^' -# operators. -split_penalty_bitwise_operator=300 - -# The penalty for characters over the column limit. -split_penalty_excess_character=4500 - -# The penalty incurred by adding a line split to the unwrapped line. The -# more line splits added the higher the penalty. -split_penalty_for_added_line_split=30 - -# The penalty of splitting a list of "import as" names. For example: -# -# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, -# long_argument_2, -# long_argument_3) -# -# would reformat to something like: -# -# from a_very_long_or_indented_module_name_yada_yad import ( -# long_argument_1, long_argument_2, long_argument_3) -split_penalty_import_names=0 - -# The penalty of splitting the line around the 'and' and 'or' -# operators. -split_penalty_logical_operator=0 - -# Use the Tab character for indentation. -use_tabs=False - -# Without this, yapf likes to write things like -# "foo bar {}". -# format(...) -# which is just awful. -split_before_dot=True diff --git a/check.sh b/check.sh index 0b66ca227b..16394ccea1 100755 --- a/check.sh +++ b/check.sh @@ -13,7 +13,7 @@ python ./trio/_tools/gen_exports.py --test \ # see https://forum.bors.tech/t/pre-test-and-pre-merge-hooks/322) # autoflake --recursive --in-place . # pyupgrade --py3-plus $(find . -name "*.py") -yapf -rpd setup.py trio \ +black --diff setup.py trio \ || EXIT_STATUS=$? # Run flake8 without pycodestyle and import-related errors @@ -31,7 +31,7 @@ Problems were found by static analysis (listed above). To fix formatting and see remaining errors, run pip install -r test-requirements.txt - yapf -rpi setup.py trio + black setup.py trio ./check.sh in your local checkout. diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 8c6a734100..d1723c7fc5 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -133,7 +133,7 @@ in separate sections below: adding a test to make sure it stays fixed. * :ref:`pull-request-formatting`: If you changed Python code, then did - you run ``yapf -rpi setup.py trio``? (Or for other packages, replace + you run ``black setup.py trio``? (Or for other packages, replace ``trio`` with the package name.) * :ref:`pull-request-release-notes`: If your change affects @@ -285,31 +285,30 @@ of eyes can be helpful when trying to come up with devious tricks. Code formatting ~~~~~~~~~~~~~~~ -Instead of wasting time arguing about code formatting, we use `yapf -`__ to automatically format all our +Instead of wasting time arguing about code formatting, we use `black +`__ to automatically format all our code to a standard style. While you're editing code you can be as sloppy as you like about whitespace; and then before you commit, just run:: - pip install -U yapf - yapf -rpi setup.py trio + pip install -U black + black setup.py trio to fix it up. (And don't worry if you forget – when you submit a pull request then we'll automatically check and remind you.) Hopefully this will let you focus on more important style issues like choosing good names, writing useful comments, and making sure your docstrings are -nicely formatted. (Yapf doesn't reformat comments or docstrings.) +nicely formatted. (black doesn't reformat comments or docstrings.) -Very occasionally, yapf will generate really ugly and unreadable -formatting (usually for large literal structures like dicts nested -inside dicts). In these cases, you can add a ``# yapf: disable`` -comment to tell it to leave that particular statement alone. +Very occasionally, you'll want to override black formatting. To do so, +you can can add ``# fmt: off`` and ``# fmt: on`` comments. -If you want to see what changes yapf will make, you can use:: +If you want to see what changes black will make, you can use:: - yapf -rpd setup.py trio + black --diff setup.py trio -(``-d`` displays a diff, versus ``-i`` which fixes files in-place.) +(``--diff`` displays a diff, versus the default mode which fixes files +in-place.) .. _pull-request-release-notes: diff --git a/pyproject.toml b/pyproject.toml index 768a4766eb..e16fa5c401 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[tool.black] +target-version = ['py36'] + + [tool.towncrier] # Usage: # - PRs should drop a file like "issuenumber.feature" in newsfragments diff --git a/test-requirements.in b/test-requirements.in index f03409d484..f393bf2f47 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -8,7 +8,7 @@ pylint # for pylint finding all symbols tests jedi # for jedi code completion tests # Tools -yapf ==0.30.0 # formatting +black # formatting flake8 astor # code generation diff --git a/test-requirements.txt b/test-requirements.txt index f32a41c163..8d2b0fe829 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,18 +4,24 @@ # # pip-compile --output-file test-requirements.txt test-requirements.in # +appdirs==1.4.4 # via black +appnope==0.1.0 # via ipython astor==0.8.1 # via -r test-requirements.in astroid==2.4.1 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==19.3.0 # via -r test-requirements.in, outcome, pytest +attrs==19.3.0 # via -r test-requirements.in, black, outcome, pytest backcall==0.1.0 # via ipython +black==19.10b0 # via -r test-requirements.in cffi==1.14.0 # via cryptography +click==7.1.2 # via black +contextvars==2.4 ; python_version < "3.7" # via -r test-requirements.in, sniffio coverage==5.1 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.1 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme -immutables==0.14 # via -r test-requirements.in +immutables==0.14 # via -r test-requirements.in, contextvars +importlib-metadata==1.6.0 # via flake8, pluggy, pytest ipython-genutils==0.2.0 # via traitlets ipython==7.14.0 # via -r test-requirements.in isort==4.3.21 # via pylint @@ -26,6 +32,7 @@ more-itertools==8.3.0 # via pytest outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest parso==0.7.0 # via jedi +pathspec==0.8.0 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest @@ -41,12 +48,14 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.8.1 # via -r test-requirements.in pytest==5.4.2 # via -r test-requirements.in, pytest-cov +regex==2020.5.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.1.0 # via -r test-requirements.in -toml==0.10.1 # via pylint +toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in +typed-ast==1.4.1 ; python_version < "3.8" and implementation_name == "cpython" # via -r test-requirements.in, astroid, black wcwidth==0.1.9 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid -yapf==0.30.0 # via -r test-requirements.in +zipp==3.1.0 # via importlib-metadata diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 53446a601f..216fd5b050 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -441,7 +441,7 @@ async def main(): ("after", tasks["t2"]) }, ("after_run",), - ] # yapf: disable + ] print(list(r.filter_tasks(tasks.values()))) check_sequence_matches(list(r.filter_tasks(tasks.values())), expected) diff --git a/trio/socket.py b/trio/socket.py index 5951b5b099..8b87ea18b5 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -20,6 +20,7 @@ # going on. There's a test in test_exports.py to make sure that the list is # kept up to date. try: + # fmt: off from socket import ( CMSG_LEN, CMSG_SPACE, CAPI, AF_UNSPEC, AF_INET, AF_UNIX, AF_IPX, AF_APPLETALK, AF_INET6, AF_ROUTE, AF_LINK, AF_SNA, PF_SYSTEM, @@ -117,6 +118,7 @@ SCM_J1939_PRIO, SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, SO_J1939_SEND_PRIO, UDPLITE_RECV_CSCOV, UDPLITE_SEND_CSCOV ) + # fmt: on except ImportError: pass diff --git a/trio/tests/test_deprecate.py b/trio/tests/test_deprecate.py index 6ecd00003e..8497f911c7 100644 --- a/trio/tests/test_deprecate.py +++ b/trio/tests/test_deprecate.py @@ -25,8 +25,8 @@ def test_warn_deprecated(recwarn_always): def deprecated_thing(): warn_deprecated("ice", "1.2", issue=1, instead="water") - filename, lineno = _here() # https://github.com/google/yapf/issues/447 deprecated_thing() + filename, lineno = _here() assert len(recwarn_always) == 1 got = recwarn_always.pop(TrioDeprecationWarning) assert "ice is deprecated" in got.message.args[0] @@ -54,7 +54,7 @@ def nested1(): def nested2(): warn_deprecated("x", "1.3", issue=7, instead="y", stacklevel=3) - filename, lineno = _here() # https://github.com/google/yapf/issues/447 + filename, lineno = _here() nested1() got = recwarn_always.pop(TrioDeprecationWarning) assert got.filename == filename @@ -214,7 +214,7 @@ def test_module_with_deprecations(recwarn_always): assert module_with_deprecations.regular == "hi" assert len(recwarn_always) == 0 - filename, lineno = _here() # https://github.com/google/yapf/issues/447 + filename, lineno = _here() assert module_with_deprecations.dep1 == "value1" got = recwarn_always.pop(TrioDeprecationWarning) assert got.filename == filename diff --git a/trio/tests/test_highlevel_ssl_helpers.py b/trio/tests/test_highlevel_ssl_helpers.py index 1583f4cd54..d18cb84347 100644 --- a/trio/tests/test_highlevel_ssl_helpers.py +++ b/trio/tests/test_highlevel_ssl_helpers.py @@ -105,7 +105,7 @@ async def test_open_ssl_over_tcp_stream_and_everything_else( async def test_open_ssl_over_tcp_listeners(): (listener,) = await open_ssl_over_tcp_listeners( 0, SERVER_CTX, host="127.0.0.1" - ) # yapf: disable + ) async with listener: assert isinstance(listener, trio.SSLListener) tl = listener.transport_listener diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 4e76711d34..f384ca4297 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -127,7 +127,7 @@ def filtered(gai_list): tsocket.IPPROTO_TCP, "", ("127.0.0.1", 12345)), - ]) # yapf: disable + ]) with assert_checkpoints(): res = await tsocket.getaddrinfo( @@ -139,7 +139,7 @@ def filtered(gai_list): tsocket.IPPROTO_UDP, "", ("::1", 12345, 0, 0)), - ]) # yapf: disable + ]) monkeygai.set("x", b"host", "port", family=0, type=0, proto=0, flags=0) with assert_checkpoints(): @@ -492,7 +492,6 @@ async def test_SocketType_resolve(socket_type, addrs): async def res(*args): return await getattr(sock, resolver)(*args) - # yapf: disable assert await res((addrs.arbitrary, "http")) == (addrs.arbitrary, 80, *addrs.extra) if v6: @@ -507,7 +506,6 @@ async def res(*args): # Check the special case, because why not assert await res(("", 123)) == (addrs.broadcast, 123, *addrs.extra) - # yapf: enable # But not if it's true (at least on systems where getaddrinfo works # correctly) From 9a8268b0a5d66453139200e17de44a7b4ff6a61f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 10:33:26 +0400 Subject: [PATCH 0161/1498] Generate export files close to black Unfortunately, full black compliance is not possible, as a few lines are too long. This is why I added the `fmt` pragmas. --- trio/_core/_generated_io_epoll.py | 16 ++++++--- trio/_core/_generated_io_kqueue.py | 29 ++++++++++------ trio/_core/_generated_io_windows.py | 40 ++++++++++++++-------- trio/_core/_generated_run.py | 53 ++++++++++++++++++----------- trio/_tools/gen_exports.py | 14 +++++--- 5 files changed, 98 insertions(+), 54 deletions(-) diff --git a/trio/_core/_generated_io_epoll.py b/trio/_core/_generated_io_epoll.py index fe63a6ee0c..9583c7ff4f 100644 --- a/trio/_core/_generated_io_epoll.py +++ b/trio/_core/_generated_io_epoll.py @@ -4,25 +4,31 @@ from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED - +# fmt: off + async def wait_readable(fd): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_readable(fd) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def wait_writable(fd): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_writable(fd) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def notify_closing(fd): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.notify_closing(fd) + return GLOBAL_RUN_CONTEXT.runner.io_manager.notify_closing(fd) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + + +# fmt: on diff --git a/trio/_core/_generated_io_kqueue.py b/trio/_core/_generated_io_kqueue.py index 059a8a95d1..ab95d1e30c 100644 --- a/trio/_core/_generated_io_kqueue.py +++ b/trio/_core/_generated_io_kqueue.py @@ -4,46 +4,55 @@ from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED - +# fmt: off + def current_kqueue(): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.current_kqueue() + return GLOBAL_RUN_CONTEXT.runner.io_manager.current_kqueue() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def monitor_kevent(ident, filter): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.monitor_kevent(ident, filter) + return GLOBAL_RUN_CONTEXT.runner.io_manager.monitor_kevent(ident, filter) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def wait_kevent(ident, filter, abort_func): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_kevent(ident, filter, abort_func) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def wait_readable(fd): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_readable(fd) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def wait_writable(fd): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_writable(fd) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def notify_closing(fd): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.notify_closing(fd) + return GLOBAL_RUN_CONTEXT.runner.io_manager.notify_closing(fd) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + + +# fmt: on diff --git a/trio/_core/_generated_io_windows.py b/trio/_core/_generated_io_windows.py index 78dd30db19..d6a5760374 100644 --- a/trio/_core/_generated_io_windows.py +++ b/trio/_core/_generated_io_windows.py @@ -4,67 +4,79 @@ from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED - +# fmt: off + async def wait_readable(sock): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_readable(sock) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def wait_writable(sock): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_writable(sock) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def notify_closing(handle): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.notify_closing(handle) + return GLOBAL_RUN_CONTEXT.runner.io_manager.notify_closing(handle) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def register_with_iocp(handle): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.register_with_iocp(handle) + return GLOBAL_RUN_CONTEXT.runner.io_manager.register_with_iocp(handle) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def wait_overlapped(handle, lpOverlapped): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.wait_overlapped(handle, lpOverlapped) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def write_overlapped(handle, data, file_offset=0): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.write_overlapped(handle, data, file_offset) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def readinto_overlapped(handle, buffer, file_offset=0): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return await GLOBAL_RUN_CONTEXT.runner.io_manager.readinto_overlapped(handle, buffer, file_offset) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def current_iocp(): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.current_iocp() + return GLOBAL_RUN_CONTEXT.runner.io_manager.current_iocp() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def monitor_completion_key(): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.io_manager.monitor_completion_key() + return GLOBAL_RUN_CONTEXT.runner.io_manager.monitor_completion_key() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + + +# fmt: on diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index 75f61bfdc5..edf46fd741 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -4,7 +4,8 @@ from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED - +# fmt: off + def current_statistics(): """Returns an object containing run-loop-level debugging information. @@ -31,9 +32,10 @@ def current_statistics(): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.current_statistics() + return GLOBAL_RUN_CONTEXT.runner.current_statistics() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def current_time(): """Returns the current time according to Trio's internal clock. @@ -47,9 +49,10 @@ def current_time(): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.current_time() + return GLOBAL_RUN_CONTEXT.runner.current_time() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def current_clock(): """Returns the current :class:`~trio.abc.Clock`. @@ -57,9 +60,10 @@ def current_clock(): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.current_clock() + return GLOBAL_RUN_CONTEXT.runner.current_clock() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def current_root_task(): """Returns the current root :class:`Task`. @@ -69,9 +73,10 @@ def current_root_task(): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.current_root_task() + return GLOBAL_RUN_CONTEXT.runner.current_root_task() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def reschedule(task, next_send=_NO_SEND): """Reschedule the given task with the given @@ -93,9 +98,10 @@ def reschedule(task, next_send=_NO_SEND): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.reschedule(task, next_send) + return GLOBAL_RUN_CONTEXT.runner.reschedule(task, next_send) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def spawn_system_task(async_fn, *args, name=None): """Spawn a "system" task. @@ -138,9 +144,10 @@ def spawn_system_task(async_fn, *args, name=None): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.spawn_system_task(async_fn, *args, name=name) + return GLOBAL_RUN_CONTEXT.runner.spawn_system_task(async_fn, *args, name=name) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def current_trio_token(): """Retrieve the :class:`TrioToken` for the current call to @@ -149,9 +156,10 @@ def current_trio_token(): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.current_trio_token() + return GLOBAL_RUN_CONTEXT.runner.current_trio_token() except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + async def wait_all_tasks_blocked(cushion=0.0, tiebreaker=0): """Block until there are no runnable tasks. @@ -217,7 +225,8 @@ async def test_lock_fairness(): try: return await GLOBAL_RUN_CONTEXT.runner.wait_all_tasks_blocked(cushion, tiebreaker) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def add_instrument(instrument): """Start instrumenting the current run loop with the given instrument. @@ -230,9 +239,10 @@ def add_instrument(instrument): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.add_instrument(instrument) + return GLOBAL_RUN_CONTEXT.runner.add_instrument(instrument) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + def remove_instrument(instrument): """Stop instrumenting the current run loop with the given instrument. @@ -249,6 +259,9 @@ def remove_instrument(instrument): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.remove_instrument(instrument) + return GLOBAL_RUN_CONTEXT.runner.remove_instrument(instrument) except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") + + +# fmt: on diff --git a/trio/_tools/gen_exports.py b/trio/_tools/gen_exports.py index 1340d049a2..9dda121b51 100755 --- a/trio/_tools/gen_exports.py +++ b/trio/_tools/gen_exports.py @@ -21,14 +21,17 @@ from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED - +# fmt: off +""" + +FOOTER = """# fmt: on """ TEMPLATE = """locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return {} GLOBAL_RUN_CONTEXT.{}.{} + return{}GLOBAL_RUN_CONTEXT.{}.{} except AttributeError: - raise RuntimeError('must be called from async context') + raise RuntimeError("must be called from async context") """ @@ -116,7 +119,7 @@ def gen_public_wrappers_source(source_path: Path, lookup_path: str) -> str: # Create export function body template = TEMPLATE.format( - 'await' if isinstance(method, ast.AsyncFunctionDef) else '', + " await " if isinstance(method, ast.AsyncFunctionDef) else " ", lookup_path, method.name + new_args, ) @@ -126,7 +129,8 @@ def gen_public_wrappers_source(source_path: Path, lookup_path: str) -> str: # Append the snippet to the corresponding module generated.append(snippet) - return "\n".join(generated) + generated.append(FOOTER) + return "\n\n".join(generated) def matches_disk_files(new_files): From 7545a976cb678798c7a2ff0bcc3ed6aa80ffff64 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 11:21:02 +0400 Subject: [PATCH 0162/1498] Install black on CPython only It depends on typed-ast, which isn't supported on PyPy. --- test-requirements.in | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index f393bf2f47..987567acc5 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -8,7 +8,7 @@ pylint # for pylint finding all symbols tests jedi # for jedi code completion tests # Tools -black # formatting +black; implementation_name == "cpython" flake8 astor # code generation diff --git a/test-requirements.txt b/test-requirements.txt index 8d2b0fe829..cb75bfb8ac 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ astroid==2.4.1 # via pylint async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, black, outcome, pytest backcall==0.1.0 # via ipython -black==19.10b0 # via -r test-requirements.in +black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.0 # via cryptography click==7.1.2 # via black contextvars==2.4 ; python_version < "3.7" # via -r test-requirements.in, sniffio From 6fa003e09644241bd1e9737934f332e9a5dadd74 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 10:42:59 +0400 Subject: [PATCH 0163/1498] Switch to black --- setup.py | 2 +- trio/__init__.py | 157 +++++++++--------- trio/_abc.py | 16 +- trio/_channel.py | 13 +- trio/_core/__init__.py | 62 +++++-- trio/_core/_entry_queue.py | 6 +- trio/_core/_exceptions.py | 1 + trio/_core/_io_epoll.py | 6 +- trio/_core/_io_kqueue.py | 15 +- trio/_core/_io_windows.py | 36 ++-- trio/_core/_ki.py | 18 +- trio/_core/_local.py | 5 +- trio/_core/_multierror.py | 27 +-- trio/_core/_run.py | 79 ++++----- trio/_core/_traps.py | 1 + trio/_core/_unbounded_queue.py | 6 +- trio/_core/_wakeup_socketpair.py | 6 +- trio/_core/tests/test_io.py | 12 +- trio/_core/tests/test_ki.py | 26 +-- trio/_core/tests/test_multierror.py | 46 ++--- .../apport_excepthook.py | 2 + .../ipython_custom_exc.py | 1 + trio/_core/tests/test_parking_lot.py | 33 ++-- trio/_core/tests/test_run.py | 97 ++++++----- trio/_core/tests/test_windows.py | 26 ++- trio/_core/tests/tutil.py | 23 +-- trio/_deprecate.py | 4 +- trio/_deprecated_ssl_reexports.py | 134 +++++++++++---- trio/_deprecated_subprocess_reexports.py | 35 +++- trio/_file_io.py | 81 +++++---- trio/_highlevel_generic.py | 5 +- trio/_highlevel_open_tcp_listeners.py | 24 +-- trio/_highlevel_open_tcp_stream.py | 2 +- trio/_highlevel_open_unix_stream.py | 1 + trio/_highlevel_serve_listeners.py | 12 +- trio/_highlevel_socket.py | 26 +-- trio/_highlevel_ssl_helpers.py | 26 +-- trio/_path.py | 28 ++-- trio/_socket.py | 43 ++--- trio/_ssl.py | 51 +++--- trio/_subprocess.py | 28 ++-- trio/_subprocess_platform/kqueue.py | 9 +- trio/_subprocess_platform/waitid.py | 9 +- trio/_sync.py | 36 ++-- trio/_threads.py | 18 +- trio/_timeouts.py | 4 +- trio/_tools/gen_exports.py | 22 +-- trio/_unix_pipes.py | 5 +- trio/_util.py | 17 +- trio/_wait_for_object.py | 14 +- trio/_windows_pipes.py | 6 +- trio/abc.py | 16 +- trio/lowlevel.py | 36 +++- trio/socket.py | 19 ++- trio/testing/__init__.py | 15 +- trio/testing/_check_streams.py | 12 +- trio/testing/_checkpoints.py | 14 +- trio/testing/_memory_streams.py | 12 +- trio/testing/_mock_clock.py | 11 +- trio/testing/_sequencer.py | 8 +- trio/testing/_trio_test.py | 4 +- trio/tests/module_with_deprecations.py | 18 +- trio/tests/test_deprecate.py | 25 ++- trio/tests/test_exports.py | 9 +- trio/tests/test_file_io.py | 26 ++- .../test_highlevel_open_tcp_listeners.py | 34 +--- trio/tests/test_highlevel_open_tcp_stream.py | 11 +- trio/tests/test_highlevel_open_unix_stream.py | 6 +- trio/tests/test_highlevel_serve_listeners.py | 5 +- trio/tests/test_highlevel_socket.py | 4 +- trio/tests/test_highlevel_ssl_helpers.py | 28 +--- trio/tests/test_path.py | 109 ++++++------ trio/tests/test_scheduler_determinism.py | 4 +- trio/tests/test_signals.py | 5 +- trio/tests/test_socket.py | 98 ++++++----- trio/tests/test_ssl.py | 77 +++------ trio/tests/test_subprocess.py | 25 +-- trio/tests/test_sync.py | 7 +- trio/tests/test_testing.py | 20 +-- trio/tests/test_threads.py | 35 ++-- trio/tests/test_unix_pipes.py | 12 +- trio/tests/test_util.py | 13 +- trio/tests/test_wait_for_object.py | 44 ++--- trio/tests/test_windows_pipes.py | 2 +- trio/tests/tools/test_gen_exports.py | 6 +- 85 files changed, 1049 insertions(+), 1083 deletions(-) diff --git a/setup.py b/setup.py index 852fa616b2..f76c36e378 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ # cffi 1.14 fixes memory leak inside ffi.getwinerror() # cffi is required on Windows, except on PyPy where it is built-in "cffi>=1.14; os_name == 'nt' and implementation_name != 'pypy'", - "contextvars>=2.1; python_version < '3.7'" + "contextvars>=2.1; python_version < '3.7'", ], # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: diff --git a/trio/__init__.py b/trio/__init__.py index 30e42da97e..5339d107eb 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -16,19 +16,42 @@ from ._version import __version__ from ._core import ( - TrioInternalError, RunFinishedError, WouldBlock, Cancelled, - BusyResourceError, ClosedResourceError, MultiError, run, open_nursery, - CancelScope, current_effective_deadline, TASK_STATUS_IGNORED, current_time, - BrokenResourceError, EndOfChannel, Nursery + TrioInternalError, + RunFinishedError, + WouldBlock, + Cancelled, + BusyResourceError, + ClosedResourceError, + MultiError, + run, + open_nursery, + CancelScope, + current_effective_deadline, + TASK_STATUS_IGNORED, + current_time, + BrokenResourceError, + EndOfChannel, + Nursery, ) from ._timeouts import ( - move_on_at, move_on_after, sleep_forever, sleep_until, sleep, fail_at, - fail_after, TooSlowError + move_on_at, + move_on_after, + sleep_forever, + sleep_until, + sleep, + fail_at, + fail_after, + TooSlowError, ) from ._sync import ( - Event, CapacityLimiter, Semaphore, Lock, StrictFIFOLock, Condition + Event, + CapacityLimiter, + Semaphore, + Lock, + StrictFIFOLock, + Condition, ) from ._threads import BlockingTrioPortal as _BlockingTrioPortal @@ -36,7 +59,9 @@ from ._highlevel_generic import aclose_forcefully, StapledStream from ._channel import ( - open_memory_channel, MemorySendChannel, MemoryReceiveChannel + open_memory_channel, + MemorySendChannel, + MemoryReceiveChannel, ) from ._signals import open_signal_receiver @@ -60,7 +85,9 @@ from ._highlevel_open_unix_stream import open_unix_socket from ._highlevel_ssl_helpers import ( - open_ssl_over_tcp_stream, open_ssl_over_tcp_listeners, serve_ssl_over_tcp + open_ssl_over_tcp_stream, + open_ssl_over_tcp_listeners, + serve_ssl_over_tcp, ) from ._deprecate import TrioDeprecationWarning @@ -71,6 +98,7 @@ from . import abc from . import from_thread from . import to_thread + # Not imported by default, but mentioned here so static analysis tools like # pylint will know that it exists. if False: @@ -81,85 +109,57 @@ _deprecate.enable_attribute_deprecations(__name__) __deprecated_attributes__ = { - "ssl": - _deprecate.DeprecatedAttribute( - _deprecated_ssl_reexports, - "0.11.0", - issue=852, - instead=( - "trio.SSLStream, trio.SSLListener, trio.NeedHandshakeError, " - "and the standard library 'ssl' module (minus SSLSocket and " - "wrap_socket())" - ), - ), - "subprocess": - _deprecate.DeprecatedAttribute( - _deprecated_subprocess_reexports, - "0.11.0", - issue=852, - instead=( - "trio.Process and the constants in the standard " - "library 'subprocess' module" - ), - ), - "run_sync_in_worker_thread": - _deprecate.DeprecatedAttribute( - to_thread.run_sync, - "0.12.0", - issue=810, + "ssl": _deprecate.DeprecatedAttribute( + _deprecated_ssl_reexports, + "0.11.0", + issue=852, + instead=( + "trio.SSLStream, trio.SSLListener, trio.NeedHandshakeError, " + "and the standard library 'ssl' module (minus SSLSocket and " + "wrap_socket())" ), - "current_default_worker_thread_limiter": - _deprecate.DeprecatedAttribute( - to_thread.current_default_thread_limiter, - "0.12.0", - issue=810, - ), - "BlockingTrioPortal": - _deprecate.DeprecatedAttribute( - _BlockingTrioPortal, - "0.12.0", - issue=810, - instead=from_thread, + ), + "subprocess": _deprecate.DeprecatedAttribute( + _deprecated_subprocess_reexports, + "0.11.0", + issue=852, + instead=( + "trio.Process and the constants in the standard " + "library 'subprocess' module" ), + ), + "run_sync_in_worker_thread": _deprecate.DeprecatedAttribute( + to_thread.run_sync, "0.12.0", issue=810, + ), + "current_default_worker_thread_limiter": _deprecate.DeprecatedAttribute( + to_thread.current_default_thread_limiter, "0.12.0", issue=810, + ), + "BlockingTrioPortal": _deprecate.DeprecatedAttribute( + _BlockingTrioPortal, "0.12.0", issue=810, instead=from_thread, + ), # NOTE: when you remove this, you should also remove the file # trio/hazmat.py. For details on why we have both, see: # # https://github.com/python-trio/trio/pull/1484#issuecomment-622574499 - "hazmat": - _deprecate.DeprecatedAttribute( - lowlevel, - "0.15.0", - issue=476, - instead="trio.lowlevel", - ), + "hazmat": _deprecate.DeprecatedAttribute( + lowlevel, "0.15.0", issue=476, instead="trio.lowlevel", + ), } _deprecate.enable_attribute_deprecations(lowlevel.__name__) lowlevel.__deprecated_attributes__ = { - "wait_socket_readable": - _deprecate.DeprecatedAttribute( - lowlevel.wait_readable, - "0.12.0", - issue=878, - ), - "wait_socket_writable": - _deprecate.DeprecatedAttribute( - lowlevel.wait_writable, - "0.12.0", - issue=878, - ), - "notify_socket_close": - _deprecate.DeprecatedAttribute( - lowlevel.notify_closing, - "0.12.0", - issue=878, - ), - "notify_fd_close": - _deprecate.DeprecatedAttribute( - lowlevel.notify_closing, - "0.12.0", - issue=878, - ), + "wait_socket_readable": _deprecate.DeprecatedAttribute( + lowlevel.wait_readable, "0.12.0", issue=878, + ), + "wait_socket_writable": _deprecate.DeprecatedAttribute( + lowlevel.wait_writable, "0.12.0", issue=878, + ), + "notify_socket_close": _deprecate.DeprecatedAttribute( + lowlevel.notify_closing, "0.12.0", issue=878, + ), + "notify_fd_close": _deprecate.DeprecatedAttribute( + lowlevel.notify_closing, "0.12.0", issue=878, + ), } # Having the public path in .__module__ attributes is important for: @@ -169,6 +169,7 @@ # - pickle # - probably other stuff from ._util import fixup_module_metadata + fixup_module_metadata(__name__, globals()) fixup_module_metadata(lowlevel.__name__, lowlevel.__dict__) fixup_module_metadata(socket.__name__, socket.__dict__) diff --git a/trio/_abc.py b/trio/_abc.py index 504c145baa..e3ccb930ee 100644 --- a/trio/_abc.py +++ b/trio/_abc.py @@ -9,6 +9,7 @@ class Clock(metaclass=ABCMeta): """The interface for custom run loop clocks. """ + __slots__ = () @abstractmethod @@ -63,6 +64,7 @@ class Instrument(metaclass=ABCMeta): of these methods are optional. This class serves mostly as documentation. """ + __slots__ = () def before_run(self): @@ -144,12 +146,11 @@ class HostnameResolver(metaclass=ABCMeta): See :func:`trio.socket.set_custom_hostname_resolver`. """ + __slots__ = () @abstractmethod - async def getaddrinfo( - self, host, port, family=0, type=0, proto=0, flags=0 - ): + async def getaddrinfo(self, host, port, family=0, type=0, proto=0, flags=0): """A custom implementation of :func:`~trio.socket.getaddrinfo`. Called by :func:`trio.socket.getaddrinfo`. @@ -181,6 +182,7 @@ class SocketFactory(metaclass=ABCMeta): See :func:`trio.socket.set_custom_socket_factory`. """ + @abstractmethod def socket(self, family=None, type=None, proto=None): """Create and return a socket object. @@ -224,6 +226,7 @@ class AsyncResource(metaclass=ABCMeta): ``__aenter__`` and ``__aexit__`` should be adequate for all subclasses. """ + __slots__ = () @abstractmethod @@ -277,6 +280,7 @@ class SendStream(AsyncResource): :class:`SendChannel`. """ + __slots__ = () @abstractmethod @@ -382,6 +386,7 @@ class ReceiveStream(AsyncResource): byte, and the loop automatically exits when reaching end-of-file. """ + __slots__ = () @abstractmethod @@ -433,6 +438,7 @@ class Stream(SendStream, ReceiveStream): step further and implement :class:`HalfCloseableStream`. """ + __slots__ = () @@ -441,6 +447,7 @@ class HalfCloseableStream(Stream): part of the stream without closing the receive part. """ + __slots__ = () @abstractmethod @@ -519,6 +526,7 @@ class Listener(AsyncResource, Generic[T_resource]): or using an ``async with`` block. """ + __slots__ = () @abstractmethod @@ -560,6 +568,7 @@ class SendChannel(AsyncResource, Generic[SendType]): `SendStream`. """ + __slots__ = () @abstractmethod @@ -604,6 +613,7 @@ class ReceiveChannel(AsyncResource, Generic[ReceiveType]): `ReceiveStream`. """ + __slots__ = () @abstractmethod diff --git a/trio/_channel.py b/trio/_channel.py index 3ec404a8a2..dac7935c0c 100644 --- a/trio/_channel.py +++ b/trio/_channel.py @@ -70,7 +70,8 @@ def open_memory_channel(max_buffer_size): raise ValueError("max_buffer_size must be >= 0") state = MemoryChannelState(max_buffer_size) return ( - MemorySendChannel._create(state), MemoryReceiveChannel._create(state) + MemorySendChannel._create(state), + MemoryReceiveChannel._create(state), ) @@ -120,10 +121,8 @@ def __attrs_post_init__(self): self._state.open_send_channels += 1 def __repr__(self): - return ( - "".format( - id(self), id(self._state) - ) + return "".format( + id(self), id(self._state) ) def statistics(self): @@ -341,9 +340,7 @@ async def aclose(self): assert not self._state.receive_tasks for task in self._state.send_tasks: task.custom_sleep_data._tasks.remove(task) - trio.lowlevel.reschedule( - task, Error(trio.BrokenResourceError()) - ) + trio.lowlevel.reschedule(task, Error(trio.BrokenResourceError())) self._state.send_tasks.clear() self._state.data.clear() await trio.lowlevel.checkpoint() diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 4b3a088d1b..c28b7f4078 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -5,31 +5,59 @@ """ from ._exceptions import ( - TrioInternalError, RunFinishedError, WouldBlock, Cancelled, - BusyResourceError, ClosedResourceError, BrokenResourceError, EndOfChannel + TrioInternalError, + RunFinishedError, + WouldBlock, + Cancelled, + BusyResourceError, + ClosedResourceError, + BrokenResourceError, + EndOfChannel, ) from ._multierror import MultiError from ._ki import ( - enable_ki_protection, disable_ki_protection, currently_ki_protected + enable_ki_protection, + disable_ki_protection, + currently_ki_protected, ) # Imports that always exist from ._run import ( - Task, CancelScope, run, open_nursery, checkpoint, current_task, - current_effective_deadline, checkpoint_if_cancelled, TASK_STATUS_IGNORED, - current_statistics, current_trio_token, reschedule, remove_instrument, - add_instrument, current_clock, current_root_task, spawn_system_task, - current_time, wait_all_tasks_blocked, wait_readable, wait_writable, - notify_closing, Nursery + Task, + CancelScope, + run, + open_nursery, + checkpoint, + current_task, + current_effective_deadline, + checkpoint_if_cancelled, + TASK_STATUS_IGNORED, + current_statistics, + current_trio_token, + reschedule, + remove_instrument, + add_instrument, + current_clock, + current_root_task, + spawn_system_task, + current_time, + wait_all_tasks_blocked, + wait_readable, + wait_writable, + notify_closing, + Nursery, ) # Has to come after _run to resolve a circular import from ._traps import ( - cancel_shielded_checkpoint, Abort, wait_task_rescheduled, - temporarily_detach_coroutine_object, permanently_detach_coroutine_object, - reattach_detached_coroutine_object + cancel_shielded_checkpoint, + Abort, + wait_task_rescheduled, + temporarily_detach_coroutine_object, + permanently_detach_coroutine_object, + reattach_detached_coroutine_object, ) from ._entry_queue import TrioToken @@ -42,15 +70,19 @@ # Kqueue imports try: - from ._run import (current_kqueue, monitor_kevent, wait_kevent) + from ._run import current_kqueue, monitor_kevent, wait_kevent except ImportError: pass # Windows imports try: from ._run import ( - monitor_completion_key, current_iocp, register_with_iocp, - wait_overlapped, write_overlapped, readinto_overlapped + monitor_completion_key, + current_iocp, + register_with_iocp, + wait_overlapped, + write_overlapped, + readinto_overlapped, ) except ImportError: pass diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index 0d29e393b0..791ab8ca6e 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -141,7 +141,7 @@ class TrioToken(metaclass=NoPublicConstructor): """ - __slots__ = ('_reentry_queue',) + __slots__ = ("_reentry_queue",) def __init__(self, reentry_queue): self._reentry_queue = reentry_queue @@ -190,6 +190,4 @@ def run_sync_soon(self, sync_fn, *args, idempotent=False): exits.) """ - self._reentry_queue.run_sync_soon( - sync_fn, *args, idempotent=idempotent - ) + self._reentry_queue.run_sync_soon(sync_fn, *args, idempotent=idempotent) diff --git a/trio/_core/_exceptions.py b/trio/_core/_exceptions.py index b481f7c50d..81958bc762 100644 --- a/trio/_core/_exceptions.py +++ b/trio/_core/_exceptions.py @@ -62,6 +62,7 @@ class Cancelled(BaseException, metaclass=NoPublicConstructor): everywhere. """ + def __str__(self): return "Cancelled" diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index 5d73a58c84..71f46c40a7 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -235,9 +235,7 @@ def _update_registrations(self, fd): self._epoll.modify(fd, wanted_flags | select.EPOLLONESHOT) except OSError: # If that fails, it might be a new fd; try EPOLL_CTL_ADD - self._epoll.register( - fd, wanted_flags | select.EPOLLONESHOT - ) + self._epoll.register(fd, wanted_flags | select.EPOLLONESHOT) waiters.current_flags = wanted_flags except OSError as exc: # If everything fails, probably it's a bad fd, e.g. because @@ -284,7 +282,7 @@ def notify_closing(self, fd): fd = fd.fileno() wake_all( self._registered[fd], - _core.ClosedResourceError("another task closed this fd") + _core.ClosedResourceError("another task closed this fd"), ) del self._registered[fd] try: diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index 8f0f492c69..dd5ecd210a 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -30,10 +30,7 @@ def statistics(self): tasks_waiting += 1 else: monitors += 1 - return _KqueueStatistics( - tasks_waiting=tasks_waiting, - monitors=monitors, - ) + return _KqueueStatistics(tasks_waiting=tasks_waiting, monitors=monitors,) def close(self): self._kqueue.close() @@ -84,8 +81,7 @@ def monitor_kevent(self, ident, filter): key = (ident, filter) if key in self._registered: raise _core.BusyResourceError( - "attempt to register multiple listeners for same " - "ident/filter pair" + "attempt to register multiple listeners for same " "ident/filter pair" ) q = _core.UnboundedQueue() self._registered[key] = q @@ -99,8 +95,7 @@ async def wait_kevent(self, ident, filter, abort_func): key = (ident, filter) if key in self._registered: raise _core.BusyResourceError( - "attempt to register multiple listeners for same " - "ident/filter pair" + "attempt to register multiple listeners for same " "ident/filter pair" ) self._registered[key] = _core.current_task() @@ -134,9 +129,7 @@ def abort(_): # the fact... oh well, you can't have everything.) # # FreeBSD reports this using EBADF. macOS uses ENOENT. - if exc.errno in ( - errno.EBADF, errno.ENOENT - ): # pragma: no branch + if exc.errno in (errno.EBADF, errno.ENOENT,): # pragma: no branch pass else: # pragma: no cover # As far as we know, this branch can't happen. diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 7e452177d1..57056e465e 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -325,9 +325,7 @@ def __init__(self): self._afd = None self._iocp = _check( - kernel32.CreateIoCompletionPort( - INVALID_HANDLE_VALUE, ffi.NULL, 0, 0 - ) + kernel32.CreateIoCompletionPort(INVALID_HANDLE_VALUE, ffi.NULL, 0, 0) ) self._events = ffi.new("OVERLAPPED_ENTRY[]", MAX_EVENTS) @@ -350,9 +348,7 @@ def __init__(self): base_handle = _get_base_socket(s, which=WSAIoctls.SIO_BASE_HANDLE) # LSPs can in theory override this, but we believe that it never # actually happens in the wild. - select_handle = _get_base_socket( - s, which=WSAIoctls.SIO_BSP_HANDLE_SELECT - ) + select_handle = _get_base_socket(s, which=WSAIoctls.SIO_BSP_HANDLE_SELECT) if base_handle != select_handle: # pragma: no cover raise RuntimeError( "Unexpected network configuration detected. " @@ -400,8 +396,7 @@ def handle_io(self, timeout): try: _check( kernel32.GetQueuedCompletionStatusEx( - self._iocp, self._events, MAX_EVENTS, received, - milliseconds, 0 + self._iocp, self._events, MAX_EVENTS, received, milliseconds, 0, ) ) except OSError as exc: @@ -476,18 +471,13 @@ def handle_io(self, timeout): overlapped = int(ffi.cast("uintptr_t", entry.lpOverlapped)) transferred = entry.dwNumberOfBytesTransferred info = CompletionKeyEventInfo( - lpOverlapped=overlapped, - dwNumberOfBytesTransferred=transferred, + lpOverlapped=overlapped, dwNumberOfBytesTransferred=transferred, ) queue.put_nowait(info) def _register_with_iocp(self, handle, completion_key): handle = _handle(handle) - _check( - kernel32.CreateIoCompletionPort( - handle, self._iocp, completion_key, 0 - ) - ) + _check(kernel32.CreateIoCompletionPort(handle, self._iocp, completion_key, 0)) # Supposedly this makes things slightly faster, by disabling the # ability to do WaitForSingleObject(handle). We would never want to do # that anyway, so might as well get the extra speed (if any). @@ -506,11 +496,7 @@ def _refresh_afd(self, base_handle): waiters = self._afd_waiters[base_handle] if waiters.current_op is not None: try: - _check( - kernel32.CancelIoEx( - self._afd, waiters.current_op.lpOverlapped - ) - ) + _check(kernel32.CancelIoEx(self._afd, waiters.current_op.lpOverlapped)) except OSError as exc: if exc.winerror != ErrorCodes.ERROR_NOT_FOUND: # I don't think this is possible, so if it happens let's @@ -530,7 +516,7 @@ def _refresh_afd(self, base_handle): lpOverlapped = ffi.new("LPOVERLAPPED") poll_info = ffi.new("AFD_POLL_INFO *") - poll_info.Timeout = 2**63 - 1 # INT64_MAX + poll_info.Timeout = 2 ** 63 - 1 # INT64_MAX poll_info.NumberOfHandles = 1 poll_info.Exclusive = 0 poll_info.Handles[0].Handle = base_handle @@ -669,9 +655,7 @@ def abort(raise_cancel_): # We didn't request this cancellation, so assume # it happened due to the underlying handle being # closed before the operation could complete. - raise _core.ClosedResourceError( - "another task closed this resource" - ) + raise _core.ClosedResourceError("another task closed this resource") else: raise_winerror(code) @@ -700,7 +684,7 @@ async def write_overlapped(self, handle, data, file_offset=0): def submit_write(lpOverlapped): # yes, these are the real documented names offset_fields = lpOverlapped.DUMMYUNIONNAME.DUMMYSTRUCTNAME - offset_fields.Offset = file_offset & 0xffffffff + offset_fields.Offset = file_offset & 0xFFFFFFFF offset_fields.OffsetHigh = file_offset >> 32 _check( kernel32.WriteFile( @@ -722,7 +706,7 @@ async def readinto_overlapped(self, handle, buffer, file_offset=0): def submit_read(lpOverlapped): offset_fields = lpOverlapped.DUMMYUNIONNAME.DUMMYSTRUCTNAME - offset_fields.Offset = file_offset & 0xffffffff + offset_fields.Offset = file_offset & 0xFFFFFFFF offset_fields.OffsetHigh = file_offset >> 32 _check( kernel32.ReadFile( diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index 9406ebb7e1..d5aa63f5d9 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -10,7 +10,8 @@ if False: from typing import Any, TypeVar, Callable - F = TypeVar('F', bound=Callable[..., Any]) + + F = TypeVar("F", bound=Callable[..., Any]) # In ordinary single-threaded Python code, when you hit control-C, it raises # an exception and automatically does all the regular unwinding stuff. @@ -77,7 +78,7 @@ # We use this special string as a unique key into the frame locals dictionary. # The @ ensures it is not a valid identifier and can't clash with any possible # real local name. See: https://github.com/python-trio/trio/issues/469 -LOCALS_KEY_KI_PROTECTION_ENABLED = '@TRIO_KI_PROTECTION_ENABLED' +LOCALS_KEY_KI_PROTECTION_ENABLED = "@TRIO_KI_PROTECTION_ENABLED" # NB: according to the signal.signal docs, 'frame' can be None on entry to @@ -119,8 +120,7 @@ def decorator(fn): def wrapper(*args, **kwargs): # See the comment for regular generators below coro = fn(*args, **kwargs) - coro.cr_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED - ] = enabled + coro.cr_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] = enabled return coro return wrapper @@ -137,8 +137,7 @@ def wrapper(*args, **kwargs): # thrown into! See: # https://bugs.python.org/issue29590 gen = fn(*args, **kwargs) - gen.gi_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED - ] = enabled + gen.gi_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] = enabled return gen return wrapper @@ -148,8 +147,7 @@ def wrapper(*args, **kwargs): def wrapper(*args, **kwargs): # See the comment for regular generators above agen = fn(*args, **kwargs) - agen.ag_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED - ] = enabled + agen.ag_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] = enabled return agen return wrapper @@ -168,9 +166,7 @@ def wrapper(*args, **kwargs): enable_ki_protection = _ki_protection_decorator(True) # type: Callable[[F], F] enable_ki_protection.__name__ = "enable_ki_protection" -disable_ki_protection = _ki_protection_decorator( - False -) # type: Callable[[F], F] +disable_ki_protection = _ki_protection_decorator(False) # type: Callable[[F], F] disable_ki_protection.__name__ = "disable_ki_protection" diff --git a/trio/_core/_local.py b/trio/_core/_local.py index aaf5f6d2f7..352caa5682 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -40,8 +40,7 @@ def get(self, default=_NO_DEFAULT): try: return _run.GLOBAL_RUN_CONTEXT.runner._locals[self] except AttributeError: - raise RuntimeError("Cannot be used outside of a run context") \ - from None + raise RuntimeError("Cannot be used outside of a run context") from None except KeyError: # contextvars consistency if default is not self._NO_DEFAULT: @@ -95,4 +94,4 @@ def reset(self, token): token.redeemed = True def __repr__(self): - return ("".format(self._name)) + return "".format(self._name) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index f98540344d..cdeeac6269 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -173,6 +173,7 @@ class MultiError(BaseException): :exc:`BaseException`. """ + def __init__(self, exceptions): # Avoid recursion when exceptions[0] returned by __new__() happens # to be a MultiError and subsequently __init__() is called. @@ -186,9 +187,7 @@ def __new__(cls, exceptions): exceptions = list(exceptions) for exc in exceptions: if not isinstance(exc, BaseException): - raise TypeError( - "Expected an exception object, not {!r}".format(exc) - ) + raise TypeError("Expected an exception object, not {!r}".format(exc)) if len(exceptions) == 1: # If this lone object happens to itself be a MultiError, then # Python will implicitly call our __init__ on it again. See @@ -280,16 +279,20 @@ def controller(operation): # no missing test we could add, and no value in coverage nagging # us about adding one. if operation.opname in [ - "__getattribute__", "__getattr__" + "__getattribute__", + "__getattr__", ]: # pragma: no cover if operation.args[0] == "tb_next": return tb_next return operation.delegate() return tputil.make_proxy(controller, type(base_tb), base_tb) + + else: # ctypes it is import ctypes + # How to handle refcounting? I don't want to use ctypes.py_object because # I don't understand or trust it, and I don't want to use # ctypes.pythonapi.Py_{Inc,Dec}Ref because we might clash with user code @@ -374,7 +377,7 @@ def traceback_exception_init( limit=None, lookup_lines=True, capture_locals=False, - _seen=None + _seen=None, ): if _seen is None: _seen = set() @@ -388,7 +391,7 @@ def traceback_exception_init( limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, - _seen=_seen + _seen=_seen, ) # Capture each of the exceptions in the MultiError along with each of their causes and contexts @@ -404,7 +407,7 @@ def traceback_exception_init( capture_locals=capture_locals, # copy the set of _seen exceptions so that duplicates # shared between sub-exceptions are not omitted - _seen=set(_seen) + _seen=set(_seen), ) ) self.embedded = embedded @@ -421,9 +424,7 @@ def traceback_exception_format(self, *, chain=True): for i, exc in enumerate(self.embedded): yield "\nDetails of embedded exception {}:\n\n".format(i + 1) - yield from ( - textwrap.indent(line, " " * 2) for line in exc.format(chain=chain) - ) + yield from (textwrap.indent(line, " " * 2) for line in exc.format(chain=chain)) traceback.TracebackException.format = traceback_exception_format @@ -438,6 +439,7 @@ def trio_excepthook(etype, value, tb): if "IPython" in sys.modules: import IPython + ip = IPython.get_ipython() if ip is not None: if ip.custom_exceptions != (): @@ -446,7 +448,7 @@ def trio_excepthook(etype, value, tb): "handler installed. I'll skip installing Trio's custom " "handler, but this means MultiErrors will not show full " "tracebacks.", - category=RuntimeWarning + category=RuntimeWarning, ) monkeypatched_or_warned = True else: @@ -477,6 +479,7 @@ def trio_show_traceback(self, etype, value, tb, tb_offset=None): # More details: https://github.com/python-trio/trio/issues/1065 if sys.excepthook.__name__ == "apport_excepthook": import apport_python_hook + assert sys.excepthook is apport_python_hook.apport_excepthook # Give it a descriptive name as a hint for anyone who's stuck trying to @@ -496,5 +499,5 @@ class TrioFakeSysModuleForApport: "You seem to already have a custom sys.excepthook handler " "installed. I'll skip installing Trio's custom handler, but this " "means MultiErrors will not show full tracebacks.", - category=RuntimeWarning + category=RuntimeWarning, ) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index f77860d2f2..4de4e9a2bf 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -21,9 +21,11 @@ from outcome import Error, Value, capture from ._entry_queue import EntryQueue, TrioToken -from ._exceptions import (TrioInternalError, RunFinishedError, Cancelled) +from ._exceptions import TrioInternalError, RunFinishedError, Cancelled from ._ki import ( - LOCALS_KEY_KI_PROTECTION_ENABLED, ki_manager, enable_ki_protection + LOCALS_KEY_KI_PROTECTION_ENABLED, + ki_manager, + enable_ki_protection, ) from ._multierror import MultiError from ._traps import ( @@ -247,7 +249,8 @@ def close(self): @property def parent_cancellation_is_visible_to_us(self): return ( - self._parent is not None and not self._scope.shield + self._parent is not None + and not self._scope.shield and self._parent.effectively_cancelled ) @@ -368,9 +371,7 @@ def __enter__(self): if current_time() >= self._deadline: self.cancel() with self._might_change_registered_deadline(): - self._cancel_status = CancelStatus( - scope=self, parent=task._cancel_status - ) + self._cancel_status = CancelStatus(scope=self, parent=task._cancel_status) task._activate_cancel_status(self._cancel_status) return self @@ -421,8 +422,10 @@ def _close(self, exc): new_exc = RuntimeError( "Cancel scope stack corrupted: attempted to exit {!r} " "in {!r} that's still within its child {!r}\n{}".format( - self, scope_task, scope_task._cancel_status._scope, - MISNESTING_ADVICE + self, + scope_task, + scope_task._cancel_status._scope, + MISNESTING_ADVICE, ) ) new_exc.__context__ = exc @@ -431,7 +434,8 @@ def _close(self, exc): else: scope_task._activate_cancel_status(self._cancel_status.parent) if ( - exc is not None and self._cancel_status.effectively_cancelled + exc is not None + and self._cancel_status.effectively_cancelled and not self._cancel_status.parent_cancellation_is_visible_to_us ): exc = MultiError.filter(self._exc_filter, exc) @@ -484,12 +488,10 @@ def __repr__(self): else: state = ", deadline is {:.2f} seconds {}".format( abs(self._deadline - now), - "from now" if self._deadline >= now else "ago" + "from now" if self._deadline >= now else "ago", ) - return "".format( - id(self), binding, state - ) + return "".format(id(self), binding, state) @contextmanager @enable_ki_protection @@ -637,9 +639,7 @@ def __repr__(self): def started(self, value=None): if self._called_started: - raise RuntimeError( - "called 'started' twice on the same task status" - ) + raise RuntimeError("called 'started' twice on the same task status") self._called_started = True self._value = value @@ -698,6 +698,7 @@ class NurseryManager: and StopAsyncIteration. """ + @enable_ki_protection async def __aenter__(self): self._scope = CancelScope() @@ -767,6 +768,7 @@ class Nursery(metaclass=NoPublicConstructor): other things, e.g. if you want to explicitly cancel all children in response to some external event. """ + def __init__(self, parent_task, cancel_scope): self._parent_task = parent_task parent_task._child_nurseries.append(self) @@ -803,9 +805,7 @@ def _add_exc(self, exc): self.cancel_scope.cancel() def _check_nursery_closed(self): - if not any( - [self._nested_child_running, self._children, self._pending_starts] - ): + if not any([self._nested_child_running, self._children, self._pending_starts]): self._closed = True if self._parent_waiting_in_aexit: self._parent_waiting_in_aexit = False @@ -949,9 +949,7 @@ async def async_fn(arg1, arg2, \*, task_status=trio.TASK_STATUS_IGNORED): # normally. The complicated logic is all in _TaskStatus.started(). # (Any exceptions propagate directly out of the above.) if not task_status._called_started: - raise RuntimeError( - "child exited without calling task_status.started()" - ) + raise RuntimeError("child exited without calling task_status.started()") return task_status._value finally: self._pending_starts -= 1 @@ -1002,7 +1000,7 @@ class Task(metaclass=NoPublicConstructor): _schedule_points = attr.ib(default=0) def __repr__(self): - return ("".format(self.name, id(self))) + return "".format(self.name, id(self)) @property def parent_nursery(self): @@ -1269,19 +1267,13 @@ async def python_wrapper(orig_coro): return await orig_coro coro = python_wrapper(coro) - coro.cr_frame.f_locals.setdefault( - LOCALS_KEY_KI_PROTECTION_ENABLED, system_task - ) + coro.cr_frame.f_locals.setdefault(LOCALS_KEY_KI_PROTECTION_ENABLED, system_task) ###### # Set up the Task object ###### task = Task._create( - coro=coro, - parent_nursery=nursery, - runner=self, - name=name, - context=context, + coro=coro, parent_nursery=nursery, runner=self, name=name, context=context, ) self.tasks.add(task) @@ -1393,9 +1385,7 @@ async def init(self, async_fn, args): async with open_nursery() as system_nursery: self.system_nursery = system_nursery try: - self.main_task = self.spawn_impl( - async_fn, args, system_nursery, None - ) + self.main_task = self.spawn_impl(async_fn, args, system_nursery, None) except BaseException as exc: self.main_task_outcome = Error(exc) system_nursery.cancel_scope.cancel() @@ -1544,7 +1534,9 @@ def instrument(self, method_name, *args): self.instruments.remove(instrument) INSTRUMENT_LOGGER.exception( "Exception raised when calling %r on instrument %r. " - "Instrument has been disabled.", method_name, instrument + "Instrument has been disabled.", + method_name, + instrument, ) @_public @@ -1592,7 +1584,7 @@ def run( *args, clock=None, instruments=(), - restrict_keyboard_interrupt_to_checkpoints=False + restrict_keyboard_interrupt_to_checkpoints=False, ): """Run a Trio-flavored async function, and return the result. @@ -1692,9 +1684,7 @@ def run( # where KeyboardInterrupt would be allowed and converted into an # TrioInternalError: try: - with ki_manager( - runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints - ): + with ki_manager(runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints): try: with closing(runner): with runner.entry_queue.wakeup.wakeup_on_signals(): @@ -1736,11 +1726,7 @@ def run_impl(runner, async_fn, args): runner.instrument("before_run") runner.clock.start_clock() runner.init_task = runner.spawn_impl( - runner.init, - (async_fn, args), - None, - "", - system_task=True, + runner.init, (async_fn, args), None, "", system_task=True, ) # You know how people talk about "event loops"? This 'while' loop right @@ -1977,9 +1963,8 @@ async def checkpoint_if_cancelled(): """ task = current_task() - if ( - task._cancel_status.effectively_cancelled or - (task is task._runner.main_task and task._runner.ki_pending) + if task._cancel_status.effectively_cancelled or ( + task is task._runner.main_task and task._runner.ki_pending ): await _core.checkpoint() assert False # pragma: no cover diff --git a/trio/_core/_traps.py b/trio/_core/_traps.py index f340481078..95cf46de9b 100644 --- a/trio/_core/_traps.py +++ b/trio/_core/_traps.py @@ -53,6 +53,7 @@ class Abort(enum.Enum): FAILED """ + SUCCEEDED = 1 FAILED = 2 diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index efa0a8d1b3..9830df4bea 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -40,11 +40,12 @@ class UnboundedQueue(metaclass=SubclassingDeprecatedIn_v0_15_0): ... """ + @deprecated( "0.9.0", issue=497, thing="trio.lowlevel.UnboundedQueue", - instead="trio.open_memory_channel(math.inf)" + instead="trio.open_memory_channel(math.inf)", ) def __init__(self): self._lot = _core.ParkingLot() @@ -140,8 +141,7 @@ def statistics(self): """ return _UnboundedQueueStats( - qsize=len(self._data), - tasks_waiting=self._lot.statistics().tasks_waiting + qsize=len(self._data), tasks_waiting=self._lot.statistics().tasks_waiting, ) def __aiter__(self): diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 0c37928a55..3513cc1ab3 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -35,9 +35,7 @@ def __init__(self): # On Windows this is a TCP socket so this might matter. On other # platforms this fails b/c AF_UNIX sockets aren't actually TCP. try: - self.write_sock.setsockopt( - socket.IPPROTO_TCP, socket.TCP_NODELAY, 1 - ) + self.write_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except OSError: pass @@ -54,7 +52,7 @@ async def wait_woken(self): def drain(self): try: while True: - self.wakeup_sock.recv(2**16) + self.wakeup_sock.recv(2 ** 16) except BlockingIOError: pass diff --git a/trio/_core/tests/test_io.py b/trio/_core/tests/test_io.py index fb459bea2d..397375503d 100644 --- a/trio/_core/tests/test_io.py +++ b/trio/_core/tests/test_io.py @@ -53,7 +53,9 @@ def fileno_wrapper(fileobj): notify_closing_options = [trio.lowlevel.notify_closing] for options_list in [ - wait_readable_options, wait_writable_options, notify_closing_options + wait_readable_options, + wait_writable_options, + notify_closing_options, ]: options_list += [using_fileno(f) for f in options_list] @@ -196,9 +198,7 @@ async def writer(): @read_socket_test @write_socket_test -async def test_socket_simultaneous_read_write( - socketpair, wait_readable, wait_writable -): +async def test_socket_simultaneous_read_write(socketpair, wait_readable, wait_writable): record = [] async def r_task(sock): @@ -226,9 +226,7 @@ async def w_task(sock): @read_socket_test @write_socket_test -async def test_socket_actual_streaming( - socketpair, wait_readable, wait_writable -): +async def test_socket_actual_streaming(socketpair, wait_readable, wait_writable): a, b = socketpair # Use a small send buffer on one of the sockets to increase the chance of diff --git a/trio/_core/tests/test_ki.py b/trio/_core/tests/test_ki.py index a63407484a..12bfe86793 100644 --- a/trio/_core/tests/test_ki.py +++ b/trio/_core/tests/test_ki.py @@ -8,7 +8,10 @@ import time from async_generator import ( - async_generator, yield_, isasyncgenfunction, asynccontextmanager + async_generator, + yield_, + isasyncgenfunction, + asynccontextmanager, ) from ... import _core @@ -107,6 +110,7 @@ async def unprotected(): async def child(expected): import traceback + traceback.print_stack() assert _core.currently_ki_protected() == expected await _core.checkpoint() @@ -259,9 +263,7 @@ async def raiser(name, record): # If we didn't raise (b/c protected), then we *should* get # cancelled at the next opportunity try: - await _core.wait_task_rescheduled( - lambda _: _core.Abort.SUCCEEDED - ) + await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) except _core.Cancelled: record.add(name + " cancel ok") @@ -288,9 +290,7 @@ async def check_protected_kill(): async with _core.open_nursery() as nursery: nursery.start_soon(sleeper, "s1", record) nursery.start_soon(sleeper, "s2", record) - nursery.start_soon( - _core.enable_ki_protection(raiser), "r1", record - ) + nursery.start_soon(_core.enable_ki_protection(raiser), "r1", record) # __aexit__ blocks, and then receives the KI with pytest.raises(KeyboardInterrupt): @@ -487,9 +487,7 @@ def test_ki_with_broken_threads(): @_core.enable_ki_protection async def inner(): - assert signal.getsignal( - signal.SIGINT - ) != signal.default_int_handler + assert signal.getsignal(signal.SIGINT) != signal.default_int_handler _core.run(inner) finally: @@ -537,9 +535,11 @@ def test_ki_wakes_us_up(): # https://bugs.python.org/issue31119 # https://bitbucket.org/pypy/pypy/issues/2623 import platform - buggy_wakeup_fd = ( - platform.python_implementation() == "CPython" and sys.version_info < - (3, 6, 2) + + buggy_wakeup_fd = platform.python_implementation() == "CPython" and sys.version_info < ( + 3, + 6, + 2, ) # lock is only needed to avoid an annoying race condition where the diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 6debf4d45c..c1444b4c0e 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -1,7 +1,12 @@ import logging import pytest -from traceback import extract_tb, print_exception, format_exception, _cause_message +from traceback import ( + extract_tb, + print_exception, + format_exception, + _cause_message, +) import sys import os import re @@ -243,9 +248,9 @@ def simple_filter(exc): assert isinstance(orig.exceptions[0].exceptions[1], KeyError) # get original traceback summary orig_extracted = ( - extract_tb(orig.__traceback__) + - extract_tb(orig.exceptions[0].__traceback__) + - extract_tb(orig.exceptions[0].exceptions[1].__traceback__) + extract_tb(orig.__traceback__) + + extract_tb(orig.exceptions[0].__traceback__) + + extract_tb(orig.exceptions[0].exceptions[1].__traceback__) ) def p(exc): @@ -495,7 +500,7 @@ def test_format_exception(): r"in raiser3", r"NameError", ], - formatted + formatted, ) # Prints duplicate exceptions in sub-exceptions @@ -556,7 +561,7 @@ def raise2_raiser1(): r"in raise2_raiser1", r" KeyError: 'bar'", ], - formatted + formatted, ) @@ -572,15 +577,14 @@ def test_logging(caplog): except MultiError as exc: logging.getLogger().exception(message) # Join lines together - formatted = "".join( - format_exception(type(exc), exc, exc.__traceback__) - ) + formatted = "".join(format_exception(type(exc), exc, exc.__traceback__)) assert message in caplog.text assert formatted in caplog.text def run_script(name, use_ipython=False): import trio + trio_path = Path(trio.__file__).parent.parent script_path = Path(__file__).parent / "test_multierror_scripts" / name @@ -605,7 +609,7 @@ def run_script(name, use_ipython=False): "IPython", # no startup files "--quick", - "--TerminalIPythonApp.code_to_run=" + '\n'.join(lines), + "--TerminalIPythonApp.code_to_run=" + "\n".join(lines), ] else: cmd = [sys.executable, "-u", str(script_path)] @@ -629,7 +633,8 @@ def check_simple_excepthook(completed): "Details of embedded exception 2", "in exc2_fn", "KeyError", - ], completed.stdout.decode("utf-8") + ], + completed.stdout.decode("utf-8"), ) @@ -652,17 +657,18 @@ def test_custom_excepthook(): # The MultiError "MultiError:", ], - completed.stdout.decode("utf-8") + completed.stdout.decode("utf-8"), ) # This warning is triggered by ipython 7.5.0 on python 3.8 import warnings + warnings.filterwarnings( "ignore", - message=".*\"@coroutine\" decorator is deprecated", + message='.*"@coroutine" decorator is deprecated', category=DeprecationWarning, - module="IPython.*" + module="IPython.*", ) try: import IPython @@ -705,7 +711,7 @@ def test_ipython_custom_exc_handler(): "ValueError", "KeyError", ], - completed.stdout.decode("utf-8") + completed.stdout.decode("utf-8"), ) # Make sure our other warning doesn't show up assert "custom sys.excepthook" not in completed.stdout.decode("utf-8") @@ -714,7 +720,7 @@ def test_ipython_custom_exc_handler(): @slow @pytest.mark.skipif( not Path("/usr/lib/python3/dist-packages/apport_python_hook.py").exists(), - reason="need Ubuntu with python3-apport installed" + reason="need Ubuntu with python3-apport installed", ) def test_apport_excepthook_monkeypatch_interaction(): completed = run_script("apport_excepthook.py") @@ -725,10 +731,6 @@ def test_apport_excepthook_monkeypatch_interaction(): # Proper traceback assert_match_in_seq( - [ - "Details of embedded", - "KeyError", - "Details of embedded", - "ValueError", - ], stdout + ["Details of embedded", "KeyError", "Details of embedded", "ValueError",], + stdout, ) diff --git a/trio/_core/tests/test_multierror_scripts/apport_excepthook.py b/trio/_core/tests/test_multierror_scripts/apport_excepthook.py index ac8110f36e..12e7fb0851 100644 --- a/trio/_core/tests/test_multierror_scripts/apport_excepthook.py +++ b/trio/_core/tests/test_multierror_scripts/apport_excepthook.py @@ -2,8 +2,10 @@ # python, and not available in venvs. So before we can import it we have to # make sure it's on sys.path. import sys + sys.path.append("/usr/lib/python3/dist-packages") import apport_python_hook + apport_python_hook.install() import trio diff --git a/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py b/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py index 017c5ea059..b3fd110e50 100644 --- a/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py +++ b/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py @@ -14,6 +14,7 @@ def custom_excepthook(*args): sys.excepthook = custom_excepthook import IPython + ip = IPython.get_ipython() diff --git a/trio/_core/tests/test_parking_lot.py b/trio/_core/tests/test_parking_lot.py index 95e4a96b50..1d1ecd111a 100644 --- a/trio/_core/tests/test_parking_lot.py +++ b/trio/_core/tests/test_parking_lot.py @@ -32,10 +32,7 @@ async def waiter(i, lot): assert len(record) == 6 check_sequence_matches( - record, [ - {"sleep 0", "sleep 1", "sleep 2"}, - {"wake 0", "wake 1", "wake 2"}, - ] + record, [{"sleep 0", "sleep 1", "sleep 2"}, {"wake 0", "wake 1", "wake 2"},], ) async with _core.open_nursery() as nursery: @@ -71,12 +68,7 @@ async def waiter(i, lot): lot.unpark(count=2) await wait_all_tasks_blocked() check_sequence_matches( - record, [ - "sleep 0", - "sleep 1", - "sleep 2", - {"wake 0", "wake 1"}, - ] + record, ["sleep 0", "sleep 1", "sleep 2", {"wake 0", "wake 1"},] ) lot.unpark_all() @@ -115,13 +107,7 @@ async def test_parking_lot_cancel(): assert len(record) == 6 check_sequence_matches( - record, [ - "sleep 1", - "sleep 2", - "sleep 3", - "cancelled 2", - {"wake 1", "wake 3"}, - ] + record, ["sleep 1", "sleep 2", "sleep 3", "cancelled 2", {"wake 1", "wake 3"},], ) @@ -160,13 +146,22 @@ async def test_parking_lot_repark(): await wait_all_tasks_blocked() assert len(lot2) == 1 assert record == [ - "sleep 1", "sleep 2", "sleep 3", "wake 1", "cancelled 2" + "sleep 1", + "sleep 2", + "sleep 3", + "wake 1", + "cancelled 2", ] lot2.unpark_all() await wait_all_tasks_blocked() assert record == [ - "sleep 1", "sleep 2", "sleep 3", "wake 1", "cancelled 2", "wake 3" + "sleep 1", + "sleep 2", + "sleep 3", + "wake 1", + "cancelled 2", + "wake 3", ] diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 216fd5b050..4368984370 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -16,8 +16,10 @@ import pytest from .tutil import ( - slow, check_sequence_matches, gc_collect_harder, - ignore_coroutine_never_awaited_warnings + slow, + check_sequence_matches, + gc_collect_harder, + ignore_coroutine_never_awaited_warnings, ) from ... import _core @@ -128,8 +130,7 @@ async def looper(whoami, record): nursery.start_soon(looper, "b", record) check_sequence_matches( - record, - [{("a", 0), ("b", 0)}, {("a", 1), ("b", 1)}, {("a", 2), ("b", 2)}] + record, [{("a", 0), ("b", 0)}, {("a", 1), ("b", 1)}, {("a", 2), ("b", 2)}], ) @@ -173,8 +174,10 @@ async def main(): with pytest.raises(_core.MultiError) as excinfo: _core.run(main) print(excinfo.value) - assert {type(exc) - for exc in excinfo.value.exceptions} == {ValueError, KeyError} + assert {type(exc) for exc in excinfo.value.exceptions} == { + ValueError, + KeyError, + } def test_two_child_crashes(): @@ -188,8 +191,10 @@ async def main(): with pytest.raises(_core.MultiError) as excinfo: _core.run(main) - assert {type(exc) - for exc in excinfo.value.exceptions} == {ValueError, KeyError} + assert {type(exc) for exc in excinfo.value.exceptions} == { + ValueError, + KeyError, + } async def test_child_crash_wakes_parent(): @@ -394,9 +399,9 @@ async def main(): # reschedules the task immediately upon yielding, before the # after_task_step event fires. expected = ( - [("before_run",), ("schedule", task)] + - [("before", task), ("schedule", task), ("after", task)] * 5 + - [("before", task), ("after", task), ("after_run",)] + [("before_run",), ("schedule", task)] + + [("before", task), ("schedule", task), ("after", task)] * 5 + + [("before", task), ("after", task), ("after_run",)] ) assert len(r1.record) > len(r2.record) > len(r3.record) assert r1.record == r2.record + r3.record @@ -432,13 +437,13 @@ async def main(): ("after", tasks["t1"]), ("before", tasks["t2"]), ("schedule", tasks["t2"]), - ("after", tasks["t2"]) + ("after", tasks["t2"]), }, { ("before", tasks["t1"]), ("after", tasks["t1"]), ("before", tasks["t2"]), - ("after", tasks["t2"]) + ("after", tasks["t2"]), }, ("after_run",), ] @@ -962,9 +967,7 @@ async def task2(): for exc in exc_info.value.__context__.exceptions: assert isinstance(exc, RuntimeError) assert "closed before the task exited" in str(exc) - cancelled_in_context |= isinstance( - exc.__context__, _core.Cancelled - ) + cancelled_in_context |= isinstance(exc.__context__, _core.Cancelled) assert cancelled_in_context # for the sleep_forever # Trying to exit a cancel scope from an unrelated task raises an error @@ -1230,9 +1233,14 @@ async def child2(): nursery.start_soon(child2) assert record == [ - "child1 raise", "child1 sleep", "child2 wake", "child2 sleep again", - "child1 re-raise", "child1 success", "child2 re-raise", - "child2 success" + "child1 raise", + "child1 sleep", + "child2 wake", + "child2 sleep again", + "child1 re-raise", + "child1 success", + "child2 re-raise", + "child2 success", ] @@ -1705,8 +1713,7 @@ async def async_gen(arg): # pragma: no cover yield arg with pytest.raises( - TypeError, - match="expected an async function but got an async generator" + TypeError, match="expected an async function but got an async generator", ): bad_call(async_gen, 0) @@ -1714,6 +1721,7 @@ async def async_gen(arg): # pragma: no cover def test_calling_asyncio_function_gives_nice_error(): async def child_xyzzy(): import asyncio + await asyncio.Future() async def misguided(): @@ -1734,6 +1742,7 @@ async def test_asyncio_function_inside_nursery_does_not_explode(): with pytest.raises(TypeError) as excinfo: async with _core.open_nursery() as nursery: import asyncio + nursery.start_soon(sleep_forever) await asyncio.Future() assert "asyncio" in str(excinfo.value) @@ -1757,10 +1766,10 @@ async def test_trivial_yields(): async with _core.open_nursery(): raise KeyError assert len(excinfo.value.exceptions) == 2 - assert {type(e) - for e in excinfo.value.exceptions} == { - KeyError, _core.Cancelled - } + assert {type(e) for e in excinfo.value.exceptions} == { + KeyError, + _core.Cancelled, + } async def test_nursery_start(autojump_clock): @@ -1772,9 +1781,7 @@ async def no_args(): # pragma: no cover with pytest.raises(TypeError): await nursery.start(no_args) - async def sleep_then_start( - seconds, *, task_status=_core.TASK_STATUS_IGNORED - ): + async def sleep_then_start(seconds, *, task_status=_core.TASK_STATUS_IGNORED): repr(task_status) # smoke test await sleep(seconds) task_status.started(seconds) @@ -1838,9 +1845,7 @@ async def just_started(task_status=_core.TASK_STATUS_IGNORED): # and if after the no-op started(), the child crashes, the error comes out # of start() - async def raise_keyerror_after_started( - task_status=_core.TASK_STATUS_IGNORED - ): + async def raise_keyerror_after_started(task_status=_core.TASK_STATUS_IGNORED,): task_status.started() raise KeyError("whoopsiedaisy") @@ -1849,8 +1854,10 @@ async def raise_keyerror_after_started( cs.cancel() with pytest.raises(_core.MultiError) as excinfo: await nursery.start(raise_keyerror_after_started) - assert {type(e) - for e in excinfo.value.exceptions} == {_core.Cancelled, KeyError} + assert {type(e) for e in excinfo.value.exceptions} == { + _core.Cancelled, + KeyError, + } # trying to start in a closed nursery raises an error immediately async with _core.open_nursery() as closed_nursery: @@ -1990,9 +1997,7 @@ def __aiter__(self): async def __anext__(self): nexts = self.nexts - items = [ - None, - ] * len(nexts) + items = [None,] * len(nexts) got_stop = False def handle(exc): @@ -2082,7 +2087,7 @@ async def t2(): def test_system_task_contexts(): - cvar = contextvars.ContextVar('qwilfish') + cvar = contextvars.ContextVar("qwilfish") cvar.set("water") async def system_task(): @@ -2132,7 +2137,7 @@ def test_Cancelled_init(): def test_Cancelled_str(): cancelled = _core.Cancelled._create() - assert str(cancelled) == 'Cancelled' + assert str(cancelled) == "Cancelled" def test_Cancelled_subclass(): @@ -2190,9 +2195,7 @@ async def detachable_coroutine(task_outcome, yield_value): await async_yield(yield_value) async with _core.open_nursery() as nursery: - nursery.start_soon( - detachable_coroutine, outcome.Value(None), "I'm free!" - ) + nursery.start_soon(detachable_coroutine, outcome.Value(None), "I'm free!") # If we get here then Trio thinks the task has exited... but the coroutine # is still iterable @@ -2207,9 +2210,7 @@ async def detachable_coroutine(task_outcome, yield_value): pdco_outcome = None with pytest.raises(KeyError): async with _core.open_nursery() as nursery: - nursery.start_soon( - detachable_coroutine, outcome.Error(KeyError()), "uh oh" - ) + nursery.start_soon(detachable_coroutine, outcome.Error(KeyError()), "uh oh") throw_in = ValueError() assert task.coro.throw(throw_in) == "uh oh" assert pdco_outcome == outcome.Error(throw_in) @@ -2219,9 +2220,7 @@ async def detachable_coroutine(task_outcome, yield_value): async def bad_detach(): async with _core.open_nursery(): with pytest.raises(RuntimeError) as excinfo: - await _core.permanently_detach_coroutine_object( - outcome.Value(None) - ) + await _core.permanently_detach_coroutine_object(outcome.Value(None)) assert "open nurser" in str(excinfo.value) async with _core.open_nursery() as nursery: @@ -2252,9 +2251,7 @@ def abort_fn(_): # pragma: no cover await async_yield(2) with pytest.raises(RuntimeError) as excinfo: - await _core.reattach_detached_coroutine_object( - unrelated_task, None - ) + await _core.reattach_detached_coroutine_object(unrelated_task, None) assert "does not match" in str(excinfo.value) await _core.reattach_detached_coroutine_object(task, "byebye") diff --git a/trio/_core/tests/test_windows.py b/trio/_core/tests/test_windows.py index 2fb8a97092..0a8179f88f 100644 --- a/trio/_core/tests/test_windows.py +++ b/trio/_core/tests/test_windows.py @@ -4,25 +4,28 @@ import pytest -on_windows = (os.name == "nt") +on_windows = os.name == "nt" # Mark all the tests in this file as being windows-only pytestmark = pytest.mark.skipif(not on_windows, reason="windows only") from .tutil import slow, gc_collect_harder from ... import _core, sleep, move_on_after from ...testing import wait_all_tasks_blocked + if on_windows: from .._windows_cffi import ( - ffi, kernel32, INVALID_HANDLE_VALUE, raise_winerror, FileFlags + ffi, + kernel32, + INVALID_HANDLE_VALUE, + raise_winerror, + FileFlags, ) # The undocumented API that this is testing should be changed to stop using # UnboundedQueue (or just removed until we have time to redo it), but until # then we filter out the warning. -@pytest.mark.filterwarnings( - "ignore:.*UnboundedQueue:trio.TrioDeprecationWarning" -) +@pytest.mark.filterwarnings("ignore:.*UnboundedQueue:trio.TrioDeprecationWarning") async def test_completion_key_listen(): async def post(key): iocp = ffi.cast("HANDLE", _core.current_iocp()) @@ -30,9 +33,7 @@ async def post(key): print("post", i) if i % 3 == 0: await _core.checkpoint() - success = kernel32.PostQueuedCompletionStatus( - iocp, i, key, ffi.NULL - ) + success = kernel32.PostQueuedCompletionStatus(iocp, i, key, ffi.NULL) assert success with _core.monitor_completion_key() as (key, queue): @@ -80,9 +81,7 @@ async def test_readinto_overlapped(): async def read_region(start, end): await _core.readinto_overlapped( - handle, - buffer_view[start:end], - start, + handle, buffer_view[start:end], start, ) _core.register_with_iocp(handle) @@ -124,10 +123,7 @@ async def main(): try: async with _core.open_nursery() as nursery: nursery.start_soon( - _core.readinto_overlapped, - read_handle, - target, - name="xyz" + _core.readinto_overlapped, read_handle, target, name="xyz", ) await wait_all_tasks_blocked() nursery.cancel_scope.cancel() diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index ac090cb8de..b569371482 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -10,15 +10,11 @@ # See trio/tests/conftest.py for the other half of this from trio.tests.conftest import RUN_SLOW -slow = pytest.mark.skipif( - not RUN_SLOW, - reason="use --run-slow to run slow tests", -) + +slow = pytest.mark.skipif(not RUN_SLOW, reason="use --run-slow to run slow tests",) try: - s = stdlib_socket.socket( - stdlib_socket.AF_INET6, stdlib_socket.SOCK_STREAM, 0 - ) + s = stdlib_socket.socket(stdlib_socket.AF_INET6, stdlib_socket.SOCK_STREAM, 0) except OSError: # pragma: no cover # Some systems don't even support creating an IPv6 socket, let alone # binding it. (ex: Linux with 'ipv6.disable=1' in the kernel command line) @@ -30,7 +26,7 @@ can_create_ipv6 = True with s: try: - s.bind(('::1', 0)) + s.bind(("::1", 0)) except OSError: can_bind_ipv6 = False else: @@ -61,9 +57,7 @@ def gc_collect_harder(): @contextmanager def ignore_coroutine_never_awaited_warnings(): with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", message="coroutine '.*' was never awaited" - ) + warnings.filterwarnings("ignore", message="coroutine '.*' was never awaited") try: yield finally: @@ -79,14 +73,15 @@ def check_sequence_matches(seq, template): for pattern in template: if not isinstance(pattern, set): pattern = {pattern} - got = set(seq[i:i + len(pattern)]) + got = set(seq[i : i + len(pattern)]) assert got == pattern i += len(got) # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 skip_if_fbsd_pipes_broken = pytest.mark.skipif( - hasattr(os, "uname") and os.uname().sysname == "FreeBSD" + hasattr(os, "uname") + and os.uname().sysname == "FreeBSD" and os.uname().release[:4] < "12.2", - reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350" + reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350", ) diff --git a/trio/_deprecate.py b/trio/_deprecate.py index b1362cbc38..4f9f15ec35 100644 --- a/trio/_deprecate.py +++ b/trio/_deprecate.py @@ -117,9 +117,7 @@ def __getattr__(self, name): if instead is DeprecatedAttribute._not_set: instead = info.value thing = "{}.{}".format(self.__name__, name) - warn_deprecated( - thing, info.version, issue=info.issue, instead=instead - ) + warn_deprecated(thing, info.version, issue=info.issue, instead=instead) return info.value msg = "module '{}' has no attribute '{}'" diff --git a/trio/_deprecated_ssl_reexports.py b/trio/_deprecated_ssl_reexports.py index 35d22f49f4..31d5fdf9f6 100644 --- a/trio/_deprecated_ssl_reexports.py +++ b/trio/_deprecated_ssl_reexports.py @@ -12,11 +12,24 @@ # Always available from ssl import ( - cert_time_to_seconds, CertificateError, create_default_context, - DER_cert_to_PEM_cert, get_default_verify_paths, match_hostname, - PEM_cert_to_DER_cert, Purpose, SSLEOFError, SSLError, SSLSyscallError, - SSLZeroReturnError, AlertDescription, SSLErrorNumber, SSLSession, - VerifyFlags, VerifyMode, Options + cert_time_to_seconds, + CertificateError, + create_default_context, + DER_cert_to_PEM_cert, + get_default_verify_paths, + match_hostname, + PEM_cert_to_DER_cert, + Purpose, + SSLEOFError, + SSLError, + SSLSyscallError, + SSLZeroReturnError, + AlertDescription, + SSLErrorNumber, + SSLSession, + VerifyFlags, + VerifyMode, + Options, ) # Added in python 3.7 @@ -35,50 +48,98 @@ # (Real import is below) try: from ssl import ( - AF_INET, ALERT_DESCRIPTION_ACCESS_DENIED, + AF_INET, + ALERT_DESCRIPTION_ACCESS_DENIED, ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE, ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE, - ALERT_DESCRIPTION_BAD_CERTIFICATE, ALERT_DESCRIPTION_BAD_RECORD_MAC, + ALERT_DESCRIPTION_BAD_CERTIFICATE, + ALERT_DESCRIPTION_BAD_RECORD_MAC, ALERT_DESCRIPTION_CERTIFICATE_EXPIRED, ALERT_DESCRIPTION_CERTIFICATE_REVOKED, ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN, ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE, - ALERT_DESCRIPTION_CLOSE_NOTIFY, ALERT_DESCRIPTION_DECODE_ERROR, + ALERT_DESCRIPTION_CLOSE_NOTIFY, + ALERT_DESCRIPTION_DECODE_ERROR, ALERT_DESCRIPTION_DECOMPRESSION_FAILURE, - ALERT_DESCRIPTION_DECRYPT_ERROR, ALERT_DESCRIPTION_HANDSHAKE_FAILURE, + ALERT_DESCRIPTION_DECRYPT_ERROR, + ALERT_DESCRIPTION_HANDSHAKE_FAILURE, ALERT_DESCRIPTION_ILLEGAL_PARAMETER, ALERT_DESCRIPTION_INSUFFICIENT_SECURITY, - ALERT_DESCRIPTION_INTERNAL_ERROR, ALERT_DESCRIPTION_NO_RENEGOTIATION, - ALERT_DESCRIPTION_PROTOCOL_VERSION, ALERT_DESCRIPTION_RECORD_OVERFLOW, - ALERT_DESCRIPTION_UNEXPECTED_MESSAGE, ALERT_DESCRIPTION_UNKNOWN_CA, + ALERT_DESCRIPTION_INTERNAL_ERROR, + ALERT_DESCRIPTION_NO_RENEGOTIATION, + ALERT_DESCRIPTION_PROTOCOL_VERSION, + ALERT_DESCRIPTION_RECORD_OVERFLOW, + ALERT_DESCRIPTION_UNEXPECTED_MESSAGE, + ALERT_DESCRIPTION_UNKNOWN_CA, ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY, ALERT_DESCRIPTION_UNRECOGNIZED_NAME, ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE, ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION, - ALERT_DESCRIPTION_USER_CANCELLED, CERT_NONE, CERT_OPTIONAL, - CERT_REQUIRED, CHANNEL_BINDING_TYPES, HAS_ALPN, HAS_ECDH, - HAS_NEVER_CHECK_COMMON_NAME, HAS_NPN, HAS_SNI, OP_ALL, - OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION, OP_COOKIE_EXCHANGE, - OP_DONT_INSERT_EMPTY_FRAGMENTS, OP_EPHEMERAL_RSA, - OP_LEGACY_SERVER_CONNECT, OP_MICROSOFT_BIG_SSLV3_BUFFER, - OP_MICROSOFT_SESS_ID_BUG, OP_MSIE_SSLV2_RSA_PADDING, - OP_NETSCAPE_CA_DN_BUG, OP_NETSCAPE_CHALLENGE_BUG, + ALERT_DESCRIPTION_USER_CANCELLED, + CERT_NONE, + CERT_OPTIONAL, + CERT_REQUIRED, + CHANNEL_BINDING_TYPES, + HAS_ALPN, + HAS_ECDH, + HAS_NEVER_CHECK_COMMON_NAME, + HAS_NPN, + HAS_SNI, + OP_ALL, + OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION, + OP_COOKIE_EXCHANGE, + OP_DONT_INSERT_EMPTY_FRAGMENTS, + OP_EPHEMERAL_RSA, + OP_LEGACY_SERVER_CONNECT, + OP_MICROSOFT_BIG_SSLV3_BUFFER, + OP_MICROSOFT_SESS_ID_BUG, + OP_MSIE_SSLV2_RSA_PADDING, + OP_NETSCAPE_CA_DN_BUG, + OP_NETSCAPE_CHALLENGE_BUG, OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG, - OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG, OP_NO_QUERY_MTU, OP_PKCS1_CHECK_1, - OP_PKCS1_CHECK_2, OP_SSLEAY_080_CLIENT_DH_BUG, - OP_SSLREF2_REUSE_CERT_TYPE_BUG, OP_TLS_BLOCK_PADDING_BUG, - OP_TLS_D5_BUG, OP_TLS_ROLLBACK_BUG, SSL_ERROR_NONE, - SSL_ERROR_NO_SOCKET, OP_CIPHER_SERVER_PREFERENCE, OP_NO_COMPRESSION, - OP_NO_RENEGOTIATION, OP_NO_TICKET, OP_SINGLE_DH_USE, - OP_SINGLE_ECDH_USE, OPENSSL_VERSION_INFO, OPENSSL_VERSION_NUMBER, - OPENSSL_VERSION, PEM_FOOTER, PEM_HEADER, PROTOCOL_TLS_CLIENT, - PROTOCOL_TLS_SERVER, PROTOCOL_TLS, SO_TYPE, SOCK_STREAM, SOL_SOCKET, - SSL_ERROR_EOF, SSL_ERROR_INVALID_ERROR_CODE, SSL_ERROR_SSL, - SSL_ERROR_SYSCALL, SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_READ, - SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_X509_LOOKUP, - SSL_ERROR_ZERO_RETURN, VERIFY_CRL_CHECK_CHAIN, VERIFY_CRL_CHECK_LEAF, - VERIFY_DEFAULT, VERIFY_X509_STRICT, VERIFY_X509_TRUSTED_FIRST, - OP_ENABLE_MIDDLEBOX_COMPAT + OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG, + OP_NO_QUERY_MTU, + OP_PKCS1_CHECK_1, + OP_PKCS1_CHECK_2, + OP_SSLEAY_080_CLIENT_DH_BUG, + OP_SSLREF2_REUSE_CERT_TYPE_BUG, + OP_TLS_BLOCK_PADDING_BUG, + OP_TLS_D5_BUG, + OP_TLS_ROLLBACK_BUG, + SSL_ERROR_NONE, + SSL_ERROR_NO_SOCKET, + OP_CIPHER_SERVER_PREFERENCE, + OP_NO_COMPRESSION, + OP_NO_RENEGOTIATION, + OP_NO_TICKET, + OP_SINGLE_DH_USE, + OP_SINGLE_ECDH_USE, + OPENSSL_VERSION_INFO, + OPENSSL_VERSION_NUMBER, + OPENSSL_VERSION, + PEM_FOOTER, + PEM_HEADER, + PROTOCOL_TLS_CLIENT, + PROTOCOL_TLS_SERVER, + PROTOCOL_TLS, + SO_TYPE, + SOCK_STREAM, + SOL_SOCKET, + SSL_ERROR_EOF, + SSL_ERROR_INVALID_ERROR_CODE, + SSL_ERROR_SSL, + SSL_ERROR_SYSCALL, + SSL_ERROR_WANT_CONNECT, + SSL_ERROR_WANT_READ, + SSL_ERROR_WANT_WRITE, + SSL_ERROR_WANT_X509_LOOKUP, + SSL_ERROR_ZERO_RETURN, + VERIFY_CRL_CHECK_CHAIN, + VERIFY_CRL_CHECK_LEAF, + VERIFY_DEFAULT, + VERIFY_X509_STRICT, + VERIFY_X509_TRUSTED_FIRST, + OP_ENABLE_MIDDLEBOX_COMPAT, ) except ImportError: pass @@ -86,10 +147,11 @@ # Dynamically re-export whatever constants this particular Python happens to # have: import ssl as _stdlib_ssl + globals().update( { _name: getattr(_stdlib_ssl, _name) for _name in _stdlib_ssl.__dict__.keys() - if _name.isupper() and not _name.startswith('_') + if _name.isupper() and not _name.startswith("_") } ) diff --git a/trio/_deprecated_subprocess_reexports.py b/trio/_deprecated_subprocess_reexports.py index b91e28784a..2d1e4eed25 100644 --- a/trio/_deprecated_subprocess_reexports.py +++ b/trio/_deprecated_subprocess_reexports.py @@ -2,16 +2,27 @@ # Reexport constants and exceptions from the stdlib subprocess module from subprocess import ( - PIPE, STDOUT, DEVNULL, CalledProcessError, SubprocessError, TimeoutExpired, - CompletedProcess + PIPE, + STDOUT, + DEVNULL, + CalledProcessError, + SubprocessError, + TimeoutExpired, + CompletedProcess, ) # Windows only try: from subprocess import ( - STARTUPINFO, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, - SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, - CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP + STARTUPINFO, + STD_INPUT_HANDLE, + STD_OUTPUT_HANDLE, + STD_ERROR_HANDLE, + SW_HIDE, + STARTF_USESTDHANDLES, + STARTF_USESHOWWINDOW, + CREATE_NEW_CONSOLE, + CREATE_NEW_PROCESS_GROUP, ) except ImportError: pass @@ -19,10 +30,16 @@ # Windows 3.7+ only try: from subprocess import ( - ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, - HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, - REALTIME_PRIORITY_CLASS, CREATE_NO_WINDOW, DETACHED_PROCESS, - CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB + ABOVE_NORMAL_PRIORITY_CLASS, + BELOW_NORMAL_PRIORITY_CLASS, + HIGH_PRIORITY_CLASS, + IDLE_PRIORITY_CLASS, + NORMAL_PRIORITY_CLASS, + REALTIME_PRIORITY_CLASS, + CREATE_NO_WINDOW, + DETACHED_PROCESS, + CREATE_DEFAULT_ERROR_MODE, + CREATE_BREAKAWAY_FROM_JOB, ) except ImportError: pass diff --git a/trio/_file_io.py b/trio/_file_io.py index 1d3508875e..15e45711c5 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -8,43 +8,43 @@ # This list is also in the docs, make sure to keep them in sync _FILE_SYNC_ATTRS = { - 'closed', - 'encoding', - 'errors', - 'fileno', - 'isatty', - 'newlines', - 'readable', - 'seekable', - 'writable', + "closed", + "encoding", + "errors", + "fileno", + "isatty", + "newlines", + "readable", + "seekable", + "writable", # not defined in *IOBase: - 'buffer', - 'raw', - 'line_buffering', - 'closefd', - 'name', - 'mode', - 'getvalue', - 'getbuffer', + "buffer", + "raw", + "line_buffering", + "closefd", + "name", + "mode", + "getvalue", + "getbuffer", } # This list is also in the docs, make sure to keep them in sync _FILE_ASYNC_METHODS = { - 'flush', - 'read', - 'read1', - 'readall', - 'readinto', - 'readline', - 'readlines', - 'seek', - 'tell', - 'truncate', - 'write', - 'writelines', + "flush", + "read", + "read1", + "readall", + "readinto", + "readline", + "readlines", + "seek", + "tell", + "truncate", + "write", + "writelines", # not defined in *IOBase: - 'readinto1', - 'peek', + "readinto1", + "peek", } @@ -57,6 +57,7 @@ class AsyncIOWrapper(AsyncResource): wrapper, if they exist in the wrapped file object. """ + def __init__(self, file): self._wrapped = file @@ -88,9 +89,7 @@ async def wrapper(*args, **kwargs): def __dir__(self): attrs = set(super().__dir__()) attrs.update(a for a in _FILE_SYNC_ATTRS if hasattr(self.wrapped, a)) - attrs.update( - a for a in _FILE_ASYNC_METHODS if hasattr(self.wrapped, a) - ) + attrs.update(a for a in _FILE_ASYNC_METHODS if hasattr(self.wrapped, a)) return attrs def __aiter__(self): @@ -131,13 +130,13 @@ async def aclose(self): async def open_file( file, - mode='r', + mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, - opener=None + opener=None, ): """Asynchronous version of :func:`io.open`. @@ -158,8 +157,7 @@ async def open_file( """ _file = wrap_file( await trio.to_thread.run_sync( - io.open, file, mode, buffering, encoding, errors, newline, closefd, - opener + io.open, file, mode, buffering, encoding, errors, newline, closefd, opener, ) ) return _file @@ -182,13 +180,14 @@ def wrap_file(file): assert await async_file.read() == 'asdf' """ + def has(attr): return hasattr(file, attr) and callable(getattr(file, attr)) - if not (has('close') and (has('read') or has('write'))): + if not (has("close") and (has("read") or has("write"))): raise TypeError( - '{} does not implement required duck-file methods: ' - 'close and (read or write)'.format(file) + "{} does not implement required duck-file methods: " + "close and (read or write)".format(file) ) return AsyncIOWrapper(file) diff --git a/trio/_highlevel_generic.py b/trio/_highlevel_generic.py index 7cdc0c75d1..d4091942db 100644 --- a/trio/_highlevel_generic.py +++ b/trio/_highlevel_generic.py @@ -37,9 +37,7 @@ async def aclose_forcefully(resource): @attr.s(eq=False, hash=False) -class StapledStream( - HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0 -): +class StapledStream(HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0): """This class `staples `__ together two unidirectional streams to make single bidirectional stream. @@ -73,6 +71,7 @@ class StapledStream( is delegated to this object. """ + send_stream = attr.ib() receive_stream = attr.ib() diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index 6ac44db43b..8e399a61b5 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -39,7 +39,7 @@ def _compute_backlog(backlog): # Many systems (Linux, BSDs, ...) store the backlog in a uint16 and are # missing overflow protection, so we apply our own overflow protection. # https://github.com/golang/go/issues/5030 - return min(backlog, 0xffff) + return min(backlog, 0xFFFF) async def open_tcp_listeners(port, *, host=None, backlog=None): @@ -92,10 +92,7 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): backlog = _compute_backlog(backlog) addresses = await tsocket.getaddrinfo( - host, - port, - type=tsocket.SOCK_STREAM, - flags=tsocket.AI_PASSIVE, + host, port, type=tsocket.SOCK_STREAM, flags=tsocket.AI_PASSIVE, ) listeners = [] @@ -119,14 +116,10 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): try: # See https://github.com/python-trio/trio/issues/39 if sys.platform != "win32": - sock.setsockopt( - tsocket.SOL_SOCKET, tsocket.SO_REUSEADDR, 1 - ) + sock.setsockopt(tsocket.SOL_SOCKET, tsocket.SO_REUSEADDR, 1) if family == tsocket.AF_INET6: - sock.setsockopt( - tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, 1 - ) + sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, 1) await sock.bind(sockaddr) sock.listen(backlog) @@ -144,7 +137,7 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): raise OSError( errno.EAFNOSUPPORT, "This system doesn't support any of the kinds of " - "socket that that address could use" + "socket that that address could use", ) from trio.MultiError(unsupported_address_families) return listeners @@ -157,7 +150,7 @@ async def serve_tcp( host=None, backlog=None, handler_nursery=None, - task_status=trio.TASK_STATUS_IGNORED + task_status=trio.TASK_STATUS_IGNORED, ): """Listen for incoming TCP connections, and for each one start a task running ``handler(stream)``. @@ -224,8 +217,5 @@ async def serve_tcp( """ listeners = await trio.open_tcp_listeners(port, host=host, backlog=backlog) await trio.serve_listeners( - handler, - listeners, - handler_nursery=handler_nursery, - task_status=task_status + handler, listeners, handler_nursery=handler_nursery, task_status=task_status, ) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 11a1e1846f..847fac6a96 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -169,7 +169,7 @@ async def open_tcp_stream( port, *, # No trailing comma b/c bpo-9232 (fixed in py36) - happy_eyeballs_delay=DEFAULT_DELAY + happy_eyeballs_delay=DEFAULT_DELAY, ): """Connect to the given host and port over TCP. diff --git a/trio/_highlevel_open_unix_stream.py b/trio/_highlevel_open_unix_stream.py index 87462c1232..eea1e77ffc 100644 --- a/trio/_highlevel_open_unix_stream.py +++ b/trio/_highlevel_open_unix_stream.py @@ -6,6 +6,7 @@ try: from trio.socket import AF_UNIX + has_unix = True except ImportError: has_unix = False diff --git a/trio/_highlevel_serve_listeners.py b/trio/_highlevel_serve_listeners.py index 0a46be780a..55216c85c5 100644 --- a/trio/_highlevel_serve_listeners.py +++ b/trio/_highlevel_serve_listeners.py @@ -39,7 +39,7 @@ async def _serve_one_listener(listener, handler_nursery, handler): errno.errorcode[exc.errno], os.strerror(exc.errno), SLEEP_TIME, - exc_info=True + exc_info=True, ) await trio.sleep(SLEEP_TIME) else: @@ -49,11 +49,7 @@ async def _serve_one_listener(listener, handler_nursery, handler): async def serve_listeners( - handler, - listeners, - *, - handler_nursery=None, - task_status=trio.TASK_STATUS_IGNORED + handler, listeners, *, handler_nursery=None, task_status=trio.TASK_STATUS_IGNORED, ): r"""Listen for incoming connections on ``listeners``, and for each one start a task running ``handler(stream)``. @@ -118,9 +114,7 @@ async def serve_listeners( if handler_nursery is None: handler_nursery = nursery for listener in listeners: - nursery.start_soon( - _serve_one_listener, listener, handler_nursery, handler - ) + nursery.start_soon(_serve_one_listener, listener, handler_nursery, handler) # The listeners are already queueing connections when we're called, # but we wait until the end to call started() just in case we get an # error or whatever. diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index abcf951028..a707f0cac6 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -28,18 +28,14 @@ def _translate_socket_errors_to_stream_errors(): yield except OSError as exc: if exc.errno in _closed_stream_errnos: - raise trio.ClosedResourceError( - "this socket was already closed" - ) from None + raise trio.ClosedResourceError("this socket was already closed") from None else: raise trio.BrokenResourceError( "socket connection broken: {}".format(exc) ) from exc -class SocketStream( - HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0 -): +class SocketStream(HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0): """An implementation of the :class:`trio.abc.HalfCloseableStream` interface based on a raw network socket. @@ -62,6 +58,7 @@ class SocketStream( The Trio socket object that this stream wraps. """ + def __init__(self, socket): if not isinstance(socket, tsocket.SocketType): raise TypeError("SocketStream requires a Trio socket object") @@ -92,9 +89,7 @@ def __init__(self, socket): # http://devstreaming.apple.com/videos/wwdc/2015/719ui2k57m/719/719_your_app_and_next_generation_networks.pdf?dl=1 # ). The theory is that you want it to be bandwidth * # rescheduling interval. - self.setsockopt( - tsocket.IPPROTO_TCP, tsocket.TCP_NOTSENT_LOWAT, 2**14 - ) + self.setsockopt(tsocket.IPPROTO_TCP, tsocket.TCP_NOTSENT_LOWAT, 2 ** 14) except OSError: pass @@ -106,9 +101,7 @@ async def send_all(self, data): with memoryview(data) as data: if not data: if self.socket.fileno() == -1: - raise trio.ClosedResourceError( - "socket was already closed" - ) + raise trio.ClosedResourceError("socket was already closed") await trio.lowlevel.checkpoint() return total_sent = 0 @@ -322,9 +315,7 @@ def getsockopt(self, level, option, buffersize=0): pass -class SocketListener( - Listener[SocketStream], metaclass=SubclassingDeprecatedIn_v0_15_0 -): +class SocketListener(Listener[SocketStream], metaclass=SubclassingDeprecatedIn_v0_15_0): """A :class:`~trio.abc.Listener` that uses a listening socket to accept incoming connections as :class:`SocketStream` objects. @@ -340,15 +331,14 @@ class SocketListener( The Trio socket object that this stream wraps. """ + def __init__(self, socket): if not isinstance(socket, tsocket.SocketType): raise TypeError("SocketListener requires a Trio socket object") if socket.type != tsocket.SOCK_STREAM: raise ValueError("SocketListener requires a SOCK_STREAM socket") try: - listening = socket.getsockopt( - tsocket.SOL_SOCKET, tsocket.SO_ACCEPTCONN - ) + listening = socket.getsockopt(tsocket.SOL_SOCKET, tsocket.SO_ACCEPTCONN) except OSError: # SO_ACCEPTCONN fails on macOS; we just have to trust the user. pass diff --git a/trio/_highlevel_ssl_helpers.py b/trio/_highlevel_ssl_helpers.py index fc604d9286..61c463c65e 100644 --- a/trio/_highlevel_ssl_helpers.py +++ b/trio/_highlevel_ssl_helpers.py @@ -20,7 +20,7 @@ async def open_ssl_over_tcp_stream( https_compatible=False, ssl_context=None, # No trailing comma b/c bpo-9232 (fixed in py36) - happy_eyeballs_delay=DEFAULT_DELAY + happy_eyeballs_delay=DEFAULT_DELAY, ): """Make a TLS-encrypted Connection to the given host and port over TCP. @@ -49,9 +49,7 @@ async def open_ssl_over_tcp_stream( """ tcp_stream = await trio.open_tcp_stream( - host, - port, - happy_eyeballs_delay=happy_eyeballs_delay, + host, port, happy_eyeballs_delay=happy_eyeballs_delay, ) if ssl_context is None: ssl_context = ssl.create_default_context() @@ -78,15 +76,10 @@ async def open_ssl_over_tcp_listeners( backlog (int or None): See :func:`open_tcp_listeners` for details. """ - tcp_listeners = await trio.open_tcp_listeners( - port, host=host, backlog=backlog - ) + tcp_listeners = await trio.open_tcp_listeners(port, host=host, backlog=backlog) ssl_listeners = [ - trio.SSLListener( - tcp_listener, - ssl_context, - https_compatible=https_compatible, - ) for tcp_listener in tcp_listeners + trio.SSLListener(tcp_listener, ssl_context, https_compatible=https_compatible,) + for tcp_listener in tcp_listeners ] return ssl_listeners @@ -100,7 +93,7 @@ async def serve_ssl_over_tcp( https_compatible=False, backlog=None, handler_nursery=None, - task_status=trio.TASK_STATUS_IGNORED + task_status=trio.TASK_STATUS_IGNORED, ): """Listen for incoming TCP connections, and for each one start a task running ``handler(stream)``. @@ -154,11 +147,8 @@ async def serve_ssl_over_tcp( ssl_context, host=host, https_compatible=https_compatible, - backlog=backlog + backlog=backlog, ) await trio.serve_listeners( - handler, - listeners, - handler_nursery=handler_nursery, - task_status=task_status + handler, listeners, handler_nursery=handler_nursery, task_status=task_status, ) diff --git a/trio/_path.py b/trio/_path.py index 75f1fab615..095bdc779f 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -88,7 +88,7 @@ def __init__(cls, name, bases, attrs): def generate_forwards(cls, attrs): # forward functions of _forwards for attr_name, attr in cls._forwards.__dict__.items(): - if attr_name.startswith('_') or attr_name in attrs: + if attr_name.startswith("_") or attr_name in attrs: continue if isinstance(attr, property): @@ -103,7 +103,7 @@ def generate_wraps(cls, attrs): # generate wrappers for functions of _wraps for attr_name, attr in cls._wraps.__dict__.items(): # .z. exclude cls._wrap_iter - if attr_name.startswith('_') or attr_name in attrs: + if attr_name.startswith("_") or attr_name in attrs: continue if isinstance(attr, classmethod): wrapper = classmethod_wrapper_factory(cls, attr_name) @@ -138,18 +138,18 @@ class Path(metaclass=AsyncAutoWrapperType): _wraps = pathlib.Path _forwards = pathlib.PurePath _forward_magic = [ - '__str__', - '__bytes__', - '__truediv__', - '__rtruediv__', - '__eq__', - '__lt__', - '__le__', - '__gt__', - '__ge__', - '__hash__', + "__str__", + "__bytes__", + "__truediv__", + "__rtruediv__", + "__eq__", + "__lt__", + "__le__", + "__gt__", + "__ge__", + "__hash__", ] - _wrap_iter = ['glob', 'rglob', 'iterdir'] + _wrap_iter = ["glob", "rglob", "iterdir"] def __init__(self, *args): self._wrapped = pathlib.Path(*args) @@ -164,7 +164,7 @@ def __dir__(self): return super().__dir__() + self._forward def __repr__(self): - return 'trio.Path({})'.format(repr(str(self))) + return "trio.Path({})".format(repr(str(self))) def __fspath__(self): return os.fspath(self._wrapped) diff --git a/trio/_socket.py b/trio/_socket.py index 5518335b44..a7b5303c21 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -150,8 +150,10 @@ async def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): # with the _NUMERIC_ONLY flags set, and then only spawn a thread if that # fails with EAI_NONAME: def numeric_only_failure(exc): - return isinstance(exc, _stdlib_socket.gaierror) and \ - exc.errno == _stdlib_socket.EAI_NONAME + return ( + isinstance(exc, _stdlib_socket.gaierror) + and exc.errno == _stdlib_socket.EAI_NONAME + ) async with _try_sync(numeric_only_failure): return _stdlib_socket.getaddrinfo( @@ -185,7 +187,7 @@ def numeric_only_failure(exc): type, proto, flags, - cancellable=True + cancellable=True, ) @@ -266,7 +268,7 @@ def socket( family=_stdlib_socket.AF_INET, type=_stdlib_socket.SOCK_STREAM, proto=0, - fileno=None + fileno=None, ): """Create a new Trio socket, like :func:`socket.socket`. @@ -279,9 +281,7 @@ def socket( if sf is not None: return sf.socket(family, type, proto) else: - family, type, proto = _sniff_sockopts_for_fileno( - family, type, proto, fileno - ) + family, type, proto = _sniff_sockopts_for_fileno(family, type, proto, fileno) stdlib_socket = _stdlib_socket.socket(family, type, proto, fileno) return from_stdlib_socket(stdlib_socket) @@ -296,6 +296,7 @@ def _sniff_sockopts_for_fileno(family, type, proto, fileno): if not _sys.platform == "linux": return family, type, proto from socket import SO_DOMAIN, SO_PROTOCOL, SOL_SOCKET, SO_TYPE + sockobj = _stdlib_socket.socket(family, type, proto, fileno=fileno) try: family = sockobj.getsockopt(SOL_SOCKET, SO_DOMAIN) @@ -338,10 +339,10 @@ def _make_simple_sock_method_wrapper(methname, wait_fn, maybe_avail=False): async def wrapper(self, *args, **kwargs): return await self._nonblocking_helper(fn, args, kwargs, wait_fn) - wrapper.__doc__ = ( - """Like :meth:`socket.socket.{}`, but async. + wrapper.__doc__ = """Like :meth:`socket.socket.{}`, but async. - """.format(methname) + """.format( + methname ) if maybe_avail: wrapper.__doc__ += ( @@ -451,7 +452,8 @@ async def bind(self, address): address = await self._resolve_local_address(address) if ( hasattr(_stdlib_socket, "AF_UNIX") - and self.family == _stdlib_socket.AF_UNIX and address[0] + and self.family == _stdlib_socket.AF_UNIX + and address[0] ): # Use a thread for the filesystem traversal (unless it's an # abstract domain socket) @@ -497,8 +499,7 @@ async def _resolve_address(self, address, flags): elif self._sock.family == _stdlib_socket.AF_INET6: if not isinstance(address, tuple) or not 2 <= len(address) <= 4: raise ValueError( - "address should be a (host, port, [flowinfo, [scopeid]]) " - "tuple" + "address should be a (host, port, [flowinfo, [scopeid]]) " "tuple" ) elif self._sock.family == _stdlib_socket.AF_UNIX: await trio.lowlevel.checkpoint() @@ -522,9 +523,7 @@ async def _resolve_address(self, address, flags): # no ipv6. # flags |= AI_ADDRCONFIG if self._sock.family == _stdlib_socket.AF_INET6: - if not self._sock.getsockopt( - IPPROTO_IPV6, _stdlib_socket.IPV6_V6ONLY - ): + if not self._sock.getsockopt(IPPROTO_IPV6, _stdlib_socket.IPV6_V6ONLY): flags |= _stdlib_socket.AI_V4MAPPED gai_res = await getaddrinfo( host, port, self._sock.family, self.type, self._sock.proto, flags @@ -669,9 +668,7 @@ async def connect(self, address): self._sock.close() raise # Okay, the connect finished, but it might have failed: - err = self._sock.getsockopt( - _stdlib_socket.SOL_SOCKET, _stdlib_socket.SO_ERROR - ) + err = self._sock.getsockopt(_stdlib_socket.SOL_SOCKET, _stdlib_socket.SO_ERROR) if err != 0: raise OSError(err, "Error in connect: " + _os.strerror(err)) @@ -685,17 +682,13 @@ async def connect(self, address): # recv_into ################################################################ - recv_into = _make_simple_sock_method_wrapper( - "recv_into", _core.wait_readable - ) + recv_into = _make_simple_sock_method_wrapper("recv_into", _core.wait_readable) ################################################################ # recvfrom ################################################################ - recvfrom = _make_simple_sock_method_wrapper( - "recvfrom", _core.wait_readable - ) + recvfrom = _make_simple_sock_method_wrapper("recvfrom", _core.wait_readable) ################################################################ # recvfrom_into diff --git a/trio/_ssl.py b/trio/_ssl.py index 12182c58af..41242f45c2 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -328,14 +328,12 @@ def __init__( server_hostname=None, server_side=False, https_compatible=False, - max_refill_bytes="unused and deprecated" + max_refill_bytes="unused and deprecated", ): self.transport_stream = transport_stream self._state = _State.OK if max_refill_bytes != "unused and deprecated": - warn_deprecated( - "max_refill_bytes=...", "0.12.0", issue=959, instead=None - ) + warn_deprecated("max_refill_bytes=...", "0.12.0", issue=959, instead=None) self._https_compatible = https_compatible self._outgoing = _stdlib_ssl.MemoryBIO() self._delayed_outgoing = None @@ -344,7 +342,7 @@ def __init__( self._incoming, self._outgoing, server_side=server_side, - server_hostname=server_hostname + server_hostname=server_hostname, ) # Tracks whether we've already done the initial handshake self._handshook = _Once(self._do_handshake) @@ -429,9 +427,7 @@ def _check_status(self): # comments, though, just make sure to think carefully if you ever have to # touch it. The big comment at the top of this file will help explain # too. - async def _retry( - self, fn, *args, ignore_want_read=False, is_handshake=False - ): + async def _retry(self, fn, *args, ignore_want_read=False, is_handshake=False): await trio.lowlevel.checkpoint_if_cancelled() yielded = False finished = False @@ -495,7 +491,9 @@ async def _retry( # # https://github.com/python-trio/trio/issues/819#issuecomment-517529763 if ( - is_handshake and not want_read and self._ssl_object.server_side + is_handshake + and not want_read + and self._ssl_object.server_side and self._ssl_object.version() == "TLSv1.3" ): assert self._delayed_outgoing is None @@ -664,11 +662,9 @@ async def receive_some(self, max_bytes=None): # For some reason, EOF before handshake sometimes raises # SSLSyscallError instead of SSLEOFError (e.g. on my linux # laptop, but not on appveyor). Thanks openssl. - if ( - self._https_compatible and isinstance( - exc.__cause__, - (_stdlib_ssl.SSLEOFError, _stdlib_ssl.SSLSyscallError) - ) + if self._https_compatible and isinstance( + exc.__cause__, + (_stdlib_ssl.SSLEOFError, _stdlib_ssl.SSLSyscallError), ): await trio.lowlevel.checkpoint() return b"" @@ -678,9 +674,7 @@ async def receive_some(self, max_bytes=None): # If we somehow have more data already in our pending buffer # than the estimate receive size, bump up our size a bit for # this read only. - max_bytes = max( - self._estimated_receive_size, self._incoming.pending - ) + max_bytes = max(self._estimated_receive_size, self._incoming.pending) else: max_bytes = _operator.index(max_bytes) if max_bytes < 1: @@ -693,9 +687,8 @@ async def receive_some(self, max_bytes=None): # BROKEN. But that's actually fine, because after getting an # EOF on TLS then the only thing you can do is close the # stream, and closing doesn't care about the state. - if ( - self._https_compatible - and isinstance(exc.__cause__, _stdlib_ssl.SSLEOFError) + if self._https_compatible and isinstance( + exc.__cause__, _stdlib_ssl.SSLEOFError ): await trio.lowlevel.checkpoint() return b"" @@ -740,8 +733,7 @@ async def unwrap(self): ``transport_stream.receive_some(...)``. """ - with self._outer_recv_conflict_detector, \ - self._outer_send_conflict_detector: + with self._outer_recv_conflict_detector, self._outer_send_conflict_detector: self._check_status() await self._handshook.ensure(checkpoint=False) await self._retry(self._ssl_object.unwrap) @@ -824,9 +816,7 @@ async def aclose(self): # going to be able to do a clean shutdown. If that happens, we'll # just do an unclean shutdown. try: - await self._retry( - self._ssl_object.unwrap, ignore_want_read=True - ) + await self._retry(self._ssl_object.unwrap, ignore_want_read=True) except (trio.BrokenResourceError, trio.BusyResourceError): pass except: @@ -882,9 +872,7 @@ async def wait_send_all_might_not_block(self): await self.transport_stream.wait_send_all_might_not_block() -class SSLListener( - Listener[SSLStream], metaclass=SubclassingDeprecatedIn_v0_15_0 -): +class SSLListener(Listener[SSLStream], metaclass=SubclassingDeprecatedIn_v0_15_0): """A :class:`~trio.abc.Listener` for SSL/TLS-encrypted servers. :class:`SSLListener` wraps around another Listener, and converts @@ -905,18 +893,17 @@ class SSLListener( passed to ``__init__``. """ + def __init__( self, transport_listener, ssl_context, *, https_compatible=False, - max_refill_bytes="unused and deprecated" + max_refill_bytes="unused and deprecated", ): if max_refill_bytes != "unused and deprecated": - warn_deprecated( - "max_refill_bytes=...", "0.12.0", issue=959, instead=None - ) + warn_deprecated("max_refill_bytes=...", "0.12.0", issue=959, instead=None) self.transport_listener = transport_listener self._ssl_context = ssl_context self._https_compatible = https_compatible diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 16a869f8ef..ac51915eb0 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -9,8 +9,9 @@ from ._highlevel_generic import StapledStream from ._sync import Lock from ._subprocess_platform import ( - wait_child_exiting, create_pipe_to_child_stdin, - create_pipe_from_child_output + wait_child_exiting, + create_pipe_to_child_stdin, + create_pipe_from_child_output, ) from ._util import NoPublicConstructor import trio @@ -23,6 +24,7 @@ except ImportError: if sys.platform == "linux": import ctypes + _cdll_for_pidfd_open = ctypes.CDLL(None, use_errno=True) _cdll_for_pidfd_open.syscall.restype = ctypes.c_long # pid and flags are actually int-sized, but the syscall() function @@ -45,6 +47,7 @@ def pidfd_open(fd, flags): err = ctypes.get_errno() raise OSError(err, os.strerror(err)) return result + else: can_try_pidfd_open = False @@ -312,12 +315,11 @@ async def open_process( specified command could not be found. """ - for key in ('universal_newlines', 'text', 'encoding', 'errors', 'bufsize'): + for key in ("universal_newlines", "text", "encoding", "errors", "bufsize"): if options.get(key): raise TypeError( "trio.Process only supports communicating over " - "unbuffered byte streams; the '{}' option is not supported" - .format(key) + "unbuffered byte streams; the '{}' option is not supported".format(key) ) if os.name == "posix": @@ -361,7 +363,7 @@ async def open_process( stdin=stdin, stdout=stdout, stderr=stderr, - **options + **options, ) ) finally: @@ -382,9 +384,7 @@ async def _windows_deliver_cancel(p): try: p.terminate() except OSError as exc: - warnings.warn( - RuntimeWarning(f"TerminateProcess on {p!r} failed with: {exc!r}") - ) + warnings.warn(RuntimeWarning(f"TerminateProcess on {p!r} failed with: {exc!r}")) async def _posix_deliver_cancel(p): @@ -401,9 +401,7 @@ async def _posix_deliver_cancel(p): p.kill() except OSError as exc: warnings.warn( - RuntimeWarning( - f"tried to kill process {p!r}, but failed with: {exc!r}" - ) + RuntimeWarning(f"tried to kill process {p!r}, but failed with: {exc!r}") ) @@ -415,7 +413,7 @@ async def run_process( capture_stderr=False, check=True, deliver_cancel=None, - **options + **options, ): """Run ``command`` in a subprocess, wait for it to complete, and return a :class:`subprocess.CompletedProcess` instance describing @@ -638,6 +636,4 @@ async def killer(): proc.returncode, proc.args, output=stdout, stderr=stderr ) else: - return subprocess.CompletedProcess( - proc.args, proc.returncode, stdout, stderr - ) + return subprocess.CompletedProcess(proc.args, proc.returncode, stdout, stderr) diff --git a/trio/_subprocess_platform/kqueue.py b/trio/_subprocess_platform/kqueue.py index 837b556fed..17e2df5c6f 100644 --- a/trio/_subprocess_platform/kqueue.py +++ b/trio/_subprocess_platform/kqueue.py @@ -13,16 +13,11 @@ async def wait_child_exiting(process: "_subprocess.Process") -> None: KQ_NOTE_EXIT = 0x80000000 make_event = lambda flags: select.kevent( - process.pid, - filter=select.KQ_FILTER_PROC, - flags=flags, - fflags=KQ_NOTE_EXIT + process.pid, filter=select.KQ_FILTER_PROC, flags=flags, fflags=KQ_NOTE_EXIT, ) try: - kqueue.control( - [make_event(select.KQ_EV_ADD | select.KQ_EV_ONESHOT)], 0 - ) + kqueue.control([make_event(select.KQ_EV_ADD | select.KQ_EV_ONESHOT)], 0) except ProcessLookupError: # pragma: no cover # This can supposedly happen if the process is in the process # of exiting, and it can even be the case that kqueue says the diff --git a/trio/_subprocess_platform/waitid.py b/trio/_subprocess_platform/waitid.py index 030c546f88..81fdf88884 100644 --- a/trio/_subprocess_platform/waitid.py +++ b/trio/_subprocess_platform/waitid.py @@ -13,10 +13,12 @@ def sync_wait_reapable(pid): waitid(os.P_PID, pid, os.WEXITED | os.WNOWAIT) + except ImportError: # pypy doesn't define os.waitid so we need to pull it out ourselves # using cffi: https://bitbucket.org/pypy/pypy/issues/2922/ import cffi + waitid_ffi = cffi.FFI() # Believe it or not, siginfo_t starts with fields in the @@ -43,7 +45,7 @@ def sync_wait_reapable(pid): def sync_wait_reapable(pid): P_PID = 1 WEXITED = 0x00000004 - if sys.platform == 'darwin': # pragma: no cover + if sys.platform == "darwin": # pragma: no cover # waitid() is not exposed on Python on Darwin but does # work through CFFI; note that we typically won't get # here since Darwin also defines kqueue @@ -75,10 +77,7 @@ async def _waitid_system_task(pid: int, event: Event) -> None: try: await to_thread_run_sync( - sync_wait_reapable, - pid, - cancellable=True, - limiter=waitid_limiter, + sync_wait_reapable, pid, cancellable=True, limiter=waitid_limiter, ) except OSError: # If waitid fails, waitpid will fail too, so it still makes diff --git a/trio/_sync.py b/trio/_sync.py index 068a5684f4..212c7927a3 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -156,6 +156,7 @@ class CapacityLimiter(metaclass=SubclassingDeprecatedIn_v0_15_0): just borrowed and then put back. """ + def __init__(self, total_tokens): self._lot = ParkingLot() self._borrowers = set() @@ -166,11 +167,8 @@ def __init__(self, total_tokens): assert self._total_tokens == total_tokens def __repr__(self): - return ( - "".format( - id(self), len(self._borrowers), self._total_tokens, - len(self._lot) - ) + return "".format( + id(self), len(self._borrowers), self._total_tokens, len(self._lot) ) @property @@ -190,9 +188,7 @@ def total_tokens(self): @total_tokens.setter def total_tokens(self, new_total_tokens): - if not isinstance( - new_total_tokens, int - ) and new_total_tokens != math.inf: + if not isinstance(new_total_tokens, int) and new_total_tokens != math.inf: raise TypeError("total_tokens must be an int or math.inf") if new_total_tokens < 1: raise ValueError("total_tokens must be >= 1") @@ -321,8 +317,7 @@ def release_on_behalf_of(self, borrower): """ if borrower not in self._borrowers: raise RuntimeError( - "this borrower isn't holding any of this CapacityLimiter's " - "tokens" + "this borrower isn't holding any of this CapacityLimiter's " "tokens" ) self._borrowers.remove(borrower) self._wake_waiters() @@ -381,6 +376,7 @@ class Semaphore(metaclass=SubclassingDeprecatedIn_v0_15_0): ``max_value``. """ + def __init__(self, initial_value, *, max_value=None): if not isinstance(initial_value, int): raise TypeError("initial_value must be an int") @@ -404,10 +400,8 @@ def __repr__(self): max_value_str = "" else: max_value_str = ", max_value={}".format(self._max_value) - return ( - "".format( - self._value, max_value_str, id(self) - ) + return "".format( + self._value, max_value_str, id(self) ) @property @@ -502,10 +496,8 @@ def __repr__(self): else: s1 = "unlocked" s2 = "" - return ( - "<{} {} object at {:#x}{}>".format( - s1, self.__class__.__name__, id(self), s2 - ) + return "<{} {} object at {:#x}{}>".format( + s1, self.__class__.__name__, id(self), s2 ) def locked(self): @@ -580,9 +572,7 @@ def statistics(self): """ return _LockStatistics( - locked=self.locked(), - owner=self._owner, - tasks_waiting=len(self._lot), + locked=self.locked(), owner=self._owner, tasks_waiting=len(self._lot), ) @@ -684,6 +674,7 @@ class Condition(metaclass=SubclassingDeprecatedIn_v0_15_0): and used. """ + def __init__(self, lock=None): if lock is None: lock = Lock() @@ -795,6 +786,5 @@ def statistics(self): """ return _ConditionStatistics( - tasks_waiting=len(self._lot), - lock_statistics=self._lock.statistics(), + tasks_waiting=len(self._lot), lock_statistics=self._lock.statistics(), ) diff --git a/trio/_threads.py b/trio/_threads.py index c03a353789..92e2b5dc0f 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -9,7 +9,12 @@ import trio from ._sync import CapacityLimiter -from ._core import enable_ki_protection, disable_ki_protection, RunVar, TrioToken +from ._core import ( + enable_ki_protection, + disable_ki_protection, + RunVar, + TrioToken, +) from ._util import coroutine_or_error # Global due to Threading API, thread local storage for trio token @@ -291,10 +296,7 @@ def worker_thread_fn(trio_token): # this case shouldn't block process exit. current_trio_token = trio.lowlevel.current_trio_token() thread = threading.Thread( - target=worker_thread_fn, - args=(current_trio_token,), - name=name, - daemon=True + target=worker_thread_fn, args=(current_trio_token,), name=name, daemon=True, ) thread.start() except: @@ -338,9 +340,7 @@ def _run_fn_as_system_task(cb, fn, *args, trio_token=None): except RuntimeError: pass else: - raise RuntimeError( - "this is a blocking function; call it from a thread" - ) + raise RuntimeError("this is a blocking function; call it from a thread") q = stdlib_queue.Queue() trio_token.run_sync_soon(cb, q, fn, args) @@ -380,6 +380,7 @@ def from_thread_run(afn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ + def callback(q, afn, args): @disable_ki_protection async def unprotected_afn(): @@ -424,6 +425,7 @@ def from_thread_run_sync(fn, *args, trio_token=None): "foreign" thread, spawned using some other framework, and still want to enter Trio. """ + def callback(q, fn, args): @disable_ki_protection def unprotected_fn(): diff --git a/trio/_timeouts.py b/trio/_timeouts.py index 9bfefe4b03..517f344bf3 100644 --- a/trio/_timeouts.py +++ b/trio/_timeouts.py @@ -37,9 +37,7 @@ async def sleep_forever(): Equivalent to calling ``await sleep(math.inf)``. """ - await trio.lowlevel.wait_task_rescheduled( - lambda _: trio.lowlevel.Abort.SUCCEEDED - ) + await trio.lowlevel.wait_task_rescheduled(lambda _: trio.lowlevel.Abort.SUCCEEDED) async def sleep_until(deadline): diff --git a/trio/_tools/gen_exports.py b/trio/_tools/gen_exports.py index 9dda121b51..33cc736ae1 100755 --- a/trio/_tools/gen_exports.py +++ b/trio/_tools/gen_exports.py @@ -13,7 +13,7 @@ from textwrap import indent -PREFIX = '_generated' +PREFIX = "_generated" HEADER = """# *********************************************************** # ******* WARNING: AUTOGENERATED! ALL EDITS WILL BE LOST ****** @@ -39,10 +39,7 @@ def is_function(node): """Check if the AST node is either a function or an async function """ - if ( - isinstance(node, ast.FunctionDef) - or isinstance(node, ast.AsyncFunctionDef) - ): + if isinstance(node, ast.FunctionDef) or isinstance(node, ast.AsyncFunctionDef): return True return False @@ -115,7 +112,7 @@ def gen_public_wrappers_source(source_path: Path, lookup_path: str) -> str: del method.body[1:] # Create the function definition including the body - func = astor.to_source(method, indent_with=' ' * 4) + func = astor.to_source(method, indent_with=" " * 4) # Create export function body template = TEMPLATE.format( @@ -125,7 +122,7 @@ def gen_public_wrappers_source(source_path: Path, lookup_path: str) -> str: ) # Assemble function definition arguments and body - snippet = func + indent(template, ' ' * 4) + snippet = func + indent(template, " " * 4) # Append the snippet to the corresponding module generated.append(snippet) @@ -169,20 +166,17 @@ def process(sources_and_lookups, *, do_test): # doesn't collect coverage. def main(): # pragma: no cover parser = argparse.ArgumentParser( - description='Generate python code for public api wrappers' + description="Generate python code for public api wrappers" ) parser.add_argument( - '--test', - '-t', - action='store_true', - help='test if code is still up to date' + "--test", "-t", action="store_true", help="test if code is still up to date", ) parsed_args = parser.parse_args() source_root = Path.cwd() # Double-check we found the right directory assert (source_root / "LICENSE").exists() - core = source_root / 'trio/_core' + core = source_root / "trio/_core" to_wrap = [ (core / "_run.py", "runner"), (core / "_io_windows.py", "runner.io_manager"), @@ -193,5 +187,5 @@ def main(): # pragma: no cover process(to_wrap, do_test=parsed_args.test) -if __name__ == '__main__': # pragma: no cover +if __name__ == "__main__": # pragma: no cover main() diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index f2afe8d2b1..cb63009249 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -105,6 +105,7 @@ class FdStream(Stream, metaclass=SubclassingDeprecatedIn_v0_15_0): Returns: A new `FdStream` object. """ + def __init__(self, fd: int): self._fd_holder = _FdHolder(fd) self._send_conflict_detector = ConflictDetector( @@ -130,9 +131,7 @@ async def send_all(self, data: bytes): try: sent += os.write(self._fd_holder.fd, remaining) except BlockingIOError: - await trio.lowlevel.wait_writable( - self._fd_holder.fd - ) + await trio.lowlevel.wait_writable(self._fd_holder.fd) except OSError as e: if e.errno == errno.EBADF: raise trio.ClosedResourceError( diff --git a/trio/_util.py b/trio/_util.py index 58ebf800ed..03b79065e2 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -123,8 +123,9 @@ def _return_value_looks_like_wrong_library(value): "Instead, you want (notice the parentheses!):\n" "\n" " trio.run({async_fn.__name__}, ...) # correct!\n" - " nursery.start_soon({async_fn.__name__}, ...) # correct!" - .format(async_fn=async_fn) + " nursery.start_soon({async_fn.__name__}, ...) # correct!".format( + async_fn=async_fn + ) ) from None # Give good error for: nursery.start_soon(future) @@ -148,8 +149,7 @@ def _return_value_looks_like_wrong_library(value): raise TypeError( "Trio got unexpected {!r} – are you trying to use a " "library written for asyncio/twisted/tornado or similar? " - "That won't work without some sort of compatibility shim." - .format(coro) + "That won't work without some sort of compatibility shim.".format(coro) ) if isasyncgen(coro): @@ -180,6 +180,7 @@ class ConflictDetector: tasks don't call sendall simultaneously on the same stream. """ + def __init__(self, msg): self._msg = msg self._held = False @@ -198,9 +199,10 @@ def async_wraps(cls, wrapped_cls, attr_name): """Similar to wraps, but for async wrappers of non-async functions. """ + def decorator(func): func.__name__ = attr_name - func.__qualname__ = '.'.join((cls.__qualname__, attr_name)) + func.__qualname__ = ".".join((cls.__qualname__, attr_name)) func.__doc__ = """Like :meth:`~{}.{}.{}`, but async. @@ -257,6 +259,7 @@ def open_memory_channel(max_buffer_size: int) -> Tuple[ and currently won't type-check without a mypy plugin or clever stubs, but at least it becomes possible to write those. """ + def __init__(self, fn): update_wrapper(self, fn) self._fn = fn @@ -296,6 +299,7 @@ class SomeClass(metaclass=Final): ------ - TypeError if a sub class is created """ + def __new__(cls, name, bases, cls_namespace): for base in bases: if isinstance(base, Final): @@ -313,7 +317,7 @@ def __new__(cls, name, bases, cls_namespace): f"subclassing {base.__module__}.{base.__qualname__}", "0.15.0", issue=1044, - instead="composition or delegation" + instead="composition or delegation", ) break return super().__new__(cls, name, bases, cls_namespace) @@ -337,6 +341,7 @@ class SomeClass(metaclass=NoPublicConstructor): ------ - TypeError if a sub class or an instance is created. """ + def __call__(self, *args, **kwargs): raise TypeError( f"{self.__module__}.{self.__qualname__} has no public constructor" diff --git a/trio/_wait_for_object.py b/trio/_wait_for_object.py index 1b209ddb26..3231e4d551 100644 --- a/trio/_wait_for_object.py +++ b/trio/_wait_for_object.py @@ -1,7 +1,13 @@ import math from . import _timeouts import trio -from ._core._windows_cffi import ffi, kernel32, ErrorCodes, raise_winerror, _handle +from ._core._windows_cffi import ( + ffi, + kernel32, + ErrorCodes, + raise_winerror, + _handle, +) async def WaitForSingleObject(obj): @@ -52,9 +58,7 @@ def WaitForMultipleObjects_sync(*handles): handle_arr = ffi.new("HANDLE[{}]".format(n)) for i in range(n): handle_arr[i] = handles[i] - timeout = 0xffffffff # INFINITE - retcode = kernel32.WaitForMultipleObjects( - n, handle_arr, False, timeout - ) # blocking + timeout = 0xFFFFFFFF # INFINITE + retcode = kernel32.WaitForMultipleObjects(n, handle_arr, False, timeout) # blocking if retcode == ErrorCodes.WAIT_FAILED: raise_winerror() diff --git a/trio/_windows_pipes.py b/trio/_windows_pipes.py index 6af69f364a..81b9834b19 100644 --- a/trio/_windows_pipes.py +++ b/trio/_windows_pipes.py @@ -41,6 +41,7 @@ class PipeSendStream(SendStream, metaclass=Final): """Represents a send stream over a Windows named pipe that has been opened in OVERLAPPED mode. """ + def __init__(self, handle: int) -> None: self._handle_holder = _HandleHolder(handle) self._conflict_detector = ConflictDetector( @@ -57,9 +58,7 @@ async def send_all(self, data: bytes): return try: - written = await _core.write_overlapped( - self._handle_holder.handle, data - ) + written = await _core.write_overlapped(self._handle_holder.handle, data) except BrokenPipeError as ex: raise _core.BrokenResourceError from ex # By my reading of MSDN, this assert is guaranteed to pass so long @@ -81,6 +80,7 @@ async def aclose(self): class PipeReceiveStream(ReceiveStream, metaclass=Final): """Represents a receive stream over an os.pipe object.""" + def __init__(self, handle: int) -> None: self._handle_holder = _HandleHolder(handle) self._conflict_detector = ConflictDetector( diff --git a/trio/abc.py b/trio/abc.py index e3348360e4..ce0a1f6c00 100644 --- a/trio/abc.py +++ b/trio/abc.py @@ -5,7 +5,17 @@ # implementation in an underscored module, and then re-export the public parts # here. from ._abc import ( - Clock, Instrument, AsyncResource, SendStream, ReceiveStream, Stream, - HalfCloseableStream, SocketFactory, HostnameResolver, Listener, - SendChannel, ReceiveChannel, Channel + Clock, + Instrument, + AsyncResource, + SendStream, + ReceiveStream, + Stream, + HalfCloseableStream, + SocketFactory, + HostnameResolver, + Listener, + SendChannel, + ReceiveChannel, + Channel, ) diff --git a/trio/lowlevel.py b/trio/lowlevel.py index 5fe32c03d9..21ec0597df 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -13,14 +13,34 @@ # Generally available symbols from ._core import ( - cancel_shielded_checkpoint, Abort, wait_task_rescheduled, - enable_ki_protection, disable_ki_protection, currently_ki_protected, Task, - checkpoint, current_task, ParkingLot, UnboundedQueue, RunVar, TrioToken, - current_trio_token, temporarily_detach_coroutine_object, - permanently_detach_coroutine_object, reattach_detached_coroutine_object, - current_statistics, reschedule, remove_instrument, add_instrument, - current_clock, current_root_task, checkpoint_if_cancelled, - spawn_system_task, wait_readable, wait_writable, notify_closing + cancel_shielded_checkpoint, + Abort, + wait_task_rescheduled, + enable_ki_protection, + disable_ki_protection, + currently_ki_protected, + Task, + checkpoint, + current_task, + ParkingLot, + UnboundedQueue, + RunVar, + TrioToken, + current_trio_token, + temporarily_detach_coroutine_object, + permanently_detach_coroutine_object, + reattach_detached_coroutine_object, + current_statistics, + reschedule, + remove_instrument, + add_instrument, + current_clock, + current_root_task, + checkpoint_if_cancelled, + spawn_system_task, + wait_readable, + wait_writable, + notify_closing, ) # Unix-specific symbols diff --git a/trio/socket.py b/trio/socket.py index 8b87ea18b5..ebbccd50ea 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -127,7 +127,7 @@ import socket as _stdlib_socket _bad_symbols = set() -if _sys.platform == 'win32': +if _sys.platform == "win32": # See https://github.com/python-trio/trio/issues/39 # Do not import for windows platform # (you can still get it from stdlib socket, of course, if you want it) @@ -143,9 +143,16 @@ # import the overwrites from ._socket import ( - fromfd, from_stdlib_socket, getprotobyname, socketpair, getnameinfo, - socket, getaddrinfo, set_custom_hostname_resolver, - set_custom_socket_factory, SocketType + fromfd, + from_stdlib_socket, + getprotobyname, + socketpair, + getnameinfo, + socket, + getaddrinfo, + set_custom_hostname_resolver, + set_custom_socket_factory, + SocketType, ) # not always available so expose only if @@ -170,9 +177,7 @@ # not always available so expose only if try: - from socket import ( - sethostname, if_nameindex, if_nametoindex, if_indextoname - ) + from socket import sethostname, if_nameindex, if_nametoindex, if_indextoname except ImportError: pass diff --git a/trio/testing/__init__.py b/trio/testing/__init__.py index df150ec62b..8c730ffeb5 100644 --- a/trio/testing/__init__.py +++ b/trio/testing/__init__.py @@ -9,13 +9,19 @@ from ._sequencer import Sequencer from ._check_streams import ( - check_one_way_stream, check_two_way_stream, check_half_closeable_stream + check_one_way_stream, + check_two_way_stream, + check_half_closeable_stream, ) from ._memory_streams import ( - MemorySendStream, MemoryReceiveStream, memory_stream_pump, - memory_stream_one_way_pair, memory_stream_pair, - lockstep_stream_one_way_pair, lockstep_stream_pair + MemorySendStream, + MemoryReceiveStream, + memory_stream_pump, + memory_stream_one_way_pair, + memory_stream_pair, + lockstep_stream_one_way_pair, + lockstep_stream_pair, ) from ._network import open_stream_to_socket_listener @@ -23,5 +29,6 @@ ################################################################ from .._util import fixup_module_metadata + fixup_module_metadata(__name__, globals()) del fixup_module_metadata diff --git a/trio/testing/_check_streams.py b/trio/testing/_check_streams.py index 48880c13c4..7a9006ff43 100644 --- a/trio/testing/_check_streams.py +++ b/trio/testing/_check_streams.py @@ -130,8 +130,7 @@ async def simple_check_wait_send_all_might_not_block(scope): async with _core.open_nursery() as nursery: nursery.start_soon( - simple_check_wait_send_all_might_not_block, - nursery.cancel_scope + simple_check_wait_send_all_might_not_block, nursery.cancel_scope ) nursery.start_soon(do_receive_some, 1) @@ -398,19 +397,18 @@ async def flipped_stream_maker(): async def flipped_clogged_stream_maker(): return reversed(await clogged_stream_maker()) + else: flipped_clogged_stream_maker = None - await check_one_way_stream( - flipped_stream_maker, flipped_clogged_stream_maker - ) + await check_one_way_stream(flipped_stream_maker, flipped_clogged_stream_maker) async with _ForceCloseBoth(await stream_maker()) as (s1, s2): assert isinstance(s1, Stream) assert isinstance(s2, Stream) # Duplex can be a bit tricky, might as well check it as well - DUPLEX_TEST_SIZE = 2**20 - CHUNK_SIZE_MAX = 2**14 + DUPLEX_TEST_SIZE = 2 ** 20 + CHUNK_SIZE_MAX = 2 ** 14 r = random.Random(0) i = r.getrandbits(8 * DUPLEX_TEST_SIZE) diff --git a/trio/testing/_checkpoints.py b/trio/testing/_checkpoints.py index 27fd4d187d..5804295300 100644 --- a/trio/testing/_checkpoints.py +++ b/trio/testing/_checkpoints.py @@ -11,19 +11,13 @@ def _assert_yields_or_not(expected): orig_schedule = task._schedule_points try: yield - if ( - expected and ( - task._cancel_points == orig_cancel - or task._schedule_points == orig_schedule - ) + if expected and ( + task._cancel_points == orig_cancel or task._schedule_points == orig_schedule ): raise AssertionError("assert_checkpoints block did not yield!") finally: - if ( - not expected and ( - task._cancel_points != orig_cancel - or task._schedule_points != orig_schedule - ) + if not expected and ( + task._cancel_points != orig_cancel or task._schedule_points != orig_schedule ): raise AssertionError("assert_no_checkpoints block yielded!") diff --git a/trio/testing/_memory_streams.py b/trio/testing/_memory_streams.py index 62d63b73d9..66f7f25d97 100644 --- a/trio/testing/_memory_streams.py +++ b/trio/testing/_memory_streams.py @@ -73,9 +73,7 @@ async def get(self, max_bytes=None): return self._get_impl(max_bytes) -class MemorySendStream( - SendStream, metaclass=_util.SubclassingDeprecatedIn_v0_15_0 -): +class MemorySendStream(SendStream, metaclass=_util.SubclassingDeprecatedIn_v0_15_0): """An in-memory :class:`~trio.abc.SendStream`. Args: @@ -95,11 +93,12 @@ class MemorySendStream( you can change them at any time. """ + def __init__( self, send_all_hook=None, wait_send_all_might_not_block_hook=None, - close_hook=None + close_hook=None, ): self._conflict_detector = _util.ConflictDetector( "another task is using this stream" @@ -208,6 +207,7 @@ class MemoryReceiveStream( change them at any time. """ + def __init__(self, receive_some_hook=None, close_hook=None): self._conflict_detector = _util.ConflictDetector( "another task is using this stream" @@ -270,9 +270,7 @@ def put_eof(self): self._incoming.close() -def memory_stream_pump( - memory_send_stream, memory_receive_stream, *, max_bytes=None -): +def memory_stream_pump(memory_send_stream, memory_receive_stream, *, max_bytes=None): """Take data out of the given :class:`MemorySendStream`'s internal buffer, and put it into the given :class:`MemoryReceiveStream`'s internal buffer. diff --git a/trio/testing/_mock_clock.py b/trio/testing/_mock_clock.py index 997e701f39..843f51197f 100644 --- a/trio/testing/_mock_clock.py +++ b/trio/testing/_mock_clock.py @@ -79,6 +79,7 @@ class MockClock(Clock, metaclass=SubclassingDeprecatedIn_v0_15_0): :func:`wait_all_tasks_blocked`. """ + def __init__(self, rate=0.0, autojump_threshold=inf): # when the real clock said 'real_base', the virtual time was # 'virtual_base', and since then it's advanced at 'rate' virtual @@ -97,10 +98,8 @@ def __init__(self, rate=0.0, autojump_threshold=inf): self.autojump_threshold = autojump_threshold def __repr__(self): - return ( - "".format( - self.current_time(), self._rate, id(self) - ) + return "".format( + self.current_time(), self._rate, id(self) ) @property @@ -141,9 +140,7 @@ async def _autojumper(self): # to raise Cancelled, which is absorbed by the cancel # scope above, and effectively just causes us to skip back # to the start the loop, like a 'continue'. - await _core.wait_all_tasks_blocked( - self._autojump_threshold, inf - ) + await _core.wait_all_tasks_blocked(self._autojump_threshold, inf) statistics = _core.current_statistics() jump = statistics.seconds_to_next_deadline if 0 < jump < inf: diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index 21fc492dff..abecf396ce 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -61,9 +61,7 @@ async def main(): @asynccontextmanager async def __call__(self, position: int): if position in self._claimed: - raise RuntimeError( - "Attempted to re-use sequence point {}".format(position) - ) + raise RuntimeError("Attempted to re-use sequence point {}".format(position)) if self._broken: raise RuntimeError("sequence broken!") self._claimed.add(position) @@ -74,9 +72,7 @@ async def __call__(self, position: int): self._broken = True for event in self._sequence_points.values(): event.set() - raise RuntimeError( - "Sequencer wait cancelled -- sequence broken" - ) + raise RuntimeError("Sequencer wait cancelled -- sequence broken") else: if self._broken: raise RuntimeError("sequence broken!") diff --git a/trio/testing/_trio_test.py b/trio/testing/_trio_test.py index 87caa5881a..4fcaeae372 100644 --- a/trio/testing/_trio_test.py +++ b/trio/testing/_trio_test.py @@ -24,8 +24,6 @@ def wrapper(**kwargs): else: raise ValueError("too many clocks spoil the broth!") instruments = [i for i in kwargs.values() if isinstance(i, Instrument)] - return _core.run( - partial(fn, **kwargs), clock=clock, instruments=instruments - ) + return _core.run(partial(fn, **kwargs), clock=clock, instruments=instruments) return wrapper diff --git a/trio/tests/module_with_deprecations.py b/trio/tests/module_with_deprecations.py index d194b6a5bd..b0f83b1540 100644 --- a/trio/tests/module_with_deprecations.py +++ b/trio/tests/module_with_deprecations.py @@ -8,22 +8,14 @@ # attributes in between calling enable_attribute_deprecations and defining # __deprecated_attributes__: import sys + this_mod = sys.modules[__name__] assert this_mod.regular == "hi" assert not hasattr(this_mod, "dep1") __deprecated_attributes__ = { - "dep1": - _deprecate.DeprecatedAttribute( - "value1", - "1.1", - issue=1, - ), - "dep2": - _deprecate.DeprecatedAttribute( - "value2", - "1.2", - issue=1, - instead="instead-string", - ), + "dep1": _deprecate.DeprecatedAttribute("value1", "1.1", issue=1,), + "dep2": _deprecate.DeprecatedAttribute( + "value2", "1.2", issue=1, instead="instead-string", + ), } diff --git a/trio/tests/test_deprecate.py b/trio/tests/test_deprecate.py index 8497f911c7..4534bbb7c9 100644 --- a/trio/tests/test_deprecate.py +++ b/trio/tests/test_deprecate.py @@ -4,7 +4,10 @@ import warnings from .._deprecate import ( - TrioDeprecationWarning, warn_deprecated, deprecated, deprecated_alias + TrioDeprecationWarning, + warn_deprecated, + deprecated, + deprecated_alias, ) from . import module_with_deprecations @@ -181,33 +184,45 @@ def docstring_test4(): # pragma: no cover def test_deprecated_docstring_munging(): - assert docstring_test1.__doc__ == """Hello! + assert ( + docstring_test1.__doc__ + == """Hello! .. deprecated:: 2.1 Use hi instead. For details, see `issue #1 `__. """ + ) - assert docstring_test2.__doc__ == """Hello! + assert ( + docstring_test2.__doc__ + == """Hello! .. deprecated:: 2.1 Use hi instead. """ + ) - assert docstring_test3.__doc__ == """Hello! + assert ( + docstring_test3.__doc__ + == """Hello! .. deprecated:: 2.1 For details, see `issue #1 `__. """ + ) - assert docstring_test4.__doc__ == """Hello! + assert ( + docstring_test4.__doc__ + == """Hello! .. deprecated:: 2.1 """ + ) def test_module_with_deprecations(recwarn_always): diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index af34070267..1d49f4765d 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -18,13 +18,12 @@ def test_core_is_properly_reexported(): # three modules: sources = [trio, trio.lowlevel, trio.testing] for symbol in dir(_core): - if symbol.startswith('_') or symbol == 'tests': + if symbol.startswith("_") or symbol == "tests": continue found = 0 for source in sources: - if ( - symbol in dir(source) - and getattr(source, symbol) is getattr(_core, symbol) + if symbol in dir(source) and getattr(source, symbol) is getattr( + _core, symbol ): found += 1 print(symbol, found) @@ -85,11 +84,13 @@ def no_underscores(symbols): if tool == "pylint": from pylint.lint import PyLinter + linter = PyLinter() ast = linter.get_ast(module.__file__, modname) static_names = no_underscores(ast) elif tool == "jedi": import jedi + # Simulate typing "import trio; trio." script = jedi.Script("import {}; {}.".format(modname, modname)) completions = script.complete() diff --git a/trio/tests/test_file_io.py b/trio/tests/test_file_io.py index fd4fa648b4..b40f7518a9 100644 --- a/trio/tests/test_file_io.py +++ b/trio/tests/test_file_io.py @@ -12,7 +12,7 @@ @pytest.fixture def path(tmpdir): - return os.fspath(tmpdir.join('test')) + return os.fspath(tmpdir.join("test")) @pytest.fixture @@ -58,9 +58,7 @@ def test_dir_matches_wrapped(async_file, wrapped): attrs = _FILE_SYNC_ATTRS.union(_FILE_ASYNC_METHODS) # all supported attrs in wrapped should be available in async_file - assert all( - attr in dir(async_file) for attr in attrs if attr in dir(wrapped) - ) + assert all(attr in dir(async_file) for attr in attrs if attr in dir(wrapped)) # all supported attrs not in wrapped should not be available in async_file assert not any( attr in dir(async_file) for attr in attrs if attr not in dir(wrapped) @@ -74,10 +72,10 @@ def unsupported_attr(self): # pragma: no cover async_file = trio.wrap_file(FakeFile()) - assert hasattr(async_file.wrapped, 'unsupported_attr') + assert hasattr(async_file.wrapped, "unsupported_attr") with pytest.raises(AttributeError): - getattr(async_file, 'unsupported_attr') + getattr(async_file, "unsupported_attr") def test_sync_attrs_forwarded(async_file, wrapped): @@ -110,10 +108,10 @@ def test_async_methods_generated_once(async_file): def test_async_methods_signature(async_file): # use read as a representative of all async methods - assert async_file.read.__name__ == 'read' - assert async_file.read.__qualname__ == 'AsyncIOWrapper.read' + assert async_file.read.__name__ == "read" + assert async_file.read.__qualname__ == "AsyncIOWrapper.read" - assert 'io.StringIO.read' in async_file.read.__doc__ + assert "io.StringIO.read" in async_file.read.__doc__ async def test_async_methods_wrap(async_file, wrapped): @@ -147,7 +145,7 @@ async def test_async_methods_match_wrapper(async_file, wrapped): async def test_open(path): - f = await trio.open_file(path, 'w') + f = await trio.open_file(path, "w") assert isinstance(f, AsyncIOWrapper) @@ -155,7 +153,7 @@ async def test_open(path): async def test_open_context_manager(path): - async with await trio.open_file(path, 'w') as f: + async with await trio.open_file(path, "w") as f: assert isinstance(f, AsyncIOWrapper) assert not f.closed @@ -163,7 +161,7 @@ async def test_open_context_manager(path): async def test_async_iter(): - async_file = trio.wrap_file(io.StringIO('test\nfoo\nbar')) + async_file = trio.wrap_file(io.StringIO("test\nfoo\nbar")) expected = list(async_file.wrapped) result = [] async_file.wrapped.seek(0) @@ -176,11 +174,11 @@ async def test_async_iter(): async def test_aclose_cancelled(path): with _core.CancelScope() as cscope: - f = await trio.open_file(path, 'w') + f = await trio.open_file(path, "w") cscope.cancel() with pytest.raises(_core.Cancelled): - await f.write('a') + await f.write("a") with pytest.raises(_core.Cancelled): await f.aclose() diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index d11c1c6e6d..ffd708807a 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -6,9 +6,7 @@ import attr import trio -from trio import ( - open_tcp_listeners, serve_tcp, SocketListener, open_tcp_stream -) +from trio import open_tcp_listeners, serve_tcp, SocketListener, open_tcp_stream from trio.testing import open_stream_to_socket_listener from .. import socket as tsocket from .._core.tests.tutil import slow, creates_ipv6, binds_ipv6 @@ -79,9 +77,7 @@ async def measure_backlog(listener, limit): # has been observed to sometimes raise ConnectionResetError. with trio.move_on_after(0.5) as cancel_scope: try: - client_stream = await open_stream_to_socket_listener( - listener - ) + client_stream = await open_stream_to_socket_listener(listener) except ConnectionResetError: # pragma: no cover break client_streams.append(client_stream) @@ -272,28 +268,18 @@ async def handler(stream): @pytest.mark.parametrize( - "try_families", [ - {tsocket.AF_INET}, - {tsocket.AF_INET6}, - {tsocket.AF_INET, tsocket.AF_INET6}, - ] + "try_families", + [{tsocket.AF_INET}, {tsocket.AF_INET6}, {tsocket.AF_INET, tsocket.AF_INET6},], ) @pytest.mark.parametrize( - "fail_families", [ - {tsocket.AF_INET}, - {tsocket.AF_INET6}, - {tsocket.AF_INET, tsocket.AF_INET6}, - ] + "fail_families", + [{tsocket.AF_INET}, {tsocket.AF_INET6}, {tsocket.AF_INET, tsocket.AF_INET6},], ) async def test_open_tcp_listeners_some_address_families_unavailable( try_families, fail_families ): fsf = FakeSocketFactory( - 10, - raise_on_family={ - family: errno.EAFNOSUPPORT - for family in fail_families - } + 10, raise_on_family={family: errno.EAFNOSUPPORT for family in fail_families}, ) tsocket.set_custom_socket_factory(fsf) tsocket.set_custom_hostname_resolver( @@ -326,13 +312,11 @@ async def test_open_tcp_listeners_socket_fails_not_afnosupport(): raise_on_family={ tsocket.AF_INET: errno.EAFNOSUPPORT, tsocket.AF_INET6: errno.EINVAL, - } + }, ) tsocket.set_custom_socket_factory(fsf) tsocket.set_custom_hostname_resolver( - FakeHostnameResolver( - [(tsocket.AF_INET, "foo"), (tsocket.AF_INET6, "bar")] - ) + FakeHostnameResolver([(tsocket.AF_INET, "foo"), (tsocket.AF_INET6, "bar")]) ) with pytest.raises(OSError) as exc_info: diff --git a/trio/tests/test_highlevel_open_tcp_stream.py b/trio/tests/test_highlevel_open_tcp_stream.py index 0c28450e5e..9fd0f3992a 100644 --- a/trio/tests/test_highlevel_open_tcp_stream.py +++ b/trio/tests/test_highlevel_open_tcp_stream.py @@ -46,7 +46,11 @@ def close(self): def test_reorder_for_rfc_6555_section_5_4(): def fake4(i): return ( - AF_INET, SOCK_STREAM, IPPROTO_TCP, "", ("10.0.0.{}".format(i), 80) + AF_INET, + SOCK_STREAM, + IPPROTO_TCP, + "", + ("10.0.0.{}".format(i), 80), ) def fake6(i): @@ -225,7 +229,7 @@ async def run_scenario( # If this is True, we require there to be an exception, and return # (exception, scenario object) expect_error=(), - **kwargs + **kwargs, ): supported_families = set() if ipv4_supported: @@ -278,8 +282,7 @@ async def test_one_host_slow_fail(autojump_clock): async def test_one_host_failed_after_connect(autojump_clock): exc, scenario = await run_scenario( - 83, [("1.2.3.4", 1, "postconnect_fail")], - expect_error=KeyboardInterrupt + 83, [("1.2.3.4", 1, "postconnect_fail")], expect_error=KeyboardInterrupt ) assert isinstance(exc, KeyboardInterrupt) diff --git a/trio/tests/test_highlevel_open_unix_stream.py b/trio/tests/test_highlevel_open_unix_stream.py index 872a43dd6d..211aff3e70 100644 --- a/trio/tests/test_highlevel_open_unix_stream.py +++ b/trio/tests/test_highlevel_open_unix_stream.py @@ -5,9 +5,7 @@ import pytest from trio import open_unix_socket, Path -from trio._highlevel_open_unix_stream import ( - close_on_error, -) +from trio._highlevel_open_unix_stream import close_on_error if not hasattr(socket, "AF_UNIX"): pytestmark = pytest.mark.skip("Needs unix socket support") @@ -30,7 +28,7 @@ def close(self): assert c.closed -@pytest.mark.parametrize('filename', [4, 4.5]) +@pytest.mark.parametrize("filename", [4, 4.5]) async def test_open_with_bad_filename_type(filename): with pytest.raises(TypeError): await open_unix_socket(filename) diff --git a/trio/tests/test_highlevel_serve_listeners.py b/trio/tests/test_highlevel_serve_listeners.py index e26a65605f..b028092eb9 100644 --- a/trio/tests/test_highlevel_serve_listeners.py +++ b/trio/tests/test_highlevel_serve_listeners.py @@ -136,8 +136,9 @@ async def connection_watcher(*, task_status=trio.TASK_STATUS_IGNORED): await nursery.start( partial( trio.serve_listeners, - handler, [listener], - handler_nursery=handler_nursery + handler, + [listener], + handler_nursery=handler_nursery, ) ) for _ in range(10): diff --git a/trio/tests/test_highlevel_socket.py b/trio/tests/test_highlevel_socket.py index dc19219e3e..f3570f743e 100644 --- a/trio/tests/test_highlevel_socket.py +++ b/trio/tests/test_highlevel_socket.py @@ -6,7 +6,9 @@ from .. import _core from ..testing import ( - check_half_closeable_stream, wait_all_tasks_blocked, assert_checkpoints + check_half_closeable_stream, + wait_all_tasks_blocked, + assert_checkpoints, ) from .._highlevel_socket import * from .. import socket as tsocket diff --git a/trio/tests/test_highlevel_ssl_helpers.py b/trio/tests/test_highlevel_ssl_helpers.py index d18cb84347..99ee46c9d0 100644 --- a/trio/tests/test_highlevel_ssl_helpers.py +++ b/trio/tests/test_highlevel_ssl_helpers.py @@ -10,7 +10,9 @@ from .test_ssl import client_ctx, SERVER_CTX from .._highlevel_ssl_helpers import ( - open_ssl_over_tcp_stream, open_ssl_over_tcp_listeners, serve_ssl_over_tcp + open_ssl_over_tcp_stream, + open_ssl_over_tcp_listeners, + serve_ssl_over_tcp, ) @@ -41,18 +43,10 @@ async def getnameinfo(self, *args): # pragma: no cover # This uses serve_ssl_over_tcp, which uses open_ssl_over_tcp_listeners... # noqa is needed because flake8 doesn't understand how pytest fixtures work. -async def test_open_ssl_over_tcp_stream_and_everything_else( - client_ctx, # noqa: F811 -): +async def test_open_ssl_over_tcp_stream_and_everything_else(client_ctx,): # noqa: F811 async with trio.open_nursery() as nursery: (listener,) = await nursery.start( - partial( - serve_ssl_over_tcp, - echo_handler, - 0, - SERVER_CTX, - host="127.0.0.1" - ) + partial(serve_ssl_over_tcp, echo_handler, 0, SERVER_CTX, host="127.0.0.1",) ) sockaddr = listener.transport_listener.socket.getsockname() hostname_resolver = FakeHostnameResolver(sockaddr) @@ -67,18 +61,14 @@ async def test_open_ssl_over_tcp_stream_and_everything_else( # We have the trust but not the hostname # (checks custom ssl_context + hostname checking) stream = await open_ssl_over_tcp_stream( - "xyzzy.example.org", - 80, - ssl_context=client_ctx, + "xyzzy.example.org", 80, ssl_context=client_ctx, ) with pytest.raises(trio.BrokenResourceError): await stream.do_handshake() # This one should work! stream = await open_ssl_over_tcp_stream( - "trio-test-1.example.org", - 80, - ssl_context=client_ctx, + "trio-test-1.example.org", 80, ssl_context=client_ctx, ) assert isinstance(stream, trio.SSLStream) assert stream.server_hostname == "trio-test-1.example.org" @@ -103,9 +93,7 @@ async def test_open_ssl_over_tcp_stream_and_everything_else( async def test_open_ssl_over_tcp_listeners(): - (listener,) = await open_ssl_over_tcp_listeners( - 0, SERVER_CTX, host="127.0.0.1" - ) + (listener,) = await open_ssl_over_tcp_listeners(0, SERVER_CTX, host="127.0.0.1") async with listener: assert isinstance(listener, trio.SSLListener) tl = listener.transport_listener diff --git a/trio/tests/test_path.py b/trio/tests/test_path.py index 9bbbfd4df2..284bcf82dd 100644 --- a/trio/tests/test_path.py +++ b/trio/tests/test_path.py @@ -10,7 +10,7 @@ @pytest.fixture def path(tmpdir): - p = str(tmpdir.join('test')) + p = str(tmpdir.join("test")) return trio.Path(p) @@ -21,32 +21,33 @@ def method_pair(path, method_name): async def test_open_is_async_context_manager(path): - async with await path.open('w') as f: + async with await path.open("w") as f: assert isinstance(f, AsyncIOWrapper) assert f.closed async def test_magic(): - path = trio.Path('test') + path = trio.Path("test") - assert str(path) == 'test' - assert bytes(path) == b'test' + assert str(path) == "test" + assert bytes(path) == b"test" cls_pairs = [ - (trio.Path, pathlib.Path), (pathlib.Path, trio.Path), - (trio.Path, trio.Path) + (trio.Path, pathlib.Path), + (pathlib.Path, trio.Path), + (trio.Path, trio.Path), ] -@pytest.mark.parametrize('cls_a,cls_b', cls_pairs) +@pytest.mark.parametrize("cls_a,cls_b", cls_pairs) async def test_cmp_magic(cls_a, cls_b): - a, b = cls_a(''), cls_b('') + a, b = cls_a(""), cls_b("") assert a == b assert not a != b - a, b = cls_a('a'), cls_b('b') + a, b = cls_a("a"), cls_b("b") assert a < b assert b > a @@ -60,24 +61,26 @@ async def test_cmp_magic(cls_a, cls_b): # __*div__ does not properly raise NotImplementedError like the other comparison # magic, so trio.Path's implementation does not get dispatched cls_pairs = [ - (trio.Path, pathlib.Path), (trio.Path, trio.Path), (trio.Path, str), - (str, trio.Path) + (trio.Path, pathlib.Path), + (trio.Path, trio.Path), + (trio.Path, str), + (str, trio.Path), ] -@pytest.mark.parametrize('cls_a,cls_b', cls_pairs) +@pytest.mark.parametrize("cls_a,cls_b", cls_pairs) async def test_div_magic(cls_a, cls_b): - a, b = cls_a('a'), cls_b('b') + a, b = cls_a("a"), cls_b("b") result = a / b assert isinstance(result, trio.Path) - assert str(result) == os.path.join('a', 'b') + assert str(result) == os.path.join("a", "b") @pytest.mark.parametrize( - 'cls_a,cls_b', [(trio.Path, pathlib.Path), (trio.Path, trio.Path)] + "cls_a,cls_b", [(trio.Path, pathlib.Path), (trio.Path, trio.Path)] ) -@pytest.mark.parametrize('path', ["foo", "foo/bar/baz", "./foo"]) +@pytest.mark.parametrize("path", ["foo", "foo/bar/baz", "./foo"]) async def test_hash_magic(cls_a, cls_b, path): a, b = cls_a(path), cls_b(path) assert hash(a) == hash(b) @@ -86,23 +89,23 @@ async def test_hash_magic(cls_a, cls_b, path): async def test_forwarded_properties(path): # use `name` as a representative of forwarded properties - assert 'name' in dir(path) - assert path.name == 'test' + assert "name" in dir(path) + assert path.name == "test" async def test_async_method_signature(path): # use `resolve` as a representative of wrapped methods - assert path.resolve.__name__ == 'resolve' - assert path.resolve.__qualname__ == 'Path.resolve' + assert path.resolve.__name__ == "resolve" + assert path.resolve.__qualname__ == "Path.resolve" - assert 'pathlib.Path.resolve' in path.resolve.__doc__ + assert "pathlib.Path.resolve" in path.resolve.__doc__ -@pytest.mark.parametrize('method_name', ['is_dir', 'is_file']) +@pytest.mark.parametrize("method_name", ["is_dir", "is_file"]) async def test_compare_async_stat_methods(method_name): - method, async_method = method_pair('.', method_name) + method, async_method = method_pair(".", method_name) result = method() async_result = await async_method() @@ -112,13 +115,13 @@ async def test_compare_async_stat_methods(method_name): async def test_invalid_name_not_wrapped(path): with pytest.raises(AttributeError): - getattr(path, 'invalid_fake_attr') + getattr(path, "invalid_fake_attr") -@pytest.mark.parametrize('method_name', ['absolute', 'resolve']) +@pytest.mark.parametrize("method_name", ["absolute", "resolve"]) async def test_async_methods_rewrap(method_name): - method, async_method = method_pair('.', method_name) + method, async_method = method_pair(".", method_name) result = method() async_result = await async_method() @@ -128,13 +131,13 @@ async def test_async_methods_rewrap(method_name): async def test_forward_methods_rewrap(path, tmpdir): - with_name = path.with_name('foo') - with_suffix = path.with_suffix('.py') + with_name = path.with_name("foo") + with_suffix = path.with_suffix(".py") assert isinstance(with_name, trio.Path) - assert with_name == tmpdir.join('foo') + assert with_name == tmpdir.join("foo") assert isinstance(with_suffix, trio.Path) - assert with_suffix == tmpdir.join('test.py') + assert with_suffix == tmpdir.join("test.py") async def test_forward_properties_rewrap(path): @@ -144,18 +147,18 @@ async def test_forward_properties_rewrap(path): async def test_forward_methods_without_rewrap(path, tmpdir): path = await path.parent.resolve() - assert path.as_uri().startswith('file:///') + assert path.as_uri().startswith("file:///") async def test_repr(): - path = trio.Path('.') + path = trio.Path(".") assert repr(path) == "trio.Path('.')" class MockWrapped: - unsupported = 'unsupported' - _private = 'private' + unsupported = "unsupported" + _private = "private" class MockWrapper: @@ -174,18 +177,18 @@ async def test_type_wraps_unsupported(): async def test_type_forwards_private(): - Type.generate_forwards(MockWrapper, {'unsupported': None}) + Type.generate_forwards(MockWrapper, {"unsupported": None}) - assert not hasattr(MockWrapper, '_private') + assert not hasattr(MockWrapper, "_private") async def test_type_wraps_private(): - Type.generate_wraps(MockWrapper, {'unsupported': None}) + Type.generate_wraps(MockWrapper, {"unsupported": None}) - assert not hasattr(MockWrapper, '_private') + assert not hasattr(MockWrapper, "_private") -@pytest.mark.parametrize('meth', [trio.Path.__init__, trio.Path.joinpath]) +@pytest.mark.parametrize("meth", [trio.Path.__init__, trio.Path.joinpath]) async def test_path_wraps_path(path, meth): wrapped = await path.absolute() result = meth(path, wrapped) @@ -201,22 +204,22 @@ async def test_path_nonpath(): async def test_open_file_can_open_path(path): - async with await trio.open_file(path, 'w') as f: + async with await trio.open_file(path, "w") as f: assert f.name == os.fspath(path) async def test_globmethods(path): # Populate a directory tree await path.mkdir() - await (path / 'foo').mkdir() - await (path / 'foo' / '_bar.txt').write_bytes(b'') - await (path / 'bar.txt').write_bytes(b'') - await (path / 'bar.dat').write_bytes(b'') + await (path / "foo").mkdir() + await (path / "foo" / "_bar.txt").write_bytes(b"") + await (path / "bar.txt").write_bytes(b"") + await (path / "bar.dat").write_bytes(b"") # Path.glob for _pattern, _results in { - '*.txt': {'bar.txt'}, - '**/*.txt': {'_bar.txt', 'bar.txt'}, + "*.txt": {"bar.txt"}, + "**/*.txt": {"_bar.txt", "bar.txt"}, }.items(): entries = set() for entry in await path.glob(_pattern): @@ -227,32 +230,32 @@ async def test_globmethods(path): # Path.rglob entries = set() - for entry in await path.rglob('*.txt'): + for entry in await path.rglob("*.txt"): assert isinstance(entry, trio.Path) entries.add(entry.name) - assert entries == {'_bar.txt', 'bar.txt'} + assert entries == {"_bar.txt", "bar.txt"} async def test_iterdir(path): # Populate a directory await path.mkdir() - await (path / 'foo').mkdir() - await (path / 'bar.txt').write_bytes(b'') + await (path / "foo").mkdir() + await (path / "bar.txt").write_bytes(b"") entries = set() for entry in await path.iterdir(): assert isinstance(entry, trio.Path) entries.add(entry.name) - assert entries == {'bar.txt', 'foo'} + assert entries == {"bar.txt", "foo"} async def test_classmethods(): assert isinstance(await trio.Path.home(), trio.Path) # pathlib.Path has only two classmethods - assert str(await trio.Path.home()) == os.path.expanduser('~') + assert str(await trio.Path.home()) == os.path.expanduser("~") assert str(await trio.Path.cwd()) == os.getcwd() # Wrapped method has docstring diff --git a/trio/tests/test_scheduler_determinism.py b/trio/tests/test_scheduler_determinism.py index ba5f469396..67b2447f0a 100644 --- a/trio/tests/test_scheduler_determinism.py +++ b/trio/tests/test_scheduler_determinism.py @@ -26,9 +26,7 @@ def test_the_trio_scheduler_is_not_deterministic(): def test_the_trio_scheduler_is_deterministic_if_seeded(monkeypatch): - monkeypatch.setattr( - trio._core._run, "_ALLOW_DETERMINISTIC_SCHEDULING", True - ) + monkeypatch.setattr(trio._core._run, "_ALLOW_DETERMINISTIC_SCHEDULING", True) traces = [] for _ in range(10): state = trio._core._run._r.getstate() diff --git a/trio/tests/test_signals.py b/trio/tests/test_signals.py index 20821b40f2..235772f900 100644 --- a/trio/tests/test_signals.py +++ b/trio/tests/test_signals.py @@ -108,6 +108,7 @@ async def test_open_signal_receiver_no_starvation(): # open_signal_receiver block might cause the signal to be # redelivered and give us a core dump instead of a traceback... import traceback + traceback.print_exc() @@ -164,9 +165,7 @@ def raise_handler(signum, _): with _signal_handler({signal.SIGILL, signal.SIGFPE}, raise_handler): with pytest.raises(RuntimeError) as excinfo: - with open_signal_receiver( - signal.SIGILL, signal.SIGFPE - ) as receiver: + with open_signal_receiver(signal.SIGILL, signal.SIGFPE) as receiver: signal_raise(signal.SIGILL) signal_raise(signal.SIGFPE) await wait_run_sync_soon_idempotent_queue_barrier() diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index f384ca4297..2a217abdf6 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -44,9 +44,7 @@ def getaddrinfo(self, *args, **kwargs): elif bound[-1] & stdlib_socket.AI_NUMERICHOST: return self._orig_getaddrinfo(*args, **kwargs) else: - raise RuntimeError( - "gai called with unexpected arguments {}".format(bound) - ) + raise RuntimeError("gai called with unexpected arguments {}".format(bound)) @pytest.fixture @@ -117,29 +115,35 @@ def filtered(gai_list): # Simple non-blocking non-error cases, ipv4 and ipv6: with assert_checkpoints(): - res = await tsocket.getaddrinfo( - "127.0.0.1", "12345", type=tsocket.SOCK_STREAM - ) - - check(res, [ - (tsocket.AF_INET, # 127.0.0.1 is ipv4 - tsocket.SOCK_STREAM, - tsocket.IPPROTO_TCP, - "", - ("127.0.0.1", 12345)), - ]) + res = await tsocket.getaddrinfo("127.0.0.1", "12345", type=tsocket.SOCK_STREAM) + + check( + res, + [ + ( + tsocket.AF_INET, # 127.0.0.1 is ipv4 + tsocket.SOCK_STREAM, + tsocket.IPPROTO_TCP, + "", + ("127.0.0.1", 12345), + ), + ], + ) with assert_checkpoints(): - res = await tsocket.getaddrinfo( - "::1", "12345", type=tsocket.SOCK_DGRAM - ) - check(res, [ - (tsocket.AF_INET6, - tsocket.SOCK_DGRAM, - tsocket.IPPROTO_UDP, - "", - ("::1", 12345, 0, 0)), - ]) + res = await tsocket.getaddrinfo("::1", "12345", type=tsocket.SOCK_DGRAM) + check( + res, + [ + ( + tsocket.AF_INET6, + tsocket.SOCK_DGRAM, + tsocket.IPPROTO_UDP, + "", + ("::1", 12345, 0, 0), + ), + ], + ) monkeygai.set("x", b"host", "port", family=0, type=0, proto=0, flags=0) with assert_checkpoints(): @@ -276,6 +280,7 @@ async def test_socket_v6(): @pytest.mark.skipif(not _sys.platform == "linux", reason="linux only") async def test_sniff_sockopts(): from socket import AF_INET, AF_INET6, SOCK_DGRAM, SOCK_STREAM + # generate the combinations of families/types we're testing: sockets = [] for family in [AF_INET, AF_INET6]: @@ -392,10 +397,11 @@ async def test_SocketType_shutdown(): @pytest.mark.parametrize( - "address, socket_type", [ - ('127.0.0.1', tsocket.AF_INET), - pytest.param('::1', tsocket.AF_INET6, marks=binds_ipv6) - ] + "address, socket_type", + [ + ("127.0.0.1", tsocket.AF_INET), + pytest.param("::1", tsocket.AF_INET6, marks=binds_ipv6), + ], ) async def test_SocketType_simple_server(address, socket_type): # listen, bind, accept, connect, getpeername, getsockname @@ -448,7 +454,8 @@ class Addresses: # Direct thorough tests of the implicit resolver helpers @pytest.mark.parametrize( - "socket_type, addrs", [ + "socket_type, addrs", + [ ( tsocket.AF_INET, Addresses( @@ -470,10 +477,10 @@ class Addresses: ), marks=creates_ipv6, ), - ] + ], ) async def test_SocketType_resolve(socket_type, addrs): - v6 = (socket_type == tsocket.AF_INET6) + v6 = socket_type == tsocket.AF_INET6 # For some reason the stdlib special-cases "" to pass NULL to getaddrinfo # They also error out on None, but whatever, None is much more consistent, @@ -492,20 +499,21 @@ async def test_SocketType_resolve(socket_type, addrs): async def res(*args): return await getattr(sock, resolver)(*args) - assert await res((addrs.arbitrary, - "http")) == (addrs.arbitrary, 80, *addrs.extra) + assert await res((addrs.arbitrary, "http")) == ( + addrs.arbitrary, + 80, + *addrs.extra, + ) if v6: assert await res(("1::2", 80, 1)) == ("1::2", 80, 1, 0) assert await res(("1::2", 80, 1, 2)) == ("1::2", 80, 1, 2) # V4 mapped addresses resolved if V6ONLY is False sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, False) - assert await res(("1.2.3.4", - "http")) == ("::ffff:1.2.3.4", 80, 0, 0) + assert await res(("1.2.3.4", "http")) == ("::ffff:1.2.3.4", 80, 0, 0,) # Check the special case, because why not - assert await res(("", - 123)) == (addrs.broadcast, 123, *addrs.extra) + assert await res(("", 123)) == (addrs.broadcast, 123, *addrs.extra,) # But not if it's true (at least on systems where getaddrinfo works # correctly) @@ -702,7 +710,7 @@ async def _resolve_remote_address(self, *args, **kwargs): sock._resolve_remote_address = _resolve_remote_address with assert_checkpoints(): with pytest.raises(_core.Cancelled): - await sock.connect('') + await sock.connect("") assert sock.fileno() == -1 @@ -848,9 +856,11 @@ async def getnameinfo(self, sockaddr, flags): (0, 0, tsocket.IPPROTO_TCP, 0), (0, 0, 0, tsocket.AI_CANONNAME), ]: - assert ( - await tsocket.getaddrinfo("localhost", "foo", *vals) == - ("custom_gai", b"localhost", "foo", *vals) + assert await tsocket.getaddrinfo("localhost", "foo", *vals) == ( + "custom_gai", + b"localhost", + "foo", + *vals, ) # IDNA encoding is handled before calling the special object @@ -858,7 +868,7 @@ async def getnameinfo(self, sockaddr, flags): expected = ("custom_gai", b"xn--f-1gaa", "foo", 0, 0, 0, 0) assert got == expected - assert (await tsocket.getnameinfo("a", 0) == ("custom_gni", "a", 0)) + assert await tsocket.getnameinfo("a", 0) == ("custom_gni", "a", 0) # We can set it back to None assert tsocket.set_custom_hostname_resolver(None) is cr @@ -901,9 +911,7 @@ async def test_SocketType_is_abstract(): tsocket.SocketType() -@pytest.mark.skipif( - not hasattr(tsocket, "AF_UNIX"), reason="no unix domain sockets" -) +@pytest.mark.skipif(not hasattr(tsocket, "AF_UNIX"), reason="no unix domain sockets") async def test_unix_domain_socket(): # Bind has a special branch to use a thread, since it has to do filesystem # traversal. Maybe connect should too? Not sure. diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 9cdad56fe3..dee8e325f7 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -146,8 +146,7 @@ async def ssl_echo_server_raw(**kwargs): # nursery context manager to exit too. with a, b: nursery.start_soon( - trio.to_thread.run_sync, - partial(ssl_echo_serve_sync, b, **kwargs) + trio.to_thread.run_sync, partial(ssl_echo_serve_sync, b, **kwargs), ) yield SocketStream(tsocket.from_stdlib_socket(a)) @@ -158,9 +157,7 @@ async def ssl_echo_server_raw(**kwargs): @asynccontextmanager async def ssl_echo_server(client_ctx, **kwargs): async with ssl_echo_server_raw(**kwargs) as sock: - yield SSLStream( - sock, client_ctx, server_hostname="trio-test-1.example.org" - ) + yield SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") # The weird in-memory server ... thing. @@ -190,6 +187,7 @@ def __init__(self, sleeper=None): # Fortunately pyopenssl uses cryptography under the hood, so we can be # confident that they're using the same version of openssl from cryptography.hazmat.bindings.openssl.binding import Binding + b = Binding() if hasattr(b.lib, "SSL_OP_NO_TLSv1_3"): ctx.set_options(b.lib.SSL_OP_NO_TLSv1_3) @@ -358,9 +356,7 @@ async def test_PyOpenSSLEchoStream_gives_resource_busy_errors(): @contextmanager def virtual_ssl_echo_server(client_ctx, **kwargs): fakesock = PyOpenSSLEchoStream(**kwargs) - yield SSLStream( - fakesock, client_ctx, server_hostname="trio-test-1.example.org" - ) + yield SSLStream(fakesock, client_ctx, server_hostname="trio-test-1.example.org") def ssl_wrap_pair( @@ -369,13 +365,13 @@ def ssl_wrap_pair( server_transport, *, client_kwargs={}, - server_kwargs={} + server_kwargs={}, ): client_ssl = SSLStream( client_transport, client_ctx, server_hostname="trio-test-1.example.org", - **client_kwargs + **client_kwargs, ) server_ssl = SSLStream( server_transport, SERVER_CTX, server_side=True, **server_kwargs @@ -385,16 +381,12 @@ def ssl_wrap_pair( def ssl_memory_stream_pair(client_ctx, **kwargs): client_transport, server_transport = memory_stream_pair() - return ssl_wrap_pair( - client_ctx, client_transport, server_transport, **kwargs - ) + return ssl_wrap_pair(client_ctx, client_transport, server_transport, **kwargs) def ssl_lockstep_stream_pair(client_ctx, **kwargs): client_transport, server_transport = lockstep_stream_pair() - return ssl_wrap_pair( - client_ctx, client_transport, server_transport, **kwargs - ) + return ssl_wrap_pair(client_ctx, client_transport, server_transport, **kwargs) # Simple smoke test for handshake/send/receive/shutdown talking to a @@ -411,9 +403,7 @@ async def test_ssl_client_basics(client_ctx): # Didn't configure the CA file, should fail async with ssl_echo_server_raw(expect_fail=True) as sock: bad_client_ctx = ssl.create_default_context() - s = SSLStream( - sock, bad_client_ctx, server_hostname="trio-test-1.example.org" - ) + s = SSLStream(sock, bad_client_ctx, server_hostname="trio-test-1.example.org") assert not s.server_side with pytest.raises(BrokenResourceError) as excinfo: await s.send_all(b"x") @@ -421,9 +411,7 @@ async def test_ssl_client_basics(client_ctx): # Trusted CA, but wrong host name async with ssl_echo_server_raw(expect_fail=True) as sock: - s = SSLStream( - sock, client_ctx, server_hostname="trio-test-2.example.org" - ) + s = SSLStream(sock, client_ctx, server_hostname="trio-test-2.example.org") assert not s.server_side with pytest.raises(BrokenResourceError) as excinfo: await s.send_all(b"x") @@ -464,9 +452,7 @@ async def test_attributes(client_ctx): async with ssl_echo_server_raw(expect_fail=True) as sock: good_ctx = client_ctx bad_ctx = ssl.create_default_context() - s = SSLStream( - sock, good_ctx, server_hostname="trio-test-1.example.org" - ) + s = SSLStream(sock, good_ctx, server_hostname="trio-test-1.example.org") assert s.transport_stream is sock @@ -593,6 +579,7 @@ async def test_renegotiation_randomized(mock_clock, client_ctx): mock_clock.autojump_threshold = 0 import random + r = random.Random(0) async def sleeper(_): @@ -629,8 +616,8 @@ async def expect(expected): await clear() for i in range(100): - b1 = bytes([i % 0xff]) - b2 = bytes([(2 * i) % 0xff]) + b1 = bytes([i % 0xFF]) + b2 = bytes([(2 * i) % 0xFF]) s.transport_stream.renegotiate() async with _core.open_nursery() as nursery: nursery.start_soon(send, b1) @@ -641,8 +628,8 @@ async def expect(expected): await clear() for i in range(100): - b1 = bytes([i % 0xff]) - b2 = bytes([(2 * i) % 0xff]) + b1 = bytes([i % 0xFF]) + b2 = bytes([(2 * i) % 0xFF]) await send(b1) s.transport_stream.renegotiate() await expect(b1) @@ -668,9 +655,7 @@ async def sleep_then_wait_writable(): await trio.sleep(1000) await s.wait_send_all_might_not_block() - with virtual_ssl_echo_server( - client_ctx, sleeper=sleeper_with_slow_send_all - ) as s: + with virtual_ssl_echo_server(client_ctx, sleeper=sleeper_with_slow_send_all) as s: await send(b"x") s.transport_stream.renegotiate() async with _core.open_nursery() as nursery: @@ -1023,7 +1008,7 @@ async def test_ssl_bad_shutdown_but_its_ok(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, server_kwargs={"https_compatible": True}, - client_kwargs={"https_compatible": True} + client_kwargs={"https_compatible": True}, ) async with _core.open_nursery() as nursery: @@ -1047,9 +1032,7 @@ async def test_ssl_handshake_failure_during_aclose(): async with ssl_echo_server_raw(expect_fail=True) as sock: # Don't configure trust correctly client_ctx = ssl.create_default_context() - s = SSLStream( - sock, client_ctx, server_hostname="trio-test-1.example.org" - ) + s = SSLStream(sock, client_ctx, server_hostname="trio-test-1.example.org") # It's a little unclear here whether aclose should swallow the error # or let it escape. We *do* swallow the error if it arrives when we're # sending close_notify, because both sides closing the connection @@ -1089,7 +1072,7 @@ async def test_ssl_https_compatibility_disagreement(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, server_kwargs={"https_compatible": False}, - client_kwargs={"https_compatible": True} + client_kwargs={"https_compatible": True}, ) async with _core.open_nursery() as nursery: @@ -1112,7 +1095,7 @@ async def test_https_mode_eof_before_handshake(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, server_kwargs={"https_compatible": True}, - client_kwargs={"https_compatible": True} + client_kwargs={"https_compatible": True}, ) async def server_expect_clean_eof(): @@ -1185,8 +1168,7 @@ async def test_selected_alpn_protocol_when_not_set(client_ctx): assert client.selected_alpn_protocol() is None assert server.selected_alpn_protocol() is None - assert client.selected_alpn_protocol() == \ - server.selected_alpn_protocol() + assert client.selected_alpn_protocol() == server.selected_alpn_protocol() async def test_selected_npn_protocol_before_handshake(client_ctx): @@ -1211,8 +1193,7 @@ async def test_selected_npn_protocol_when_not_set(client_ctx): assert client.selected_npn_protocol() is None assert server.selected_npn_protocol() is None - assert client.selected_npn_protocol() == \ - server.selected_npn_protocol() + assert client.selected_npn_protocol() == server.selected_npn_protocol() async def test_get_channel_binding_before_handshake(client_ctx): @@ -1235,8 +1216,7 @@ async def test_get_channel_binding_after_handshake(client_ctx): assert client.get_channel_binding() is not None assert server.get_channel_binding() is not None - assert client.get_channel_binding() == \ - server.get_channel_binding() + assert client.get_channel_binding() == server.get_channel_binding() async def test_getpeercert(client_ctx): @@ -1249,10 +1229,7 @@ async def test_getpeercert(client_ctx): assert server.getpeercert() is None print(client.getpeercert()) - assert ( - ("DNS", "trio-test-1.example.org") - in client.getpeercert()["subjectAltName"] - ) + assert ("DNS", "trio-test-1.example.org") in client.getpeercert()["subjectAltName"] async def test_SSLListener(client_ctx): @@ -1265,9 +1242,7 @@ async def setup(**kwargs): transport_client = await open_tcp_stream(*listen_sock.getsockname()) ssl_client = SSLStream( - transport_client, - client_ctx, - server_hostname="trio-test-1.example.org" + transport_client, client_ctx, server_hostname="trio-test-1.example.org", ) return listen_sock, ssl_listener, ssl_client diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index c5be0e393e..1f489db50c 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -7,8 +7,15 @@ from functools import partial from .. import ( - _core, move_on_after, fail_after, sleep, sleep_forever, Process, - open_process, run_process, TrioDeprecationWarning + _core, + move_on_after, + fail_after, + sleep, + sleep_forever, + Process, + open_process, + run_process, + TrioDeprecationWarning, ) from .._core.tests.tutil import slow, skip_if_fbsd_pipes_broken from ..testing import wait_all_tasks_blocked @@ -183,9 +190,7 @@ async def drain_one(stream, count, digit): assert await stream.receive_some(len(newline)) == newline nursery.start_soon(drain_one, proc.stdout, request, idx * 2) - nursery.start_soon( - drain_one, proc.stderr, request * 2, idx * 2 + 1 - ) + nursery.start_soon(drain_one, proc.stderr, request * 2, idx * 2 + 1) with fail_after(5): await proc.stdin.send_all(b"12") @@ -210,7 +215,7 @@ async def drain_one(stream, count, digit): async def test_run(): - data = bytes(random.randint(0, 255) for _ in range(2**18)) + data = bytes(random.randint(0, 255) for _ in range(2 ** 18)) result = await run_process( CAT, stdin=data, capture_stdout=True, capture_stderr=True @@ -269,8 +274,7 @@ async def test_run_check(): @skip_if_fbsd_pipes_broken async def test_run_with_broken_pipe(): result = await run_process( - [sys.executable, "-c", "import sys; sys.stdin.close()"], - stdin=b"x" * 131072, + [sys.executable, "-c", "import sys; sys.stdin.close()"], stdin=b"x" * 131072, ) assert result.returncode == 0 assert result.stdout is result.stderr is None @@ -402,6 +406,7 @@ def test_waitid_eintr(): # This only matters on PyPy (where we're coding EINTR handling # ourselves) but the test works on all waitid platforms. from .._subprocess_platform import wait_child_exiting + if not wait_child_exiting.__module__.endswith("waitid"): pytest.skip("waitid only") from .._subprocess_platform.waitid import sync_wait_reapable @@ -444,9 +449,7 @@ async def custom_deliver_cancel(proc): async with _core.open_nursery() as nursery: nursery.start_soon( - partial( - run_process, SLEEP(9999), deliver_cancel=custom_deliver_cancel - ) + partial(run_process, SLEEP(9999), deliver_cancel=custom_deliver_cancel) ) await wait_all_tasks_blocked() nursery.cancel_scope.cancel() diff --git a/trio/tests/test_sync.py b/trio/tests/test_sync.py index e4de476993..229dea301c 100644 --- a/trio/tests/test_sync.py +++ b/trio/tests/test_sync.py @@ -111,6 +111,7 @@ async def test_CapacityLimiter(): async def test_CapacityLimiter_inf(): from math import inf + c = CapacityLimiter(inf) repr(c) # smoke test assert c.total_tokens == inf @@ -240,9 +241,7 @@ async def test_Semaphore_bounded(): assert bs.value == 1 -@pytest.mark.parametrize( - "lockcls", [Lock, StrictFIFOLock], ids=lambda fn: fn.__name__ -) +@pytest.mark.parametrize("lockcls", [Lock, StrictFIFOLock], ids=lambda fn: fn.__name__) async def test_Lock_and_StrictFIFOLock(lockcls): l = lockcls() # noqa assert not l.locked() @@ -548,7 +547,7 @@ async def loopy(name, lock_like): # The first three could be in any order due to scheduling randomness, # but after that they should repeat in the same order for i in range(LOOPS): - assert record[3 * i:3 * (i + 1)] == initial_order + assert record[3 * i : 3 * (i + 1)] == initial_order @generic_lock_test diff --git a/trio/tests/test_testing.py b/trio/tests/test_testing.py index e73624a67c..460200b28c 100644 --- a/trio/tests/test_testing.py +++ b/trio/tests/test_testing.py @@ -146,7 +146,8 @@ async def test_assert_checkpoints(recwarn): # if you have a schedule point but not a cancel point, or vice-versa, then # that's not a checkpoint. for partial_yield in [ - _core.checkpoint_if_cancelled, _core.cancel_shielded_checkpoint + _core.checkpoint_if_cancelled, + _core.cancel_shielded_checkpoint, ]: print(partial_yield) with pytest.raises(AssertionError): @@ -171,7 +172,8 @@ async def test_assert_no_checkpoints(recwarn): # if you have a schedule point but not a cancel point, or vice-versa, then # that doesn't make *either* version of assert_{no_,}yields happy. for partial_yield in [ - _core.checkpoint_if_cancelled, _core.cancel_shielded_checkpoint + _core.checkpoint_if_cancelled, + _core.cancel_shielded_checkpoint, ]: print(partial_yield) with pytest.raises(AssertionError): @@ -215,9 +217,7 @@ async def f2(seq): nursery.start_soon(f2, seq) async with seq(5): await wait_all_tasks_blocked() - assert record == [ - ("f2", 0), ("f1", 1), ("f2", 2), ("f1", 3), ("f1", 4) - ] + assert record == [("f2", 0), ("f1", 1), ("f2", 2), ("f1", 3), ("f1", 4)] seq = Sequencer() # Catches us if we try to re-use a sequence point: @@ -321,11 +321,7 @@ async def test_mock_clock_autojump(mock_clock): virtual_start = _core.current_time() real_duration = time.perf_counter() - real_start - print( - "Slept {} seconds in {} seconds".format( - 10 * sum(range(10)), real_duration - ) - ) + print("Slept {} seconds in {} seconds".format(10 * sum(range(10)), real_duration)) assert real_duration < 1 mock_clock.autojump_threshold = 0.02 @@ -571,9 +567,7 @@ def close_hook(): record.append("close_hook") mss2 = MemorySendStream( - send_all_hook, - wait_send_all_might_not_block_hook, - close_hook, + send_all_hook, wait_send_all_might_not_block_hook, close_hook, ) assert mss2.send_all_hook is send_all_hook diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 6f5d2b6229..b4acae8b58 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -8,8 +8,11 @@ from .. import Event, CapacityLimiter, sleep from ..testing import wait_all_tasks_blocked from .._threads import ( - to_thread_run_sync, current_default_thread_limiter, from_thread_run, - from_thread_run_sync, BlockingTrioPortal + to_thread_run_sync, + current_default_thread_limiter, + from_thread_run, + from_thread_run_sync, + BlockingTrioPortal, ) from .._core.tests.test_ki import ki_self @@ -35,9 +38,7 @@ def threadfn(): while child_thread.is_alive(): print("yawn") await sleep(0.01) - assert record == [ - ("start", child_thread), ("f", trio_thread), expected - ] + assert record == [("start", child_thread), ("f", trio_thread), expected] token = _core.current_trio_token() @@ -53,9 +54,7 @@ def f(record): record.append(("f", threading.current_thread())) raise ValueError - await check_case( - from_thread_run_sync, f, ("error", ValueError), trio_token=token - ) + await check_case(from_thread_run_sync, f, ("error", ValueError), trio_token=token) async def f(record): assert not _core.currently_ki_protected() @@ -101,6 +100,7 @@ def trio_thread_fn(): ki_self() finally: import sys + print("finally", sys.exc_info()) async def trio_thread_afn(): @@ -332,15 +332,9 @@ def thread_fn(cancel_scope): async def run_thread(event): with _core.CancelScope() as cancel_scope: await to_thread_run_sync( - thread_fn, - cancel_scope, - limiter=limiter_arg, - cancellable=cancel + thread_fn, cancel_scope, limiter=limiter_arg, cancellable=cancel, ) - print( - "run_thread finished, cancelled:", - cancel_scope.cancelled_caught - ) + print("run_thread finished, cancelled:", cancel_scope.cancelled_caught) event.set() async with _core.open_nursery() as nursery: @@ -521,9 +515,7 @@ async def test_trio_from_thread_token_kwarg(): # Test that to_thread_run_sync and spawned trio.from_thread.run_sync() can # use an explicitly defined token def thread_fn(token): - callee_token = from_thread_run_sync( - _core.current_trio_token, trio_token=token - ) + callee_token = from_thread_run_sync(_core.current_trio_token, trio_token=token) return callee_token caller_token = _core.current_trio_token() @@ -541,9 +533,7 @@ async def test_from_thread_no_token(): def test_run_fn_as_system_task_catched_badly_typed_token(): with pytest.raises(RuntimeError): - from_thread_run_sync( - _core.current_time, trio_token="Not TrioTokentype" - ) + from_thread_run_sync(_core.current_time, trio_token="Not TrioTokentype") async def test_do_in_trio_thread_from_trio_thread_legacy(): @@ -580,5 +570,6 @@ def worker_thread(token): def test_BlockingTrioPortal_deprecated_export(recwarn): import trio + btp = trio.BlockingTrioPortal assert btp is BlockingTrioPortal diff --git a/trio/tests/test_unix_pipes.py b/trio/tests/test_unix_pipes.py index 5b85716b75..55dd4e3734 100644 --- a/trio/tests/test_unix_pipes.py +++ b/trio/tests/test_unix_pipes.py @@ -74,7 +74,7 @@ async def test_receive_pipe(): async def test_pipes_combined(): write, read = await make_pipe() - count = 2**20 + count = 2 ** 20 async def sender(): big = bytearray(count) @@ -195,9 +195,7 @@ async def patched_wait_readable(*args, **kwargs): await orig_wait_readable(*args, **kwargs) await r.aclose() - monkeypatch.setattr( - _core._run.TheIOManager, "wait_readable", patched_wait_readable - ) + monkeypatch.setattr(_core._run.TheIOManager, "wait_readable", patched_wait_readable) s, r = await make_pipe() async with s, r: async with _core.open_nursery() as nursery: @@ -225,9 +223,7 @@ async def patched_wait_writable(*args, **kwargs): await orig_wait_writable(*args, **kwargs) await s.aclose() - monkeypatch.setattr( - _core._run.TheIOManager, "wait_writable", patched_wait_writable - ) + monkeypatch.setattr(_core._run.TheIOManager, "wait_writable", patched_wait_writable) s, r = await make_clogged_pipe() async with s, r: async with _core.open_nursery() as nursery: @@ -243,7 +239,7 @@ async def patched_wait_writable(*args, **kwargs): # other platforms is probably good enough. @pytest.mark.skipif( sys.platform.startswith("freebsd"), - reason="no way to make read() return a bizarro error on FreeBSD" + reason="no way to make read() return a bizarro error on FreeBSD", ) async def test_bizarro_OSError_from_receive(): # Make sure that if the read syscall returns some bizarro error, then we diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 009f9fa8f7..b842b930cf 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -5,9 +5,14 @@ from .. import _core from .._core.tests.tutil import ignore_coroutine_never_awaited_warnings from .._util import ( - signal_raise, ConflictDetector, is_main_thread, coroutine_or_error, - generic_function, Final, NoPublicConstructor, - SubclassingDeprecatedIn_v0_15_0 + signal_raise, + ConflictDetector, + is_main_thread, + coroutine_or_error, + generic_function, + Final, + NoPublicConstructor, + SubclassingDeprecatedIn_v0_15_0, ) from ..testing import wait_all_tasks_blocked @@ -53,11 +58,13 @@ async def wait_with_ul1(): def test_module_metadata_is_fixed_up(): import trio + assert trio.Cancelled.__module__ == "trio" assert trio.open_nursery.__module__ == "trio" assert trio.abc.Stream.__module__ == "trio.abc" assert trio.lowlevel.wait_task_rescheduled.__module__ == "trio.lowlevel" import trio.testing + assert trio.testing.trio_test.__module__ == "trio.testing" # Also check methods diff --git a/trio/tests/test_wait_for_object.py b/trio/tests/test_wait_for_object.py index 3c3830ea39..ac507a26f9 100644 --- a/trio/tests/test_wait_for_object.py +++ b/trio/tests/test_wait_for_object.py @@ -2,7 +2,7 @@ import pytest -on_windows = (os.name == "nt") +on_windows = os.name == "nt" # Mark all the tests in this file as being windows-only pytestmark = pytest.mark.skipif(not on_windows, reason="windows only") @@ -10,9 +10,13 @@ import trio from .. import _core from .. import _timeouts + if on_windows: from .._core._windows_cffi import ffi, kernel32 - from .._wait_for_object import WaitForSingleObject, WaitForMultipleObjects_sync + from .._wait_for_object import ( + WaitForSingleObject, + WaitForMultipleObjects_sync, + ) async def test_WaitForMultipleObjects_sync(): @@ -29,7 +33,7 @@ async def test_WaitForMultipleObjects_sync(): kernel32.SetEvent(handle1) WaitForMultipleObjects_sync(handle1) kernel32.CloseHandle(handle1) - print('test_WaitForMultipleObjects_sync one OK') + print("test_WaitForMultipleObjects_sync one OK") # Two handles, signal first handle1 = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) @@ -38,7 +42,7 @@ async def test_WaitForMultipleObjects_sync(): WaitForMultipleObjects_sync(handle1, handle2) kernel32.CloseHandle(handle1) kernel32.CloseHandle(handle2) - print('test_WaitForMultipleObjects_sync set first OK') + print("test_WaitForMultipleObjects_sync set first OK") # Two handles, signal second handle1 = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) @@ -47,7 +51,7 @@ async def test_WaitForMultipleObjects_sync(): WaitForMultipleObjects_sync(handle1, handle2) kernel32.CloseHandle(handle1) kernel32.CloseHandle(handle2) - print('test_WaitForMultipleObjects_sync set second OK') + print("test_WaitForMultipleObjects_sync set second OK") # Two handles, close first handle1 = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) @@ -56,7 +60,7 @@ async def test_WaitForMultipleObjects_sync(): with pytest.raises(OSError): WaitForMultipleObjects_sync(handle1, handle2) kernel32.CloseHandle(handle2) - print('test_WaitForMultipleObjects_sync close first OK') + print("test_WaitForMultipleObjects_sync close first OK") # Two handles, close second handle1 = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) @@ -65,7 +69,7 @@ async def test_WaitForMultipleObjects_sync(): with pytest.raises(OSError): WaitForMultipleObjects_sync(handle1, handle2) kernel32.CloseHandle(handle1) - print('test_WaitForMultipleObjects_sync close second OK') + print("test_WaitForMultipleObjects_sync close second OK") @slow @@ -89,7 +93,7 @@ async def test_WaitForMultipleObjects_sync_slow(): t1 = _core.current_time() assert TIMEOUT <= (t1 - t0) < 2.0 * TIMEOUT kernel32.CloseHandle(handle1) - print('test_WaitForMultipleObjects_sync_slow one OK') + print("test_WaitForMultipleObjects_sync_slow one OK") # Two handles, signal first handle1 = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) @@ -97,8 +101,7 @@ async def test_WaitForMultipleObjects_sync_slow(): t0 = _core.current_time() async with _core.open_nursery() as nursery: nursery.start_soon( - trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, - handle2 + trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, handle2, ) await _timeouts.sleep(TIMEOUT) kernel32.SetEvent(handle1) @@ -106,7 +109,7 @@ async def test_WaitForMultipleObjects_sync_slow(): assert TIMEOUT <= (t1 - t0) < 2.0 * TIMEOUT kernel32.CloseHandle(handle1) kernel32.CloseHandle(handle2) - print('test_WaitForMultipleObjects_sync_slow thread-set first OK') + print("test_WaitForMultipleObjects_sync_slow thread-set first OK") # Two handles, signal second handle1 = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) @@ -114,8 +117,7 @@ async def test_WaitForMultipleObjects_sync_slow(): t0 = _core.current_time() async with _core.open_nursery() as nursery: nursery.start_soon( - trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, - handle2 + trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, handle2, ) await _timeouts.sleep(TIMEOUT) kernel32.SetEvent(handle2) @@ -123,7 +125,7 @@ async def test_WaitForMultipleObjects_sync_slow(): assert TIMEOUT <= (t1 - t0) < 2.0 * TIMEOUT kernel32.CloseHandle(handle1) kernel32.CloseHandle(handle2) - print('test_WaitForMultipleObjects_sync_slow thread-set second OK') + print("test_WaitForMultipleObjects_sync_slow thread-set second OK") async def test_WaitForSingleObject(): @@ -135,7 +137,7 @@ async def test_WaitForSingleObject(): kernel32.SetEvent(handle) await WaitForSingleObject(handle) # should return at once kernel32.CloseHandle(handle) - print('test_WaitForSingleObject already set OK') + print("test_WaitForSingleObject already set OK") # Test already set, as int handle = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) @@ -143,21 +145,21 @@ async def test_WaitForSingleObject(): kernel32.SetEvent(handle) await WaitForSingleObject(handle_int) # should return at once kernel32.CloseHandle(handle) - print('test_WaitForSingleObject already set OK') + print("test_WaitForSingleObject already set OK") # Test already closed handle = kernel32.CreateEventA(ffi.NULL, True, False, ffi.NULL) kernel32.CloseHandle(handle) with pytest.raises(OSError): await WaitForSingleObject(handle) # should return at once - print('test_WaitForSingleObject already closed OK') + print("test_WaitForSingleObject already closed OK") # Not a handle with pytest.raises(TypeError): await WaitForSingleObject("not a handle") # Wrong type # with pytest.raises(OSError): # await WaitForSingleObject(99) # If you're unlucky, it actually IS a handle :( - print('test_WaitForSingleObject not a handle OK') + print("test_WaitForSingleObject not a handle OK") @slow @@ -185,7 +187,7 @@ async def signal_soon_async(handle): kernel32.CloseHandle(handle) t1 = _core.current_time() assert TIMEOUT <= (t1 - t0) < 2.0 * TIMEOUT - print('test_WaitForSingleObject_slow set from task OK') + print("test_WaitForSingleObject_slow set from task OK") # Test handle is SET after TIMEOUT in separate coroutine, as int @@ -200,7 +202,7 @@ async def signal_soon_async(handle): kernel32.CloseHandle(handle) t1 = _core.current_time() assert TIMEOUT <= (t1 - t0) < 2.0 * TIMEOUT - print('test_WaitForSingleObject_slow set from task as int OK') + print("test_WaitForSingleObject_slow set from task as int OK") # Test handle is CLOSED after 1 sec - NOPE see comment above @@ -215,4 +217,4 @@ async def signal_soon_async(handle): kernel32.CloseHandle(handle) t1 = _core.current_time() assert TIMEOUT <= (t1 - t0) < 2.0 * TIMEOUT - print('test_WaitForSingleObject_slow cancellation OK') + print("test_WaitForSingleObject_slow cancellation OK") diff --git a/trio/tests/test_windows_pipes.py b/trio/tests/test_windows_pipes.py index 864aaf768e..8fb29b632f 100644 --- a/trio/tests/test_windows_pipes.py +++ b/trio/tests/test_windows_pipes.py @@ -47,7 +47,7 @@ async def test_pipe_error_on_close(): async def test_pipes_combined(): write, read = await make_pipe() - count = 2**20 + count = 2 ** 20 replicas = 3 async def sender(): diff --git a/trio/tests/tools/test_gen_exports.py b/trio/tests/tools/test_gen_exports.py index 6c1fb0d668..e4e388c226 100644 --- a/trio/tests/tools/test_gen_exports.py +++ b/trio/tests/tools/test_gen_exports.py @@ -6,7 +6,9 @@ from shutil import copyfile from trio._tools.gen_exports import ( - get_public_methods, create_passthrough_args, process + get_public_methods, + create_passthrough_args, + process, ) SOURCE = '''from _run import _public @@ -43,7 +45,7 @@ def test_create_pass_through_args(): ("def f(one, *args)", "(one, *args)"), ( "def f(one, *args, kw1, kw2=None, **kwargs)", - "(one, *args, kw1=kw1, kw2=kw2, **kwargs)" + "(one, *args, kw1=kw1, kw2=kw2, **kwargs)", ), ] From 94a587f758f0597cd790505ca7bfbec17a247fb1 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 10:53:47 +0400 Subject: [PATCH 0164/1498] Fix test_warn_deprecated test --- trio/tests/test_deprecate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_deprecate.py b/trio/tests/test_deprecate.py index 4534bbb7c9..a11e9b8d1f 100644 --- a/trio/tests/test_deprecate.py +++ b/trio/tests/test_deprecate.py @@ -37,7 +37,7 @@ def deprecated_thing(): assert "water instead" in got.message.args[0] assert "/issues/1" in got.message.args[0] assert got.filename == filename - assert got.lineno == lineno + 1 + assert got.lineno == lineno - 1 def test_warn_deprecated_no_instead_or_issue(recwarn_always): From 8be9e123954482149c2de1f437e4f196dabc2e5a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 11:27:50 +0400 Subject: [PATCH 0165/1498] Fix a few ugly formattings --- trio/_core/tests/test_ki.py | 6 ++---- trio/_socket.py | 10 ++++------ trio/tests/test_util.py | 3 +-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/trio/_core/tests/test_ki.py b/trio/_core/tests/test_ki.py index 12bfe86793..51b60aebbf 100644 --- a/trio/_core/tests/test_ki.py +++ b/trio/_core/tests/test_ki.py @@ -536,10 +536,8 @@ def test_ki_wakes_us_up(): # https://bitbucket.org/pypy/pypy/issues/2623 import platform - buggy_wakeup_fd = platform.python_implementation() == "CPython" and sys.version_info < ( - 3, - 6, - 2, + buggy_wakeup_fd = ( + sys.version_info < (3, 6, 2) and platform.python_implementation() == "CPython" ) # lock is only needed to avoid an annoying race condition where the diff --git a/trio/_socket.py b/trio/_socket.py index a7b5303c21..1e544aece5 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -339,15 +339,13 @@ def _make_simple_sock_method_wrapper(methname, wait_fn, maybe_avail=False): async def wrapper(self, *args, **kwargs): return await self._nonblocking_helper(fn, args, kwargs, wait_fn) - wrapper.__doc__ = """Like :meth:`socket.socket.{}`, but async. + wrapper.__doc__ = f"""Like :meth:`socket.socket.{methname}`, but async. - """.format( - methname - ) + """ if maybe_avail: wrapper.__doc__ += ( - "Only available on platforms where :meth:`socket.socket.{}` " - "is available.".format(methname) + f"Only available on platforms where :meth:`socket.socket.{methname}`" + " is available." ) return wrapper diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index b842b930cf..b08676e622 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -58,13 +58,12 @@ async def wait_with_ul1(): def test_module_metadata_is_fixed_up(): import trio + import trio.testing assert trio.Cancelled.__module__ == "trio" assert trio.open_nursery.__module__ == "trio" assert trio.abc.Stream.__module__ == "trio.abc" assert trio.lowlevel.wait_task_rescheduled.__module__ == "trio.lowlevel" - import trio.testing - assert trio.testing.trio_test.__module__ == "trio.testing" # Also check methods From 2e42b1c16acddd4585769c18a0887b6d164aee80 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 14:43:11 +0400 Subject: [PATCH 0166/1498] Merge strings now that we have 88 characters --- trio/_core/_io_kqueue.py | 4 ++-- trio/_socket.py | 2 +- trio/_sync.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index dd5ecd210a..b194e85f53 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -81,7 +81,7 @@ def monitor_kevent(self, ident, filter): key = (ident, filter) if key in self._registered: raise _core.BusyResourceError( - "attempt to register multiple listeners for same " "ident/filter pair" + "attempt to register multiple listeners for same ident/filter pair" ) q = _core.UnboundedQueue() self._registered[key] = q @@ -95,7 +95,7 @@ async def wait_kevent(self, ident, filter, abort_func): key = (ident, filter) if key in self._registered: raise _core.BusyResourceError( - "attempt to register multiple listeners for same " "ident/filter pair" + "attempt to register multiple listeners for same ident/filter pair" ) self._registered[key] = _core.current_task() diff --git a/trio/_socket.py b/trio/_socket.py index 1e544aece5..2c367a055c 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -497,7 +497,7 @@ async def _resolve_address(self, address, flags): elif self._sock.family == _stdlib_socket.AF_INET6: if not isinstance(address, tuple) or not 2 <= len(address) <= 4: raise ValueError( - "address should be a (host, port, [flowinfo, [scopeid]]) " "tuple" + "address should be a (host, port, [flowinfo, [scopeid]]) tuple" ) elif self._sock.family == _stdlib_socket.AF_UNIX: await trio.lowlevel.checkpoint() diff --git a/trio/_sync.py b/trio/_sync.py index 212c7927a3..9c4fad3a18 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -317,7 +317,7 @@ def release_on_behalf_of(self, borrower): """ if borrower not in self._borrowers: raise RuntimeError( - "this borrower isn't holding any of this CapacityLimiter's " "tokens" + "this borrower isn't holding any of this CapacityLimiter's tokens" ) self._borrowers.remove(borrower) self._wake_waiters() From 614fe56c7e9e0ebf33d334cda2abf4b166fbb7e2 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 22 May 2020 14:44:03 +0400 Subject: [PATCH 0167/1498] Restore space at end of line We now have 88 characters. --- trio/_socket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_socket.py b/trio/_socket.py index 2c367a055c..ccd2a6ff20 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -344,8 +344,8 @@ async def wrapper(self, *args, **kwargs): """ if maybe_avail: wrapper.__doc__ += ( - f"Only available on platforms where :meth:`socket.socket.{methname}`" - " is available." + f"Only available on platforms where :meth:`socket.socket.{methname}` is " + "available." ) return wrapper From 37a9de79d1ca31c4653a949559f39947c03d2704 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 22 May 2020 21:54:44 -0700 Subject: [PATCH 0168/1498] Note-to-self for windows WaitableTimer objects --- notes-to-self/win-waitable-timer.py | 207 ++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 notes-to-self/win-waitable-timer.py diff --git a/notes-to-self/win-waitable-timer.py b/notes-to-self/win-waitable-timer.py new file mode 100644 index 0000000000..c831761358 --- /dev/null +++ b/notes-to-self/win-waitable-timer.py @@ -0,0 +1,207 @@ +# Sandbox for exploring the Windows "waitable timer" API. +# Cf https://github.com/python-trio/trio/issues/173 +# +# Observations: +# - if you set a timer in the far future, then block in +# WaitForMultipleObjects, then set the computer's clock forward by a few +# years (past the target sleep time), then the timer immediately wakes up +# (which is good!) +# - if you set a timer in the past, then it wakes up immediately + +# Random thoughts: +# - top-level API sleep_until_datetime +# - portable manages the heap of outstanding sleeps, runs a system task to +# wait for the next one, wakes up tasks when their deadline arrives, etc. +# - non-portable code: async def sleep_until_datetime_raw, which simply blocks +# until the given time using system-specific methods. Can assume that there +# is only one call to this method at a time. +# Actually, this should be a method, so it can hold persistent state (e.g. +# timerfd). +# Can assume that the datetime passed in has tzinfo=timezone.utc +# Need a way to override this object for testing. +# +# should we expose wake-system-on-alarm functionality? windows and linux both +# make this fairly straightforward, but you obviously need to use a separate +# time source + +import cffi +from datetime import datetime, timedelta, timezone +import time + +import trio +from trio._core._windows_cffi import (ffi, kernel32, raise_winerror) + +try: + ffi.cdef( + """ +typedef struct _PROCESS_LEAP_SECOND_INFO { + ULONG Flags; + ULONG Reserved; +} PROCESS_LEAP_SECOND_INFO, *PPROCESS_LEAP_SECOND_INFO; + +typedef struct _SYSTEMTIME { + WORD wYear; + WORD wMonth; + WORD wDayOfWeek; + WORD wDay; + WORD wHour; + WORD wMinute; + WORD wSecond; + WORD wMilliseconds; +} SYSTEMTIME, *PSYSTEMTIME, *LPSYSTEMTIME; +""" + ) +except cffi.CDefError: + pass + +ffi.cdef( + """ +typedef LARGE_INTEGER FILETIME; +typedef FILETIME* LPFILETIME; + +HANDLE CreateWaitableTimerW( + LPSECURITY_ATTRIBUTES lpTimerAttributes, + BOOL bManualReset, + LPCWSTR lpTimerName +); + +BOOL SetWaitableTimer( + HANDLE hTimer, + const LPFILETIME lpDueTime, + LONG lPeriod, + void* pfnCompletionRoutine, + LPVOID lpArgToCompletionRoutine, + BOOL fResume +); + +BOOL SetProcessInformation( + HANDLE hProcess, + /* Really an enum, PROCESS_INFORMATION_CLASS */ + int32_t ProcessInformationClass, + LPVOID ProcessInformation, + DWORD ProcessInformationSize +); + +void GetSystemTimeAsFileTime( + LPFILETIME lpSystemTimeAsFileTime +); + +BOOL SystemTimeToFileTime( + const SYSTEMTIME *lpSystemTime, + LPFILETIME lpFileTime +); +""", + override=True +) + +ProcessLeapSecondInfo = 8 +PROCESS_LEAP_SECOND_INFO_FLAG_ENABLE_SIXTY_SECOND = 1 + + +def set_leap_seconds_enabled(enabled): + plsi = ffi.new("PROCESS_LEAP_SECOND_INFO*") + if enabled: + plsi.Flags = PROCESS_LEAP_SECOND_INFO_FLAG_ENABLE_SIXTY_SECOND + else: + plsi.Flags = 0 + plsi.Reserved = 0 + if not kernel32.SetProcessInformation( + ffi.cast("HANDLE", -1), # current process + ProcessLeapSecondInfo, + plsi, + ffi.sizeof("PROCESS_LEAP_SECOND_INFO"), + ): + raise_winerror() + + +def now_as_filetime(): + ft = ffi.new("LARGE_INTEGER*") + kernel32.GetSystemTimeAsFileTime(ft) + return ft[0] + + +# "FILETIME" is a specific Windows time representation, that I guess was used +# for files originally but now gets used in all kinds of non-file-related +# places. Essentially: integer count of "ticks" since an epoch in 1601, where +# each tick is 100 nanoseconds, in UTC but pretending that leap seconds don't +# exist. (Fortunately, the Python datetime module also pretends that +# leapseconds don't exist, so we can use datetime arithmetic to compute +# FILETIME values.) +# +# https://docs.microsoft.com/en-us/windows/win32/sysinfo/file-times +# +# This page has FILETIME convertors and can be useful for debugging: +# +# https://www.epochconverter.com/ldap +# +FILETIME_TICKS_PER_SECOND = 10**7 +FILETIME_EPOCH = datetime.strptime( + '1601-01-01 00:00:00 Z', '%Y-%m-%d %H:%M:%S %z' +) +# XXX THE ABOVE IS WRONG: +# +# https://techcommunity.microsoft.com/t5/networking-blog/leap-seconds-for-the-appdev-what-you-should-know/ba-p/339813# +# +# Sometimes Windows FILETIME does include leap seconds! It depends on Windows +# version, process-global state, environment state, registry settings, and who +# knows what else! +# +# So actually the only correct way to convert a YMDhms-style representation of +# a time into a FILETIME is to use SystemTimeToFileTime +# +# ...also I can't even run this test on my VM, because it's running an ancient +# version of Win10 that doesn't have leap second support. Also also, Windows +# only tracks leap seconds since they added leap second support, and there +# haven't been any, so right now things work correctly either way. +# +# It is possible to insert some fake leap seconds for testing, if you want. + + +def py_datetime_to_win_filetime(dt): + # We'll want to call this on every datetime as it comes in + #dt = dt.astimezone(timezone.utc) + assert dt.tzinfo is timezone.utc + return round( + (dt - FILETIME_EPOCH).total_seconds() * FILETIME_TICKS_PER_SECOND + ) + + +async def main(): + h = kernel32.CreateWaitableTimerW(ffi.NULL, True, ffi.NULL) + if not h: + raise_winerror() + print(h) + + SECONDS = 2 + + wakeup = datetime.now(timezone.utc) + timedelta(seconds=SECONDS) + wakeup_filetime = py_datetime_to_win_filetime(wakeup) + wakeup_cffi = ffi.new("LARGE_INTEGER *") + wakeup_cffi[0] = wakeup_filetime + + print(wakeup_filetime, wakeup_cffi) + + print(f"Sleeping for {SECONDS} seconds (until {wakeup})") + + if not kernel32.SetWaitableTimer( + h, + wakeup_cffi, + 0, + ffi.NULL, + ffi.NULL, + False, + ): + raise_winerror() + + await trio.hazmat.WaitForSingleObject(h) + + print(f"Current FILETIME: {now_as_filetime()}") + set_leap_seconds_enabled(False) + print(f"Current FILETIME: {now_as_filetime()}") + set_leap_seconds_enabled(True) + print(f"Current FILETIME: {now_as_filetime()}") + set_leap_seconds_enabled(False) + print(f"Current FILETIME: {now_as_filetime()}") + + +trio.run(main) From 1d1f871153b636fb18efe7091f7382e4fa57c41d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 22 May 2020 23:50:26 -0700 Subject: [PATCH 0169/1498] Add thread cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On my Linux laptop, this makes 'await trio.to_thread.run_sync(lambda: None)' about twice as fast, from ~150 µs to ~75 µs. Closes: gh-6 Test program: import trio import time COUNT = 10000 async def main(): while True: start = time.monotonic() for _ in range(COUNT): await trio.to_thread.run_sync(lambda: None) end = time.monotonic() print("{:.2f} µs/job".format((end - start) / COUNT * 1e6)) trio.run(main) --- docs/source/conf.py | 1 + docs/source/reference-lowlevel.rst | 6 + newsfragments/6.feature.rst | 6 + notes-to-self/tiny-thread-pool.py | 143 ---------------------- trio/_core/__init__.py | 2 + trio/_core/_thread_cache.py | 168 ++++++++++++++++++++++++++ trio/_core/tests/test_thread_cache.py | 120 ++++++++++++++++++ trio/_threads.py | 37 +++--- trio/lowlevel.py | 1 + trio/tests/test_threads.py | 8 +- 10 files changed, 325 insertions(+), 167 deletions(-) create mode 100644 newsfragments/6.feature.rst delete mode 100644 notes-to-self/tiny-thread-pool.py create mode 100644 trio/_core/_thread_cache.py create mode 100644 trio/_core/tests/test_thread_cache.py diff --git a/docs/source/conf.py b/docs/source/conf.py index 9dddc93400..6c7fb02455 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -50,6 +50,7 @@ ("py:class", "math.inf"), ("py:exc", "Anything else"), ("py:class", "async function"), + ("py:class", "sync function"), ] autodoc_inherit_docstrings = False default_role = "obj" diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 53e5cf8a8c..b196cd0b43 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -270,6 +270,12 @@ Trio tokens .. autofunction:: current_trio_token +Spawning threads +================ + +.. autofunction:: start_thread_soon + + Safer KeyboardInterrupt handling ================================ diff --git a/newsfragments/6.feature.rst b/newsfragments/6.feature.rst new file mode 100644 index 0000000000..8baa3a27d9 --- /dev/null +++ b/newsfragments/6.feature.rst @@ -0,0 +1,6 @@ +To speed up `trio.to_thread.run_sync`, Trio now caches and re-uses +worker threads. + +And in case you have some exotic use case where you need to spawn +threads manually, but want to take advantage of Trio's cache, you can +do that using the new `trio.lowlevel.start_thread_soon`. diff --git a/notes-to-self/tiny-thread-pool.py b/notes-to-self/tiny-thread-pool.py deleted file mode 100644 index b5de1c37b3..0000000000 --- a/notes-to-self/tiny-thread-pool.py +++ /dev/null @@ -1,143 +0,0 @@ -# This is some very messy notes on how we might implement a thread cache - -import threading -import Queue - -# idea: -# -# unbounded thread pool; tracks how many threads are "available" and how much -# work there is to do; if work > available threads, spawn a new thread -# -# if a thread sits idle for >N ms, exit -# -# we don't need to support job cancellation -# -# we do need to mark a thread as "available" just before it -# signals back to Trio that it's done, to maintain the invariant that all -# unavailable threads are inside the limiter= protection -# -# maintaining this invariant while exiting can be a bit tricky -# -# maybe a simple target should be to always have 1 idle thread - -# XX we can't use a single shared dispatch queue, because we need LIFO -# scheduling, or else the idle-thread timeout won't work! -# -# instead, keep a list/deque/OrderedDict/something of idle threads, and -# dispatch by popping one off; put things back by pushing them on the end -# maybe one shared dispatch Lock, plus a Condition for each thread -# dispatch by dropping the job into the place where the thread can see it and -# then signalling its Condition? or could have separate locks - -@attr.s(frozen=True) -class Job: - main = attr.ib() - main_args = attr.ib() - finish = attr.ib() - finish_args = attr.ib() - -class EXIT: - pass - -class ThreadCache: - def __init__(self): - self._lock = threading.Lock() - self._idle_workers = deque() - self._closed = False - - def close(self): - self._closed = True - with self._lock: - while self._idle_workers: - self._idle_workers.pop().submit(None) - - def submit(self, job): - with self._lock: - if not self._idle_workers: - WorkerThread(self, self._lock, job) - else: - worker = self._idle_workers.pop() - worker.submit(job) - - # Called from another thread - # Must be called with the lock held - def remove_idle_worker(self, worker): - self._idle_workers.remove(worker) - - # Called from another thread - # Lock is *not* held - def add_idle_worker(self, worker): - if self._closed: - with self._lock: - worker.submit - self._idle_workers.append(worker) - -# XX thread name - -IDLE_TIMEOUT = 1.0 - -class WorkerThread: - def __init__(self, cache, lock, initial_job): - self._cache = cache - self._condition = threading.Condition(lock) - self._job = None - self._thread = threading.Thread( - target=self._loop, args=(initial_job,), daemon=True) - self._thread.start() - - # Must be called with the lock held - def submit(self, job): - assert self._job is None - self._job = job - self._condition.notify() - - def _loop(self, initial_job): - self._run_job(initial_job) - while True: - with self._condition: - self._condition.wait(IDLE_TIMEOUT): - job = self._job - self._job = None - if job is None: - self._cache.remove_idle_worker(self) - return - # Dropped the lock, and have a job to do - self._run_job(job) - - def _run_job(self, job): - job.main(*job.main_args) - self._cache.add_idle_worker(self) - job.finish(*job.finish_args) - - -# Probably the interface should be: trio.lowlevel.call_soon_in_worker_thread? - -# Enqueueing work: -# put into unbounded queue -# with lock: -# if idle_threads: -# idle_threads -= 1 -# else: -# spawn a new thread (it starts out non-idle) -# -# Thread shutdown: -# with lock: -# idle_threads -= 1 -# check for work one last time, and then either exit or do it -# -# Thread startup: -# -# check for work -# while True: -# mark self as idle -# check for work (with timeout) -# either do work or shutdown - -# if we want to support QueueUserAPC cancellation, we need a way to get back -# the thread id... maybe that just works like -# -# def WaitForSingleObjectEx_thread_fn(...): -# with lock: -# check if already cancelled -# put our thread id where main thread can find it -# WaitForSingleObjectEx(...) diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index c28b7f4078..136bfe6b98 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -68,6 +68,8 @@ from ._local import RunVar +from ._thread_cache import start_thread_soon + # Kqueue imports try: from ._run import current_kqueue, monitor_kevent, wait_kevent diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py new file mode 100644 index 0000000000..1ecd4bdfec --- /dev/null +++ b/trio/_core/_thread_cache.py @@ -0,0 +1,168 @@ +from threading import Thread, Lock +import sys +import outcome +from itertools import count + +# The "thread cache" is a simple unbounded thread pool, i.e., it automatically +# spawns as many threads as needed to handle all the requests its given. Its +# only purpose is to cache worker threads so that they don't have to be +# started from scratch every time we want to delegate some work to a thread. +# It's expected that some higher-level code will track how many threads are in +# use to avoid overwhelming the system (e.g. the limiter= argument to +# trio.to_thread.run_sync). +# +# To maximize sharing, there's only one thread cache per process, even if you +# have multiple calls to trio.run. +# +# Guarantees: +# +# It's safe to call start_thread_soon simultaneously from +# multiple threads. +# +# Idle threads are chosen in LIFO order, i.e. we *don't* spread work evenly +# over all threads. Instead we try to let some threads do most of the work +# while others sit idle as much as possible. Compared to FIFO, this has better +# memory cache behavior, and it makes it easier to detect when we have too +# many threads, so idle ones can exit. +# +# This code assumes that 'dict' has the following properties: +# +# - __setitem__, __delitem__, and popitem are all thread-safe and atomic with +# respect to each other. This is guaranteed by the GIL. +# +# - popitem returns the most-recently-added item (i.e., __setitem__ + popitem +# give you a LIFO queue). This relies on dicts being insertion-ordered, like +# they are in py36+. + +# How long a thread will idle waiting for new work before gives up and exits. +# This value is pretty arbitrary; I don't think it matters too much. +IDLE_TIMEOUT = 10 # seconds + +name_counter = count() + + +class WorkerThread: + def __init__(self, thread_cache): + self._job = None + self._thread_cache = thread_cache + # This Lock is used in an unconventional way. + # + # "Unlocked" means we have a pending job that's been assigned to us; + # "locked" means that we don't. + # + # Initially we have no job, so it starts out in locked state. + self._worker_lock = Lock() + self._worker_lock.acquire() + thread = Thread(target=self._work, daemon=True) + thread.name = f"Trio worker thread {next(name_counter)}" + thread.start() + + def _work(self): + while True: + if self._worker_lock.acquire(timeout=IDLE_TIMEOUT): + # We got a job + fn, deliver = self._job + self._job = None + result = outcome.capture(fn) + # Tell the cache that we're available to be assigned a new + # job. We do this *before* calling 'deliver', so that if + # 'deliver' triggers a new job, it can be assigned to us + # instead of spawning a new thread. + self._thread_cache._idle_workers[self] = None + deliver(result) + else: + # Timeout acquiring lock, so we can probably exit. But, + # there's a race condition: we might be assigned a job *just* + # as we're about to exit. So we have to check. + try: + del self._thread_cache._idle_workers[self] + except KeyError: + # Someone else removed us from the idle worker queue, so + # they must be in the process of assigning us a job - loop + # around and wait for it. + continue + else: + # We successfully removed ourselves from the idle + # worker queue, so no more jobs are incoming; it's safe to + # exit. + return + + +class ThreadCache: + def __init__(self): + self._idle_workers = {} + self._cache_lock = Lock() + + def start_thread_soon(self, deliver, fn): + try: + worker, _ = self._idle_workers.popitem() + except KeyError: + worker = WorkerThread(self) + worker._job = (fn, deliver) + worker._worker_lock.release() + + +THREAD_CACHE = ThreadCache() + + +def start_thread_soon(deliver, fn): + """Runs ``deliver(outcome.capture(fn))`` in a worker thread. + + Generally ``fn`` does some blocking work, and ``deliver`` delivers the + result back to whoever is interested. + + This is a low-level, no-frills interface, very similar to using + `threading.Thread` to spawn a thread directly. The main difference is + that this function tries to re-use threads when possible, so it can be + a bit faster than `threading.Thread`. + + Worker threads have the `~threading.Thread.daemon` flag set, which means + that if your main thread exits, worker threads will automatically be + killed. If you want to make sure that your ``fn`` runs to completion, then + you should make sure that the main thread remains alive until ``deliver`` + is called. + + It is safe to call this function simultaneously from multiple threads. + + Args: + + deliver (sync function): Takes the `outcome.Outcome` of ``fn``, and + delivers it. *Must not block.* + + fn (sync function): Performs arbitrary blocking work. + + Because worker threads are cached and reused for multiple calls, neither + function should mutate thread-level state, like `threading.local` objects + – or if they do, they should be careful to revert their changes before + returning. + + Note: + + The split between ``fn`` and ``deliver`` serves two purposes. First, + it's convenient, since most callers need something like this anyway. + + Second, it avoids a small race condition that could cause too many + threads to be spawned. Consider a program that wants to run several + jobs sequentially on a thread, so the main thread submits a job, waits + for it to finish, submits another job, etc. In theory, this program + should only need one worker thread. But what could happen is: + + 1. Worker thread: First job finishes, and calls ``deliver``. + + 2. Main thread: receives notification that the job finished, and calls + ``start_thread_soon``. + + 3. Main thread: sees that no worker threads are marked idle, so spawns + a second worker thread. + + 4. Original worker thread: marks itself as idle. + + To avoid this, threads mark themselves as idle *before* calling + ``deliver``. + + Is this potential extra thread a major problem? Maybe not, but it's + easy enough to avoid, and we figure that if the user is trying to + limit how many threads they're using then it's polite to respect that. + + """ + THREAD_CACHE.start_thread_soon(deliver, fn) diff --git a/trio/_core/tests/test_thread_cache.py b/trio/_core/tests/test_thread_cache.py new file mode 100644 index 0000000000..6c6fc3a104 --- /dev/null +++ b/trio/_core/tests/test_thread_cache.py @@ -0,0 +1,120 @@ +import pytest +import threading +from queue import Queue +import time + +from .tutil import slow +from .. import _thread_cache +from .._thread_cache import start_thread_soon, ThreadCache + + +def test_thread_cache_basics(): + q = Queue() + + def fn(): + raise RuntimeError("hi") + + def deliver(outcome): + q.put(outcome) + + start_thread_soon(deliver, fn) + + outcome = q.get() + with pytest.raises(RuntimeError, match="hi"): + outcome.unwrap() + + +@slow +def test_spawning_new_thread_from_deliver_reuses_starting_thread(): + # We know that no-one else is using the thread cache, so if we keep + # submitting new jobs the instant the previous one is finished, we should + # keep getting the same thread over and over. This tests both that the + # thread cache is LIFO, and that threads can be assigned new work *before* + # deliver exits. + + # Make sure there are a few threads running, so if we weren't LIFO then we + # could grab the wrong one. + q = Queue() + COUNT = 5 + for _ in range(COUNT): + start_thread_soon(lambda result: q.put(result), lambda: time.sleep(1)) + for _ in range(COUNT): + q.get().unwrap() + + seen_threads = set() + done = threading.Event() + + def deliver(n, _): + print(n) + seen_threads.add(threading.current_thread()) + if n == 0: + done.set() + else: + start_thread_soon(lambda _: deliver(n - 1, _), lambda: None) + + start_thread_soon(lambda _: deliver(5, _), lambda: None) + + done.wait() + + assert len(seen_threads) == 1 + + +@slow +def test_idle_threads_exit(monkeypatch): + # Temporarily set the idle timeout to something tiny, to speed up the + # test. (But non-zero, so that the worker loop will at least yield the + # CPU.) + monkeypatch.setattr(_thread_cache, "IDLE_TIMEOUT", 0.0001) + + q = Queue() + start_thread_soon(lambda _: q.put(threading.current_thread()), lambda: None) + seen_thread = q.get() + # Since the idle timeout is 0, after sleeping for 1 second, the thread + # should have exited + time.sleep(1) + assert not seen_thread.is_alive() + + +def test_race_between_idle_exit_and_job_assignment(monkeypatch): + # This is a lock where the first few times you try to acquire it with a + # timeout, it waits until the lock is available and then pretends to time + # out. Using this in our thread cache implementation causes the following + # sequence: + # + # 1. start_thread_soon grabs the worker thread, assigns it a job, and + # releases its lock. + # 2. The worker thread wakes up (because the lock has been released), but + # the JankyLock lies to it and tells it that the lock timed out. So the + # worker thread tries to exit. + # 3. The worker thread checks for the race between exiting and being + # assigned a job, and discovers that it *is* in the process of being + # assigned a job, so it loops around and tries to acquire the lock + # again. + # 4. Eventually the JankyLock admits that the lock is available, and + # everything proceeds as normal. + + class JankyLock: + def __init__(self): + self._lock = threading.Lock() + self._counter = 3 + + def acquire(self, timeout=None): + self._lock.acquire() + if timeout is None: + return True + else: + if self._counter > 0: + self._counter -= 1 + self._lock.release() + return False + return True + + def release(self): + self._lock.release() + + monkeypatch.setattr(_thread_cache, "Lock", JankyLock) + + tc = ThreadCache() + done = threading.Event() + tc.start_thread_soon(lambda _: done.set(), lambda: None) + done.wait() diff --git a/trio/_threads.py b/trio/_threads.py index 92e2b5dc0f..f441952b55 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -14,6 +14,7 @@ disable_ki_protection, RunVar, TrioToken, + start_thread_soon, ) from ._util import coroutine_or_error @@ -253,7 +254,7 @@ async def to_thread_run_sync(sync_fn, *args, cancellable=False, limiter=None): # for the result – or None if this function was cancelled and we should # discard the result. task_register = [trio.lowlevel.current_task()] - name = "trio-worker-{}".format(next(_thread_counter)) + name = f"trio.to_thread.run_sync-{next(_thread_counter)}" placeholder = ThreadPlaceholder(name) # This function gets scheduled into the Trio run loop to deliver the @@ -273,32 +274,26 @@ def do_release_then_return_result(): if task_register[0] is not None: trio.lowlevel.reschedule(task_register[0], result) - # This is the function that runs in the worker thread to do the actual - # work and then schedule the call to report_back_in_trio_thread_fn - # Since this is spawned in a new thread, the trio token needs to be passed - # explicitly to it so it can inject it into thread local storage - def worker_thread_fn(trio_token): - TOKEN_LOCAL.token = trio_token + current_trio_token = trio.lowlevel.current_trio_token() + + def worker_fn(): + TOKEN_LOCAL.token = current_trio_token try: - result = outcome.capture(sync_fn, *args) - try: - trio_token.run_sync_soon(report_back_in_trio_thread_fn, result) - except trio.RunFinishedError: - # The entire run finished, so our particular task is certainly - # long gone -- it must have cancelled. - pass + return sync_fn(*args) finally: del TOKEN_LOCAL.token + def deliver_worker_fn_result(result): + try: + current_trio_token.run_sync_soon(report_back_in_trio_thread_fn, result) + except trio.RunFinishedError: + # The entire run finished, so our particular task is certainly + # long gone -- it must have been cancelled and abandoned us. + pass + await limiter.acquire_on_behalf_of(placeholder) try: - # daemon=True because it might get left behind if we cancel, and in - # this case shouldn't block process exit. - current_trio_token = trio.lowlevel.current_trio_token() - thread = threading.Thread( - target=worker_thread_fn, args=(current_trio_token,), name=name, daemon=True, - ) - thread.start() + start_thread_soon(deliver_worker_fn_result, worker_fn) except: limiter.release_on_behalf_of(placeholder) raise diff --git a/trio/lowlevel.py b/trio/lowlevel.py index 21ec0597df..3ce3e741ba 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -41,6 +41,7 @@ wait_readable, wait_writable, notify_closing, + start_thread_soon, ) # Unix-specific symbols diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index b4acae8b58..632ce13656 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -239,7 +239,9 @@ async def child(q, cancellable): # Make sure that if trio.run exits, and then the thread finishes, then that's # handled gracefully. (Requires that the thread result machinery be prepared # for call_soon to raise RunFinishedError.) -def test_run_in_worker_thread_abandoned(capfd): +def test_run_in_worker_thread_abandoned(capfd, monkeypatch): + monkeypatch.setattr(_core._thread_cache, "IDLE_TIMEOUT", 0.01) + q1 = stdlib_queue.Queue() q2 = stdlib_queue.Queue() @@ -426,10 +428,10 @@ def release_on_behalf_of(self, borrower): async def test_run_in_worker_thread_fail_to_spawn(monkeypatch): # Test the unlikely but possible case where trying to spawn a thread fails - def bad_start(self): + def bad_start(self, *args): raise RuntimeError("the engines canna take it captain") - monkeypatch.setattr(threading.Thread, "start", bad_start) + monkeypatch.setattr(_core._thread_cache.ThreadCache, "start_thread_soon", bad_start) limiter = current_default_thread_limiter() assert limiter.borrowed_tokens == 0 From d248eb44e635ddf4c2457adeb3bc279f859a5b6b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 23 May 2020 00:58:02 -0700 Subject: [PATCH 0170/1498] [ci] Now that cpython 3.9 has branched off, start testing on it --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d87147f8e5..c6d93471db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ jobs: - python: 3.6-dev - python: 3.7-dev - python: 3.8-dev + - python: 3.9-dev - python: nightly script: From f4a57d5a9de99ac67d991f1c090e3814e9b86687 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 23 May 2020 01:01:23 -0700 Subject: [PATCH 0171/1498] Remove long, obsolete comment --- trio/_threads.py | 100 ----------------------------------------------- 1 file changed, 100 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index f441952b55..922e102d30 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -35,106 +35,6 @@ def run_sync(self, fn, *args): return from_thread_run_sync(fn, *args, trio_token=self._trio_token) -################################################################ - -# XX at some point it probably makes sense to implement some sort of thread -# pool? Or at least that's what everyone says. -# -# There are two arguments for thread pools: -# - speed (re-using threads instead of starting new ones) -# - throttling (if you have 1000 tasks, queue them up instead of spawning 1000 -# threads and running out of memory) -# -# Regarding speed, it's not clear how much of an advantage this is. Some -# numbers on my Linux laptop: -# -# Spawning and then joining a thread: -# -# In [25]: %timeit t = threading.Thread(target=lambda: None); t.start(); t.join() -# 10000 loops, best of 3: 44 µs per loop -# -# Using a thread pool: -# -# In [26]: tpp = concurrent.futures.ThreadPoolExecutor() -# In [27]: %timeit tpp.submit(lambda: None).result() -# -# In [28]: %timeit tpp.submit(lambda: None).result() -# 10000 loops, best of 3: 40.8 µs per loop -# -# What's a fast getaddrinfo look like? -# -# # with hot DNS cache: -# In [23]: %timeit socket.getaddrinfo("google.com", "80") -# 10 loops, best of 3: 50.9 ms per loop -# -# In [29]: %timeit socket.getaddrinfo("127.0.0.1", "80") -# 100000 loops, best of 3: 9.73 µs per loop -# -# -# So... maybe we can beat concurrent.futures with a super-efficient thread -# pool or something, but there really is not a lot of headroom here. -# -# Of course other systems might be different... here's CPython 3.6 in a -# Virtualbox VM running Windows 10 on that same Linux laptop: -# -# In [13]: %timeit t = threading.Thread(target=lambda: None); t.start(); t.join() -# 10000 loops, best of 3: 127 µs per loop -# -# In [18]: %timeit tpp.submit(lambda: None).result() -# 10000 loops, best of 3: 31.9 µs per loop -# -# So on Windows there *might* be an advantage? You've gotta be doing a lot of -# connections, with very fast DNS indeed, for that 100 us to matter. But maybe -# someone is. -# -# -# Regarding throttling: this is very much a trade-off. On the one hand, you -# don't want to overwhelm the machine, obviously. On the other hand, queueing -# up work on a central thread-pool creates a central coordination point which -# can potentially create deadlocks and all kinds of fun things. This is very -# context dependent. For getaddrinfo, whatever, they'll make progress and -# complete (we hope), and you want to throttle them to some reasonable -# amount. For calling waitpid() (because just say no to SIGCHLD), then you -# really want one thread-per-waitpid(), because for all you know the user has -# written some ridiculous thing like: -# -# for p in processes: -# await spawn(p.wait) -# # Deadlock here if there are enough processes: -# await some_other_subprocess.wait() -# for p in processes: -# p.terminate() -# -# This goes doubly for the sort of wacky thread usage we see in curio.abide -# (though, I'm not sure if that's actually useful in practice in our context, -# run_in_trio_thread seems like it might be a nicer synchronization primitive -# for most uses than trying to make threading.Lock awaitable). -# -# See also this very relevant discussion: -# -# https://twistedmatrix.com/trac/ticket/5298 -# -# "Interacting with the products at Rackspace which use Twisted, I've seen -# problems caused by thread-pool maximum sizes with some annoying -# regularity. The basic problem is this: if you have a hard limit on the -# number of threads, *it is not possible to write a correct program which may -# require starting a new thread to un-block a blocked pool thread*" - glyph -# -# For now, if we want to throttle getaddrinfo I think the simplest thing is -# for the socket code to have a semaphore for getaddrinfo calls. -# -# Regarding the memory overhead of threads, in theory one should be able to -# reduce this a *lot* for a thread that's just calling getaddrinfo or -# (especially) waitpid. Windows and pthreads both offer the ability to set -# thread stack size on a thread-by-thread basis. Unfortunately as of 3.6 -# CPython doesn't expose this in a useful way (all you can do is set it -# globally for the whole process, so it's - ironically - not thread safe). -# -# (It's also unclear how much stack size actually matters; on a 64-bit Linux -# server with overcommit -- i.e., the most common configuration -- then AFAICT -# really the only real limit is on stack size actually *used*; how much you -# *allocate* should be pretty much irrelevant.) - _limiter_local = RunVar("limiter") # I pulled this number out of the air; it isn't based on anything. Probably we # should make some kind of measurements to pick a good value. From edd2ff2bc95d22df93ec53a0e60be60112263749 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 23 May 2020 01:12:58 -0700 Subject: [PATCH 0172/1498] Fix flakiness in test_subprocess::test_basic gh-1514 introduced a race condition in this test: now if the process runs quickly enough, the returncode might have magically become non-None before we check it. Example failure: https://travis-ci.org/github/python-trio/trio/jobs/690291915 There are other tests that check the 'p.returncode is None' case (e.g. test_auto_update_returncode), so we can just delete this. --- trio/tests/test_subprocess.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 1f489db50c..ae19f8b789 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -51,7 +51,6 @@ async def test_basic(): repr_template = "".format(EXIT_TRUE) async with await open_process(EXIT_TRUE) as proc: assert isinstance(proc, Process) - assert proc.returncode is None assert repr(proc) == repr_template.format( "running with PID {}".format(proc.pid) ) From 853bd3436e00be6faf10921b9363464713ab816d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 23 May 2020 02:38:22 -0700 Subject: [PATCH 0173/1498] Another attempt to fix flakiness in test_subprocess::test_basic In gh-1546 I fixed the flakiness due to the returncode changing when the process exits, but I missed that the repr also changes... --- trio/tests/test_subprocess.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index ae19f8b789..17f1cb941c 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -48,15 +48,12 @@ def got_signal(proc, sig): async def test_basic(): - repr_template = "".format(EXIT_TRUE) async with await open_process(EXIT_TRUE) as proc: - assert isinstance(proc, Process) - assert repr(proc) == repr_template.format( - "running with PID {}".format(proc.pid) - ) + pass + assert isinstance(proc, Process) assert proc._pidfd is None assert proc.returncode == 0 - assert repr(proc) == repr_template.format("exited with status 0") + assert repr(proc) == f"" async with await open_process(EXIT_FALSE) as proc: pass @@ -69,9 +66,11 @@ async def test_basic(): async def test_auto_update_returncode(): p = await open_process(SLEEP(9999)) assert p.returncode is None + assert "running" in repr(p) p.kill() p._proc.wait() assert p.returncode is not None + assert "exited" in repr(p) assert p._pidfd is None assert p.returncode is not None From b2af320e38a42c12900c6485386d2934db9e59c0 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 25 May 2020 10:37:59 +0400 Subject: [PATCH 0174/1498] Ignore two type annotation resolution failures --- docs/source/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2b19282742..d827d21edb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -37,6 +37,9 @@ ("py:mod", "trio.abc"), ("py:class", "math.inf"), ("py:exc", "Anything else"), + # https://github.com/sphinx-doc/sphinx/issues/7722 + ("py:class", "SendType"), + ("py:class", "ReceiveType"), ] autodoc_inherit_docstrings = False default_role = "obj" From b18a333c546c29a06d8fa284b56565ccd875177a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 25 May 2020 06:58:07 +0000 Subject: [PATCH 0175/1498] Bump flake8 from 3.8.1 to 3.8.2 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.8.1 to 3.8.2. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.8.1...3.8.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index cb75bfb8ac..4b132f7c69 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,6 @@ # pip-compile --output-file test-requirements.txt test-requirements.in # appdirs==1.4.4 # via black -appnope==0.1.0 # via ipython astor==0.8.1 # via -r test-requirements.in astroid==2.4.1 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -14,14 +13,12 @@ backcall==0.1.0 # via ipython black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.0 # via cryptography click==7.1.2 # via black -contextvars==2.4 ; python_version < "3.7" # via -r test-requirements.in, sniffio coverage==5.1 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets -flake8==3.8.1 # via -r test-requirements.in +flake8==3.8.2 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme -immutables==0.14 # via -r test-requirements.in, contextvars -importlib-metadata==1.6.0 # via flake8, pluggy, pytest +immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.14.0 # via -r test-requirements.in isort==4.3.21 # via pylint @@ -55,7 +52,6 @@ sortedcontainers==2.1.0 # via -r test-requirements.in toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in -typed-ast==1.4.1 ; python_version < "3.8" and implementation_name == "cpython" # via -r test-requirements.in, astroid, black +typed-ast==1.4.1 # via black wcwidth==0.1.9 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid -zipp==3.1.0 # via importlib-metadata From 9d15b57edfa0a9d1747e3cf2a7f430fc08eac406 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 25 May 2020 11:12:42 +0400 Subject: [PATCH 0176/1498] Tell dependabot that typed-ast is CPython only As far as I can tell, dependabot does not look at test-requirements.in. --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4b132f7c69..88892583a4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -52,6 +52,6 @@ sortedcontainers==2.1.0 # via -r test-requirements.in toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in -typed-ast==1.4.1 # via black +typed_ast==1.4.1 ; implementation_name == "cpython" # via black wcwidth==0.1.9 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid From 41982714c93054d9ff5b9987d468e530f947c2e4 Mon Sep 17 00:00:00 2001 From: Peter Sutton Date: Mon, 25 May 2020 12:20:18 +0100 Subject: [PATCH 0177/1498] Replace os.dup(0) with os.dup(sys.stdin.fileno()) in docs for clarity --- trio/_unix_pipes.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index cb63009249..faf4162551 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -91,13 +91,14 @@ class FdStream(Stream, metaclass=SubclassingDeprecatedIn_v0_15_0): or processes are using file descriptors that are related through `os.dup` or inheritance across `os.fork` to the one that Trio is using, they are unlikely to be prepared to have non-blocking I/O semantics suddenly - thrust upon them. For example, you can use ``FdStream(os.dup(0))`` to - obtain a stream for reading from standard input, but it is only safe to - do so with heavy caveats: your stdin must not be shared by any other - processes and you must not make any calls to synchronous methods of - `sys.stdin` until the stream returned by `FdStream` is closed. See - `issue #174 `__ for a - discussion of the challenges involved in relaxing this restriction. + thrust upon them. For example, you can use + ``FdStream(os.dup(sys.stdin.fileno()))`` to obtain a stream for reading + from standard input, but it is only safe to do so with heavy caveats: your + stdin must not be shared by any other processes and you must not make any + calls to synchronous methods of `sys.stdin` until the stream returned by + `FdStream` is closed. See `issue #174 + `__ for a discussion of the + challenges involved in relaxing this restriction. Args: fd (int): The fd to be wrapped. From 047c07d60a0218f6e2fcbbaa949e16037aaf45b3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 27 May 2020 06:24:55 +0000 Subject: [PATCH 0178/1498] Bump sphinx from 3.0.3 to 3.0.4 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.0.3 to 3.0.4. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.0.3...v3.0.4) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 2f343019b8..f4956a8ea6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -29,7 +29,7 @@ sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.1.0 # via -r docs-requirements.in sphinx-rtd-theme==0.4.3 # via -r docs-requirements.in -sphinx==3.0.3 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx==3.0.4 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 2751323b52cada57cb021a751a047830a72c14c0 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 26 May 2020 23:39:58 -0700 Subject: [PATCH 0179/1498] Swap order of arguments to start_thread_soon --- trio/_core/_thread_cache.py | 10 +++++----- trio/_core/tests/test_thread_cache.py | 12 ++++++------ trio/_threads.py | 7 ++++--- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index 1ecd4bdfec..8716e206a6 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -93,7 +93,7 @@ def __init__(self): self._idle_workers = {} self._cache_lock = Lock() - def start_thread_soon(self, deliver, fn): + def start_thread_soon(self, fn, deliver): try: worker, _ = self._idle_workers.popitem() except KeyError: @@ -105,7 +105,7 @@ def start_thread_soon(self, deliver, fn): THREAD_CACHE = ThreadCache() -def start_thread_soon(deliver, fn): +def start_thread_soon(fn, deliver): """Runs ``deliver(outcome.capture(fn))`` in a worker thread. Generally ``fn`` does some blocking work, and ``deliver`` delivers the @@ -126,11 +126,11 @@ def start_thread_soon(deliver, fn): Args: + fn (sync function): Performs arbitrary blocking work. + deliver (sync function): Takes the `outcome.Outcome` of ``fn``, and delivers it. *Must not block.* - fn (sync function): Performs arbitrary blocking work. - Because worker threads are cached and reused for multiple calls, neither function should mutate thread-level state, like `threading.local` objects – or if they do, they should be careful to revert their changes before @@ -165,4 +165,4 @@ def start_thread_soon(deliver, fn): limit how many threads they're using then it's polite to respect that. """ - THREAD_CACHE.start_thread_soon(deliver, fn) + THREAD_CACHE.start_thread_soon(fn, deliver) diff --git a/trio/_core/tests/test_thread_cache.py b/trio/_core/tests/test_thread_cache.py index 6c6fc3a104..895c1c2e5f 100644 --- a/trio/_core/tests/test_thread_cache.py +++ b/trio/_core/tests/test_thread_cache.py @@ -17,7 +17,7 @@ def fn(): def deliver(outcome): q.put(outcome) - start_thread_soon(deliver, fn) + start_thread_soon(fn, deliver) outcome = q.get() with pytest.raises(RuntimeError, match="hi"): @@ -37,7 +37,7 @@ def test_spawning_new_thread_from_deliver_reuses_starting_thread(): q = Queue() COUNT = 5 for _ in range(COUNT): - start_thread_soon(lambda result: q.put(result), lambda: time.sleep(1)) + start_thread_soon(lambda: time.sleep(1), lambda result: q.put(result)) for _ in range(COUNT): q.get().unwrap() @@ -50,9 +50,9 @@ def deliver(n, _): if n == 0: done.set() else: - start_thread_soon(lambda _: deliver(n - 1, _), lambda: None) + start_thread_soon(lambda: None, lambda _: deliver(n - 1, _)) - start_thread_soon(lambda _: deliver(5, _), lambda: None) + start_thread_soon(lambda: None, lambda _: deliver(5, _)) done.wait() @@ -67,7 +67,7 @@ def test_idle_threads_exit(monkeypatch): monkeypatch.setattr(_thread_cache, "IDLE_TIMEOUT", 0.0001) q = Queue() - start_thread_soon(lambda _: q.put(threading.current_thread()), lambda: None) + start_thread_soon(lambda: None, lambda _: q.put(threading.current_thread())) seen_thread = q.get() # Since the idle timeout is 0, after sleeping for 1 second, the thread # should have exited @@ -116,5 +116,5 @@ def release(self): tc = ThreadCache() done = threading.Event() - tc.start_thread_soon(lambda _: done.set(), lambda: None) + tc.start_thread_soon(lambda: None, lambda _: done.set()) done.wait() diff --git a/trio/_threads.py b/trio/_threads.py index 922e102d30..8867388bd7 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -187,13 +187,14 @@ def deliver_worker_fn_result(result): try: current_trio_token.run_sync_soon(report_back_in_trio_thread_fn, result) except trio.RunFinishedError: - # The entire run finished, so our particular task is certainly - # long gone -- it must have been cancelled and abandoned us. + # The entire run finished, so the task we're trying to contact is + # certainly long gone -- it must have been cancelled and abandoned + # us. pass await limiter.acquire_on_behalf_of(placeholder) try: - start_thread_soon(deliver_worker_fn_result, worker_fn) + start_thread_soon(worker_fn, deliver_worker_fn_result) except: limiter.release_on_behalf_of(placeholder) raise From ca6cc7caffc0aa9d0cfc4b07abbdbbdd320a3a79 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 26 May 2020 23:40:35 -0700 Subject: [PATCH 0180/1498] Remove unused/unnecessary lock --- trio/_core/_thread_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index 8716e206a6..c71a9a37ea 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -91,7 +91,6 @@ def _work(self): class ThreadCache: def __init__(self): self._idle_workers = {} - self._cache_lock = Lock() def start_thread_soon(self, fn, deliver): try: From 11d7db26cd5d20601788a8a2d4dd78b6b56357bb Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 25 May 2020 02:43:49 -0700 Subject: [PATCH 0181/1498] First pass attempt at "guest mode" To allow Trio to efficiently cohabitate with arbitrary other loops. --- docs/source/design.rst | 71 +++++++++++++++ notes-to-self/aio-guest-test.py | 48 ++++++++++ trio/_core/__init__.py | 1 + trio/_core/_io_epoll.py | 30 ++++++- trio/_core/_io_kqueue.py | 22 ++++- trio/_core/_io_windows.py | 19 +++- trio/_core/_run.py | 153 ++++++++++++++++++++++++++------ trio/lowlevel.py | 1 + 8 files changed, 306 insertions(+), 39 deletions(-) create mode 100644 notes-to-self/aio-guest-test.py diff --git a/docs/source/design.rst b/docs/source/design.rst index 6251f22cdb..55eaf6d652 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -534,3 +534,74 @@ there is no provision for "pluggable" backends. The intuition here is that we'd rather focus our energy on making one set of solid, official backends that provide a high-quality experience out-of-the-box on all supported systems. + + +Guest mode +---------- + +XX TODO: document this properly + +the basic idea of pushing ``get_events`` into a thread + +actual core logic is identical whether running in regular mode or +guest mode; alternate between waiting for I/O+timeout vs running trio tasks + +one extra wrinkle is: normally tasks can only become runnable, and +deadline can only change, if trio task is running. In guest mode, +that's no longer true. So reschedule() and deadline changes need to +potentially trigger the scheduler, or at least update the I/O +deadline. Do this in the simplest possible way: force the I/O thread +to return immediately via the normal path. + +subtlety around wait_all_tasks_blocked and 'events' semantics + +diagram:: + + + Normal mode + + Main thread executing trio.run: + + +---------------------------+ + | wait for I/O+timeout | + +---------------------------+ + | run trio tasks | + +---------------------------+ + | wait for I/O+timeout | + +---------------------------+ + | run trio tasks | + +---------------------------+ + | wait for I/O+timeout | + +---------------------------+ + | run trio tasks | + +---------------------------+ + . + . + . + + + Guest mode + + Main thread executing host loop: Trio I/O thread: + + +---------------------------+ + | host loop does its thing | +---------------------------+ + | | | wait for trio I/O+timeout | + +---------------------------+ +---------------------------+ + / + +---------------------------+ <---------------/ + | run trio tasks | + +---------------------------+ ----------------\ + \ + +---------------------------+ v + | host loop does its thing | +---------------------------+ + | | | wait for trio I/O+timeout | + +---------------------------+ +---------------------------+ + / + +---------------------------+ <---------------/ + | run trio tasks | + +---------------------------+ ----------------\ + \ + . + . + . diff --git a/notes-to-self/aio-guest-test.py b/notes-to-self/aio-guest-test.py new file mode 100644 index 0000000000..5e9b398132 --- /dev/null +++ b/notes-to-self/aio-guest-test.py @@ -0,0 +1,48 @@ +import asyncio +import trio + +async def aio_main(): + loop = asyncio.get_running_loop() + + trio_done_fut = asyncio.Future() + def trio_done_callback(main_outcome): + print(f"trio_main finished: {main_outcome!r}") + trio_done_fut.set_result(main_outcome) + + trio.lowlevel.start_guest_run( + trio_main, + run_sync_soon_threadsafe=loop.call_soon_threadsafe, + done_callback=trio_done_callback, + ) + + (await trio_done_fut).unwrap() + + +async def trio_main(): + print("trio_main!") + + to_trio, from_aio = trio.open_memory_channel(float("inf")) + from_trio = asyncio.Queue() + + asyncio.create_task(aio_pingpong(from_trio, to_trio)) + + from_trio.put_nowait(0) + + async for n in from_aio: + print(f"trio got: {n}") + await trio.sleep(1) + from_trio.put_nowait(n + 1) + if n >= 10: + return + +async def aio_pingpong(from_trio, to_trio): + print("aio_pingpong!") + + while True: + n = await from_trio.get() + print(f"aio got: {n}") + await asyncio.sleep(1) + to_trio.send_nowait(n + 1) + + +asyncio.run(aio_main()) diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 136bfe6b98..e22485149d 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -48,6 +48,7 @@ wait_writable, notify_closing, Nursery, + start_guest_run, ) # Has to come after _run to resolve a circular import diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index 71f46c40a7..da331b35e8 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -5,6 +5,7 @@ from .. import _core from ._run import _public from ._io_common import wake_all +from ._wakeup_socketpair import WakeupSocketpair @attr.s(slots=True, eq=False, frozen=True) @@ -184,6 +185,10 @@ class EpollIOManager: _epoll = attr.ib(factory=select.epoll) # {fd: EpollWaiters} _registered = attr.ib(factory=lambda: defaultdict(EpollWaiters)) + _force_wakeup = attr.ib(factory=WakeupSocketpair) + + def __attrs_post_init__(self): + self._epoll.register(self._force_wakeup.wakeup_sock, select.EPOLLIN) def statistics(self): tasks_waiting_read = 0 @@ -200,14 +205,31 @@ def statistics(self): def close(self): self._epoll.close() + self._force_wakeup.close() + + def force_wakeup(self): + self._force_wakeup.wakeup_thread_and_signal_safe() - # Called internally by the task runner: - def handle_io(self, timeout): + # Return value must be False-y IFF the timeout expired, NOT if any I/O + # happened or force_wakeup was called. Otherwise it can be anything; gets + # passed straight through to process_events. + def get_events(self, timeout): # max_events must be > 0 or epoll gets cranky + # accessing self._registered from a thread looks dangerous, but it's + # OK because it doesn't matter if our value is a little bit off. max_events = max(1, len(self._registered)) - events = self._epoll.poll(timeout, max_events) + return self._epoll.poll(timeout, max_events) + + def process_events(self, events): for fd, flags in events: - waiters = self._registered[fd] + try: + waiters = self._registered[fd] + except KeyError: + if fd == self._force_wakeup.wakeup_sock.fileno(): + self._force_wakeup.drain() + continue + else: + raise # EPOLLONESHOT always clears the flags when an event is delivered waiters.current_flags = 0 # Clever hack stolen from selectors.EpollSelector: an event diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index b194e85f53..3eee7d4988 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -7,6 +7,7 @@ from .. import _core from ._run import _public +from ._wakeup_socketpair import WakeupSocketpair @attr.s(slots=True, eq=False, frozen=True) @@ -21,6 +22,13 @@ class KqueueIOManager: _kqueue = attr.ib(factory=select.kqueue) # {(ident, filter): Task or UnboundedQueue} _registered = attr.ib(factory=dict) + _force_wakeup = attr.ib(factory=WakeupSocketpair) + + def __attrs_post_init__(self): + force_wakeup_event = select.kevent( + self._force_wakeup.wakeup_sock, select.KQ_FILTER_READ, select.KQ_EV_ADD + ) + self._kqueue.control([force_wakeup_event], 0) def statistics(self): tasks_waiting = 0 @@ -35,7 +43,7 @@ def statistics(self): def close(self): self._kqueue.close() - def handle_io(self, timeout): + def get_events(self, timeout): # max_events must be > 0 or kqueue gets cranky # and we generally want this to be strictly larger than the actual # number of events we get, so that we can tell that we've gotten @@ -50,9 +58,19 @@ def handle_io(self, timeout): else: timeout = 0 # and loop back to the start + return events + + def process_events(self, events): for event in events: key = (event.ident, event.filter) - receiver = self._registered[key] + try: + receiver = self._registered[key] + except KeyError: + if event.ident == self._force_wakeup.wakeup_sock.fileno(): + self._force_wakeup.drain() + continue + else: + raise if event.flags & select.KQ_EV_ONESHOT: del self._registered[key] if type(receiver) is _core.Task: diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 57056e465e..bd034c27fb 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -171,7 +171,8 @@ class CKeys(enum.IntEnum): AFD_POLL = 0 WAIT_OVERLAPPED = 1 LATE_CANCEL = 2 - USER_DEFINED = 3 # and above + FORCE_WAKEUP = 3 + USER_DEFINED = 4 # and above def _check(success): @@ -388,7 +389,12 @@ def statistics(self): completion_key_monitors=len(self._completion_key_queues), ) - def handle_io(self, timeout): + def force_wakeup(self): + _check( + kernel32.PostQueuedCompletionStatus(self._iocp, 0, CKeys.FORCE_WAKEUP, 0) + ) + + def get_events(self, timeout): received = ffi.new("PULONG") milliseconds = round(1000 * timeout) if timeout > 0 and milliseconds == 0: @@ -402,8 +408,11 @@ def handle_io(self, timeout): except OSError as exc: if exc.winerror != ErrorCodes.WAIT_TIMEOUT: # pragma: no cover raise - return - for i in range(received[0]): + return 0 + return received[0] + + def process_events(received): + for i in range(received): entry = self._events[i] if entry.lpCompletionKey == CKeys.AFD_POLL: lpo = entry.lpOverlapped @@ -465,6 +474,8 @@ def handle_io(self, timeout): # try changing this line to # _core.reschedule(waiter, outcome.Error(exc)) raise exc + elif entry.lpCompletionKey == CKeys.FORCE_WAKEUP: + pass else: # dispatch on lpCompletionKey queue = self._completion_key_queues[entry.lpCompletionKey] diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 4de4e9a2bf..763bcfa70b 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -507,10 +507,22 @@ def _might_change_registered_deadline(self): if old != new: self._registered_deadline = new runner = GLOBAL_RUN_CONTEXT.runner + if runner.is_guest: + try: + (old_next_deadline, _), _ = runner.deadlines.peekitem(0) + except IndexError: + old_next_deadline = None if old != inf: del runner.deadlines[old, id(self)] if new != inf: runner.deadlines[new, id(self)] = self + if runner.is_guest: + try: + (new_next_deadline, _), _ = runner.deadlines.peekitem(0) + except IndexError: + new_next_deadline = None + if old_next_deadline != new_next_deadline: + runner.force_guest_tick_asap() @property def deadline(self): @@ -1118,6 +1130,62 @@ class Runner: entry_queue = attr.ib(factory=EntryQueue) trio_token = attr.ib(default=None) + # Guest mode stuff + is_guest = attr.ib(default=False) + run_sync_soon_threadsafe = attr.ib(default=None) + done_callback = attr.ib(default=None) + unrolled_run_gen = attr.ib(default=None) + unrolled_run_next_send = attr.ib(default=None) + guest_tick_scheduled = attr.ib(default=False) + + def guest_tick(self): + # XX no signal handling support at all currently + assert self.is_guest + try: + timeout = self.unrolled_run_gen.send(self.unrolled_run_next_send) + except StopIteration: + GLOBAL_RUN_CONTEXT.__dict__.clear() + self.close() + # XX if we had KI support, we'd have to do something with it here + self.done_callback(self.main_task_outcome) + return + + self.unrolled_run_next_send = None + + # Optimization: do a zero-timeout check for already-pending I/O from + # the main thread + events = self.io_manager.get_events(0) + if events or timeout <= 0: + self.unrolled_run_next_send = events + self.guest_tick_scheduled = True + self.run_sync_soon_threadsafe(self.guest_tick) + else: + self.guest_tick_scheduled = False + + def get_events(): + return self.io_manager.get_events(timeout) + + def deliver(events_outcome): + def in_main_thread(): + self.unrolled_run_next_send = events_outcome.unwrap() + self.guest_tick_scheduled = True + self.guest_tick() + + self.run_sync_soon_threadsafe(in_main_thread) + + # XX temporary placeholder until #1545 is merged + def start_thread_soon(d, fn): + t = threading.Thread(daemon=True, target=lambda: deliver(capture(fn))) + t.start() + + start_thread_soon(deliver, get_events) + + def force_guest_tick_asap(self): + if self.guest_tick_scheduled: + return + self.guest_tick_scheduled = True + self.io_manager.force_wakeup() + def close(self): self.io_manager.close() self.entry_queue.close() @@ -1222,6 +1290,8 @@ def reschedule(self, task, next_send=_NO_SEND): task._next_send = next_send task._abort_func = None task.custom_sleep_data = None + if not self.runq and self.is_guest: + self.force_guest_tick_asap() self.runq.append(task) if self.instruments: self.instrument("task_scheduled", task) @@ -1579,6 +1649,30 @@ def remove_instrument(self, instrument): ################################################################ +def setup_runner(clock, instruments): + """Create a Runner object and install it as the GLOBAL_RUN_CONTEXT.""" + # It wouldn't be *hard* to support nested calls to run(), but I can't + # think of a single good reason for it, so let's be conservative for + # now: + if hasattr(GLOBAL_RUN_CONTEXT, "runner"): + raise RuntimeError("Attempted to call run() from inside a run()") + + if clock is None: + clock = SystemClock() + instruments = list(instruments) + io_manager = TheIOManager() + system_context = copy_context() + system_context.run(current_async_library_cvar.set, "trio") + runner = Runner( + clock=clock, + instruments=instruments, + io_manager=io_manager, + system_context=system_context, + ) + GLOBAL_RUN_CONTEXT.runner = runner + return runner + + def run( async_fn, *args, @@ -1656,28 +1750,7 @@ def run( __tracebackhide__ = True - # Do error-checking up front, before we enter the TrioInternalError - # try/catch - # - # It wouldn't be *hard* to support nested calls to run(), but I can't - # think of a single good reason for it, so let's be conservative for - # now: - if hasattr(GLOBAL_RUN_CONTEXT, "runner"): - raise RuntimeError("Attempted to call run() from inside a run()") - - if clock is None: - clock = SystemClock() - instruments = list(instruments) - io_manager = TheIOManager() - system_context = copy_context() - system_context.run(current_async_library_cvar.set, "trio") - runner = Runner( - clock=clock, - instruments=instruments, - io_manager=io_manager, - system_context=system_context, - ) - GLOBAL_RUN_CONTEXT.runner = runner + runner = setup_runner(clock, instruments) locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True # KI handling goes outside the core try/except/finally to avoid a window @@ -1688,10 +1761,14 @@ def run( try: with closing(runner): with runner.entry_queue.wakeup.wakeup_on_signals(): - # The main reason this is split off into its own - # function is just to get rid of this extra - # indentation. - run_impl(runner, async_fn, args) + gen = unrolled_run(runner, async_fn, args) + next_send = None + while True: + try: + timeout = gen.send(next_send) + except StopIteration: + break + next_send = runner.io_manager.get_events(timeout) except TrioInternalError: raise except BaseException as exc: @@ -1714,12 +1791,29 @@ def run( raise KeyboardInterrupt +def start_guest_run( + async_fn, + *args, + run_sync_soon_threadsafe, + done_callback, + clock=None, + instruments=(), +): + runner = setup_runner(clock, instruments) + runner.is_guest = True + runner.run_sync_soon_threadsafe = run_sync_soon_threadsafe + runner.done_callback = done_callback + runner.unrolled_run_gen = unrolled_run(runner, async_fn, args) + runner.guest_tick_scheduled = True + run_sync_soon_threadsafe(runner.guest_tick) + + # 24 hours is arbitrary, but it avoids issues like people setting timeouts of # 10**20 and then getting integer overflows in the underlying system calls. _MAX_TIMEOUT = 24 * 60 * 60 -def run_impl(runner, async_fn, args): +def unrolled_run(runner, async_fn, args): __tracebackhide__ = True if runner.instruments: @@ -1751,7 +1845,8 @@ def run_impl(runner, async_fn, args): if runner.instruments: runner.instrument("before_io_wait", timeout) - runner.io_manager.handle_io(timeout) + events = yield timeout + runner.io_manager.process_events(events) if runner.instruments: runner.instrument("after_io_wait", timeout) @@ -1767,7 +1862,7 @@ def run_impl(runner, async_fn, args): else: break - if not runner.runq and idle_primed: + if not runner.runq and not events and idle_primed: while runner.waiting_for_idle: key, task = runner.waiting_for_idle.peekitem(0) if key[:2] == (cushion, tiebreaker): diff --git a/trio/lowlevel.py b/trio/lowlevel.py index 3ce3e741ba..b54b3dba52 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -42,6 +42,7 @@ wait_writable, notify_closing, start_thread_soon, + start_guest_run, ) # Unix-specific symbols From 5b75987eeb3214541483282fc228fcad571bd49b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 25 May 2020 03:07:00 -0700 Subject: [PATCH 0182/1498] fix stupid missing arg in _io_windows.py --- trio/_core/_io_windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index bd034c27fb..18b0876ca5 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -411,7 +411,7 @@ def get_events(self, timeout): return 0 return received[0] - def process_events(received): + def process_events(self, received): for i in range(received): entry = self._events[i] if entry.lpCompletionKey == CKeys.AFD_POLL: From 2b8ae74984bac481d05bb327aa2c8d247fd48b23 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 27 May 2020 07:28:06 +0000 Subject: [PATCH 0183/1498] Remove constraint on typed-ast requirement Now that Dependabot is using 3.8, a requirement only applicable below 3.8 is completely ignored. But black pulls in typed-ast unconditionally, and pip-tools isn't smart enough to propagate environment constraints along dependency edges, so we wind up with typed-ast as an unconditional dependency which makes pypy builds fail. Constraining it in requirements.in based on only the interpreter, not also the Python version, resolves this (tested by running pip-compile on 3.8 directly). --- test-requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.in b/test-requirements.in index 987567acc5..faa8d75935 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -13,7 +13,7 @@ flake8 astor # code generation # https://github.com/python-trio/trio/pull/654#issuecomment-420518745 -typed_ast; python_version < "3.8" and implementation_name == "cpython" +typed_ast; implementation_name == "cpython" # Trio's own dependencies cffi; os_name == "nt" From 90180663680b46438ffc34df1fc99228d440052c Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 27 May 2020 00:33:07 -0700 Subject: [PATCH 0184/1498] Use thread cache to vroom vroom faster --- trio/_core/_run.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 763bcfa70b..c113e124c1 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -35,6 +35,7 @@ PermanentlyDetachCoroutineObject, WaitTaskRescheduled, ) +from ._thread_cache import start_thread_soon from .. import _core from .._deprecate import deprecated from .._util import Final, NoPublicConstructor, coroutine_or_error @@ -1173,12 +1174,7 @@ def in_main_thread(): self.run_sync_soon_threadsafe(in_main_thread) - # XX temporary placeholder until #1545 is merged - def start_thread_soon(d, fn): - t = threading.Thread(daemon=True, target=lambda: deliver(capture(fn))) - t.start() - - start_thread_soon(deliver, get_events) + start_thread_soon(get_events, deliver) def force_guest_tick_asap(self): if self.guest_tick_scheduled: From ea6fad645b6d3abc852d00ed8c6ea257dc1e609f Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 27 May 2020 07:50:34 +0000 Subject: [PATCH 0185/1498] Add Task.eventual_parent_nursery introspection attribute --- newsfragments/1558.feature.rst | 11 +++++++++++ trio/_core/_run.py | 24 ++++++++++++++++++++++-- trio/_core/tests/test_run.py | 15 ++++++++++----- 3 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 newsfragments/1558.feature.rst diff --git a/newsfragments/1558.feature.rst b/newsfragments/1558.feature.rst new file mode 100644 index 0000000000..9b30ecf845 --- /dev/null +++ b/newsfragments/1558.feature.rst @@ -0,0 +1,11 @@ +Tasks spawned with `nursery.start() ` aren't treated as +direct children of their nursery until they call ``task_status.started()``. +This is visible through the task tree introspection attributes such as +`Task.parent_nursery `. Sometimes, though, +you want to know where the task is going to wind up, even if it hasn't finished +initializing yet. To support this, we added a new attribute +`Task.eventual_parent_nursery `. +For a task spawned with :meth:`~trio.Nursery.start` that hasn't yet called +``started()``, this is the nursery that the task was nominally started in, +where it will be running once it finishes starting up. In all other cases, +it hsa the same value as `~trio.lowlevel.Task.parent_nursery`. diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 4de4e9a2bf..997e77c9ca 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1,3 +1,5 @@ +# coding: utf-8 + import functools import itertools import logging @@ -862,7 +864,7 @@ def start_soon(self, async_fn, *args, name=None): If you want to run a function and immediately wait for its result, then you don't need a nursery; just use ``await async_fn(*args)``. If you want to wait for the task to initialize itself before - continuing, see :meth:`start()`. + continuing, see :meth:`start`. It's possible to pass a nursery object into another task, which allows that task to start new child tasks in the first task's @@ -942,7 +944,10 @@ async def async_fn(arg1, arg2, \*, task_status=trio.TASK_STATUS_IGNORED): async with open_nursery() as old_nursery: task_status = _TaskStatus(old_nursery, self) thunk = functools.partial(async_fn, task_status=task_status) - old_nursery.start_soon(thunk, *args, name=name) + task = GLOBAL_RUN_CONTEXT.runner.spawn_impl( + thunk, args, old_nursery, name + ) + task._eventual_parent_nursery = self # Wait for either _TaskStatus.started or an exception to # cancel this nursery: # If we get here, then the child either got reparented or exited @@ -992,6 +997,9 @@ class Task(metaclass=NoPublicConstructor): # For introspection and nursery.start() _child_nurseries = attr.ib(factory=list) + _eventual_parent_nursery = attr.ib( + default=attr.Factory(lambda self: self._parent_nursery, takes_self=True), + ) # these are counts of how many cancel/schedule points this task has # executed, for assert{_no,}_checkpoints @@ -1013,6 +1021,18 @@ def parent_nursery(self): """ return self._parent_nursery + @property + def eventual_parent_nursery(self): + """The nursery this task will be inside after it calls + ``task_status.started()``. + + If this task has already called ``started()``, or if it was not + spawned using `nursery.start() `, then + `eventual_parent_nursery` is the same as `parent_nursery`. + + """ + return self._eventual_parent_nursery + @property def child_nurseries(self): """The nurseries this task contains. diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 4368984370..7039286c4a 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -272,7 +272,7 @@ async def child(): async def test_root_task(): root = _core.current_root_task() - assert root.parent_nursery is None + assert root.parent_nursery is root.eventual_parent_nursery is None def test_out_of_context(): @@ -1588,7 +1588,7 @@ async def test_task_tree_introspection(): tasks = {} nurseries = {} - async def parent(): + async def parent(task_status=_core.TASK_STATUS_IGNORED): tasks["parent"] = _core.current_task() assert tasks["parent"].child_nurseries == [] @@ -1601,7 +1601,7 @@ async def parent(): async with _core.open_nursery() as nursery: nurseries["parent"] = nursery - nursery.start_soon(child1) + await nursery.start(child1) # Upward links survive after tasks/nurseries exit assert nurseries["parent"].parent_task is tasks["parent"] @@ -1624,8 +1624,13 @@ async def child2(): assert nurseries["child1"].child_tasks == frozenset({tasks["child2"]}) assert tasks["child2"].child_nurseries == [] - async def child1(): - tasks["child1"] = _core.current_task() + async def child1(task_status=_core.TASK_STATUS_IGNORED): + me = tasks["child1"] = _core.current_task() + assert me.parent_nursery.parent_task is tasks["parent"] + assert me.parent_nursery is not nurseries["parent"] + assert me.eventual_parent_nursery is nurseries["parent"] + task_status.started() + assert me.parent_nursery is me.eventual_parent_nursery is nurseries["parent"] async with _core.open_nursery() as nursery: nurseries["child1"] = nursery nursery.start_soon(child2) From 176164fa318f62faf0da342f78a0dbc1d6b7bc81 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 27 May 2020 08:04:14 +0000 Subject: [PATCH 0186/1498] Fix nondeterminism and add attribute to docs --- docs/source/reference-lowlevel.rst | 6 ++++-- trio/_core/tests/test_run.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index b196cd0b43..0148995649 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -226,11 +226,11 @@ Windows-specific API .. function:: WaitForSingleObject(handle) :async: - + Async and cancellable variant of `WaitForSingleObject `__. Windows only. - + :arg handle: A Win32 object handle, as a Python integer. :raises OSError: @@ -517,6 +517,8 @@ Task API .. autoattribute:: parent_nursery + .. autoattribute:: eventual_parent_nursery + .. autoattribute:: child_nurseries .. attribute:: custom_sleep_data diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 7039286c4a..d3e7ad36b1 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1631,6 +1631,11 @@ async def child1(task_status=_core.TASK_STATUS_IGNORED): assert me.eventual_parent_nursery is nurseries["parent"] task_status.started() assert me.parent_nursery is me.eventual_parent_nursery is nurseries["parent"] + + # Wait for the start() call to return and close its internal nursery, to + # ensure consistent results in child2: + await _core.wait_all_tasks_blocked() + async with _core.open_nursery() as nursery: nurseries["child1"] = nursery nursery.start_soon(child2) From f67599b5afbd8a8148e63e604d88cdfa9a874851 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 27 May 2020 10:45:47 +0000 Subject: [PATCH 0187/1498] Actually enforce black formatting --- check.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/check.sh b/check.sh index 16394ccea1..c1e81986ae 100755 --- a/check.sh +++ b/check.sh @@ -13,8 +13,10 @@ python ./trio/_tools/gen_exports.py --test \ # see https://forum.bors.tech/t/pre-test-and-pre-merge-hooks/322) # autoflake --recursive --in-place . # pyupgrade --py3-plus $(find . -name "*.py") -black --diff setup.py trio \ - || EXIT_STATUS=$? +if ! black --check setup.py trio; then + EXIT_STATUS=1 + black --diff setup.py trio +fi # Run flake8 without pycodestyle and import-related errors flake8 trio/ \ From 69756d006906a043a7fd290e24e384cd942577e6 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 27 May 2020 09:24:55 -0700 Subject: [PATCH 0188/1498] Update PyPI search link for Framework::Trio It looks like PyPI now have a specific url for trove serach pattern. This filter down the result to "only" 3 pages instead of 500. --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 0eead76b76..da3213330c 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -15,7 +15,7 @@ However, to get much useful work done you will want to use some of the great libraries that support Trio-flavoured concurrency. This list is not complete, but gives a starting point. Another great way to find Trio-compatible libraries is to search on PyPI for the ``Framework :: Trio`` -tag -> `PyPI Search `__ +tag -> `PyPI Search `__ Getting Started From 06e7523b432ff504ba30fcb0a0af13ad27e95625 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 27 May 2020 00:33:22 -0700 Subject: [PATCH 0189/1498] Add a big comment explaining the trickiest change --- trio/_core/_run.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index c113e124c1..d3d229bf3b 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1858,7 +1858,27 @@ def unrolled_run(runner, async_fn, args): else: break - if not runner.runq and not events and idle_primed: + # idle_primed=True means: if the IO wait hit the timeout, and still + # nothing is happening, then we should start waking up + # wait_all_tasks_blocked tasks. But there are some subtleties in + # defining "nothing is happening". + # + # 'not runner.runq' means that no tasks are currently runnable. 'not + # events' means that the last IO wait call hit its full timeout. These + # are very similar, and if idle_primed=True and we're running in + # regular mode then they always go together. But, in *guest* mode, + # they can happen independently, even when idle_primed=True: + # + # - runner.runq=empty and events=True: the host loop adjusted a + # deadline and that forced an IO wakeup before the timeout expired, + # even though no actual tasks were scheduled. + # + # - runner.runq=nonempty and events=False: the IO wait hit its + # timeout, but then some code in the host thread rescheduled a task + # before we got here. + # + # So we need to check both. + if idle_primed and not runner.runq and not events: while runner.waiting_for_idle: key, task = runner.waiting_for_idle.peekitem(0) if key[:2] == (cushion, tiebreaker): From 513d7440d99c4613be689f14ab87e36725800a4e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 12:03:45 -0700 Subject: [PATCH 0190/1498] reduce indentation --- trio/_core/_run.py | 20 ++++++++++---------- trio/_core/_wakeup_socketpair.py | 29 +++++++++++++++++++---------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index d3d229bf3b..77c6c182c3 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1755,16 +1755,15 @@ def run( try: with ki_manager(runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints): try: - with closing(runner): - with runner.entry_queue.wakeup.wakeup_on_signals(): - gen = unrolled_run(runner, async_fn, args) - next_send = None - while True: - try: - timeout = gen.send(next_send) - except StopIteration: - break - next_send = runner.io_manager.get_events(timeout) + runner.entry_queue.wakeup.wakeup_on_signals() + gen = unrolled_run(runner, async_fn, args) + next_send = None + while True: + try: + timeout = gen.send(next_send) + except StopIteration: + break + next_send = runner.io_manager.get_events(timeout) except TrioInternalError: raise except BaseException as exc: @@ -1773,6 +1772,7 @@ def run( ) from exc finally: GLOBAL_RUN_CONTEXT.__dict__.clear() + runner.close() # Inlined copy of runner.main_task_outcome.unwrap() to avoid # cluttering every single Trio traceback with an extra frame. if type(runner.main_task_outcome) is Value: diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 3513cc1ab3..aefdb14ca3 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -38,6 +38,7 @@ def __init__(self): self.write_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except OSError: pass + self.old_wakeup_fd = None def wakeup_thread_and_signal_safe(self): try: @@ -56,21 +57,29 @@ def drain(self): except BlockingIOError: pass - @contextmanager - def wakeup_on_signals(self): - if not is_main_thread(): - yield + def wakeup_on_signals(self, trust_host_loop_to_wake_on_signals=False): + assert self.old_wakeup_fd is None + if not is_main_thread() or trust_host_loop_to_wake_on_signals: return fd = self.write_sock.fileno() if HAVE_WARN_ON_FULL_BUFFER: - old_wakeup_fd = signal.set_wakeup_fd(fd, warn_on_full_buffer=False) + self.old_wakeup_fd = signal.set_wakeup_fd(fd, warn_on_full_buffer=False) else: - old_wakeup_fd = signal.set_wakeup_fd(fd) - try: - yield - finally: - signal.set_wakeup_fd(old_wakeup_fd) + self.old_wakeup_fd = signal.set_wakeup_fd(fd) + if self.old_wakeup_fd != -1: + warnings.warn( + RuntimeWarning( + "It looks like Trio's signal handling code might have " + "collided with another library you're using. If you're " + "running Trio in guest mode, then this might mean you " + "should set trust_host_loop_to_wake_on_signals=True. " + "Otherwise, file a bug on Trio and we'll help you figure " + "out what's going on." + ) + ) def close(self): self.wakeup_sock.close() self.write_sock.close() + if self.old_wakeup_fd is not None: + signal.set_wakeup_fd(self.old_wakeup_fd) From 5631de48d60add125ed969495a2c82bef613a636 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 12:34:09 -0700 Subject: [PATCH 0191/1498] guest mode: Add basic signal handling, and send TrioInternalError to done callback --- trio/_core/_run.py | 418 ++++++++++++++++--------------- trio/_core/_wakeup_socketpair.py | 4 +- 2 files changed, 223 insertions(+), 199 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 77c6c182c3..fbca089ec1 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1136,31 +1136,30 @@ class Runner: run_sync_soon_threadsafe = attr.ib(default=None) done_callback = attr.ib(default=None) unrolled_run_gen = attr.ib(default=None) - unrolled_run_next_send = attr.ib(default=None) + unrolled_run_next_send = attr.ib(factory=lambda: Value(None)) guest_tick_scheduled = attr.ib(default=False) def guest_tick(self): - # XX no signal handling support at all currently + locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True assert self.is_guest try: - timeout = self.unrolled_run_gen.send(self.unrolled_run_next_send) + timeout = self.unrolled_run_next_send.send(self.unrolled_run_gen) except StopIteration: - GLOBAL_RUN_CONTEXT.__dict__.clear() - self.close() # XX if we had KI support, we'd have to do something with it here self.done_callback(self.main_task_outcome) return - - self.unrolled_run_next_send = None - - # Optimization: do a zero-timeout check for already-pending I/O from - # the main thread - events = self.io_manager.get_events(0) - if events or timeout <= 0: - self.unrolled_run_next_send = events + except TrioInternalError as exc: + self.done_callback(Error(exc)) + + # Optimization: try to skip going into the thread if we can avoid it + events_outcome = capture(self.io_manager.get_events, 0) + if timeout <= 0 or type(events_outcome) is Error or events_outcome.value: + # No need to go into the thread + self.unrolled_run_next_send = events_outcome self.guest_tick_scheduled = True self.run_sync_soon_threadsafe(self.guest_tick) else: + # Need to go into the thread and call get_events() there self.guest_tick_scheduled = False def get_events(): @@ -1168,7 +1167,7 @@ def get_events(): def deliver(events_outcome): def in_main_thread(): - self.unrolled_run_next_send = events_outcome.unwrap() + self.unrolled_run_next_send = events_outcome self.guest_tick_scheduled = True self.guest_tick() @@ -1744,35 +1743,24 @@ def run( """ + locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True __tracebackhide__ = True runner = setup_runner(clock, instruments) - locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True - # KI handling goes outside the core try/except/finally to avoid a window - # where KeyboardInterrupt would be allowed and converted into an + # KI handling goes outside unrolled_run to avoid an interval where + # KeyboardInterrupt would be allowed and converted into an # TrioInternalError: try: with ki_manager(runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints): - try: - runner.entry_queue.wakeup.wakeup_on_signals() - gen = unrolled_run(runner, async_fn, args) - next_send = None - while True: - try: - timeout = gen.send(next_send) - except StopIteration: - break - next_send = runner.io_manager.get_events(timeout) - except TrioInternalError: - raise - except BaseException as exc: - raise TrioInternalError( - "internal error in Trio - please file a bug!" - ) from exc - finally: - GLOBAL_RUN_CONTEXT.__dict__.clear() - runner.close() + gen = unrolled_run(runner, async_fn, args) + next_send = None + while True: + try: + timeout = gen.send(next_send) + except StopIteration: + break + next_send = runner.io_manager.get_events(timeout) # Inlined copy of runner.main_task_outcome.unwrap() to avoid # cluttering every single Trio traceback with an extra frame. if type(runner.main_task_outcome) is Value: @@ -1792,6 +1780,7 @@ def start_guest_run( *args, run_sync_soon_threadsafe, done_callback, + trust_host_loop_to_wake_on_signals=False, clock=None, instruments=(), ): @@ -1799,7 +1788,12 @@ def start_guest_run( runner.is_guest = True runner.run_sync_soon_threadsafe = run_sync_soon_threadsafe runner.done_callback = done_callback - runner.unrolled_run_gen = unrolled_run(runner, async_fn, args) + runner.unrolled_run_gen = unrolled_run( + runner, + async_fn, + args, + trust_host_loop_to_wake_on_signals=trust_host_loop_to_wake_on_signals, + ) runner.guest_tick_scheduled = True run_sync_soon_threadsafe(runner.guest_tick) @@ -1809,178 +1803,208 @@ def start_guest_run( _MAX_TIMEOUT = 24 * 60 * 60 -def unrolled_run(runner, async_fn, args): +# Weird quirk: this is written as a generator in order to support "guest +# mode", where our core event loop gets unrolled into a series of callbacks on +# the host loop. If you're doing a regular trio.run then this gets run +# straight through. +def unrolled_run(runner, async_fn, args, trust_host_loop_to_wake_on_signals=False): + locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True __tracebackhide__ = True - if runner.instruments: - runner.instrument("before_run") - runner.clock.start_clock() - runner.init_task = runner.spawn_impl( - runner.init, (async_fn, args), None, "", system_task=True, - ) - - # You know how people talk about "event loops"? This 'while' loop right - # here is our event loop: - while runner.tasks: - if runner.runq: - timeout = 0 - elif runner.deadlines: - deadline, _ = runner.deadlines.keys()[0] - timeout = runner.clock.deadline_to_sleep_time(deadline) - else: - timeout = _MAX_TIMEOUT - timeout = min(max(0, timeout), _MAX_TIMEOUT) - - idle_primed = False - if runner.waiting_for_idle: - cushion, tiebreaker, _ = runner.waiting_for_idle.keys()[0] - if cushion < timeout: - timeout = cushion - idle_primed = True + try: + if not trust_host_loop_to_wake_on_signals: + runner.entry_queue.wakeup.wakeup_on_signals() if runner.instruments: - runner.instrument("before_io_wait", timeout) - - events = yield timeout - runner.io_manager.process_events(events) + runner.instrument("before_run") + runner.clock.start_clock() + runner.init_task = runner.spawn_impl( + runner.init, (async_fn, args), None, "", system_task=True, + ) - if runner.instruments: - runner.instrument("after_io_wait", timeout) - - # Process cancellations due to deadline expiry - now = runner.clock.current_time() - while runner.deadlines: - (deadline, _), cancel_scope = runner.deadlines.peekitem(0) - if deadline <= now: - # This removes the given scope from runner.deadlines: - cancel_scope.cancel() - idle_primed = False + # You know how people talk about "event loops"? This 'while' loop right + # here is our event loop: + while runner.tasks: + if runner.runq: + timeout = 0 + elif runner.deadlines: + deadline, _ = runner.deadlines.keys()[0] + timeout = runner.clock.deadline_to_sleep_time(deadline) else: - break - - # idle_primed=True means: if the IO wait hit the timeout, and still - # nothing is happening, then we should start waking up - # wait_all_tasks_blocked tasks. But there are some subtleties in - # defining "nothing is happening". - # - # 'not runner.runq' means that no tasks are currently runnable. 'not - # events' means that the last IO wait call hit its full timeout. These - # are very similar, and if idle_primed=True and we're running in - # regular mode then they always go together. But, in *guest* mode, - # they can happen independently, even when idle_primed=True: - # - # - runner.runq=empty and events=True: the host loop adjusted a - # deadline and that forced an IO wakeup before the timeout expired, - # even though no actual tasks were scheduled. - # - # - runner.runq=nonempty and events=False: the IO wait hit its - # timeout, but then some code in the host thread rescheduled a task - # before we got here. - # - # So we need to check both. - if idle_primed and not runner.runq and not events: - while runner.waiting_for_idle: - key, task = runner.waiting_for_idle.peekitem(0) - if key[:2] == (cushion, tiebreaker): - del runner.waiting_for_idle[key] - runner.reschedule(task) - else: - break + timeout = _MAX_TIMEOUT + timeout = min(max(0, timeout), _MAX_TIMEOUT) - # Process all runnable tasks, but only the ones that are already - # runnable now. Anything that becomes runnable during this cycle needs - # to wait until the next pass. This avoids various starvation issues - # by ensuring that there's never an unbounded delay between successive - # checks for I/O. - # - # Also, we randomize the order of each batch to avoid assumptions - # about scheduling order sneaking in. In the long run, I suspect we'll - # either (a) use strict FIFO ordering and document that for - # predictability/determinism, or (b) implement a more sophisticated - # scheduler (e.g. some variant of fair queueing), for better behavior - # under load. For now, this is the worst of both worlds - but it keeps - # our options open. (If we do decide to go all in on deterministic - # scheduling, then there are other things that will probably need to - # change too, like the deadlines tie-breaker and the non-deterministic - # ordering of task._notify_queues.) - batch = list(runner.runq) - if _ALLOW_DETERMINISTIC_SCHEDULING: - # We're running under Hypothesis, and pytest-trio has patched this - # in to make the scheduler deterministic and avoid flaky tests. - # It's not worth the (small) performance cost in normal operation, - # since we'll shuffle the list and _r is only seeded for tests. - batch.sort(key=lambda t: t._counter) - runner.runq.clear() - _r.shuffle(batch) - while batch: - task = batch.pop() - GLOBAL_RUN_CONTEXT.task = task + idle_primed = False + if runner.waiting_for_idle: + cushion, tiebreaker, _ = runner.waiting_for_idle.keys()[0] + if cushion < timeout: + timeout = cushion + idle_primed = True if runner.instruments: - runner.instrument("before_task_step", task) + runner.instrument("before_io_wait", timeout) - next_send_fn = task._next_send_fn - next_send = task._next_send - task._next_send_fn = task._next_send = None - final_outcome = None - try: - # We used to unwrap the Outcome object here and send/throw its - # contents in directly, but it turns out that .throw() is - # buggy, at least on CPython 3.6: - # https://bugs.python.org/issue29587 - # https://bugs.python.org/issue29590 - # So now we send in the Outcome object and unwrap it on the - # other side. - msg = task.context.run(next_send_fn, next_send) - except StopIteration as stop_iteration: - final_outcome = Value(stop_iteration.value) - except BaseException as task_exc: - # Store for later, removing uninteresting top frames: 1 frame - # we always remove, because it's this function catching it, - # and then in addition we remove however many more Context.run - # adds. - tb = task_exc.__traceback__.tb_next - for _ in range(CONTEXT_RUN_TB_FRAMES): - tb = tb.tb_next - final_outcome = Error(task_exc.with_traceback(tb)) - - if final_outcome is not None: - # We can't call this directly inside the except: blocks above, - # because then the exceptions end up attaching themselves to - # other exceptions as __context__ in unwanted ways. - runner.task_exited(task, final_outcome) - else: - task._schedule_points += 1 - if msg is CancelShieldedCheckpoint: - runner.reschedule(task) - elif type(msg) is WaitTaskRescheduled: - task._cancel_points += 1 - task._abort_func = msg.abort_func - # KI is "outside" all cancel scopes, so check for it - # before checking for regular cancellation: - if runner.ki_pending and task is runner.main_task: - task._attempt_delivery_of_pending_ki() - task._attempt_delivery_of_any_pending_cancel() - elif type(msg) is PermanentlyDetachCoroutineObject: - # Pretend the task just exited with the given outcome - runner.task_exited(task, msg.final_outcome) - else: - exc = TypeError( - "trio.run received unrecognized yield message {!r}. " - "Are you trying to use a library written for some " - "other framework like asyncio? That won't work " - "without some kind of compatibility shim.".format(msg) - ) - # The foreign library probably doesn't adhere to our - # protocol of unwrapping whatever outcome gets sent in. - # Instead, we'll arrange to throw `exc` in directly, - # which works for at least asyncio and curio. - runner.reschedule(task, exc) - task._next_send_fn = task.coro.throw + # Driver will call io_manager.get_events(timeout) and pass it back + # in throuh the yield + events = yield timeout + runner.io_manager.process_events(events) if runner.instruments: - runner.instrument("after_task_step", task) - del GLOBAL_RUN_CONTEXT.task + runner.instrument("after_io_wait", timeout) + + # Process cancellations due to deadline expiry + now = runner.clock.current_time() + while runner.deadlines: + (deadline, _), cancel_scope = runner.deadlines.peekitem(0) + if deadline <= now: + # This removes the given scope from runner.deadlines: + cancel_scope.cancel() + idle_primed = False + else: + break + + # idle_primed=True means: if the IO wait hit the timeout, and still + # nothing is happening, then we should start waking up + # wait_all_tasks_blocked tasks. But there are some subtleties in + # defining "nothing is happening". + # + # 'not runner.runq' means that no tasks are currently runnable. + # 'not events' means that the last IO wait call hit its full + # timeout. These are very similar, and if idle_primed=True and + # we're running in regular mode then they always go together. But, + # in *guest* mode, they can happen independently, even when + # idle_primed=True: + # + # - runner.runq=empty and events=True: the host loop adjusted a + # deadline and that forced an IO wakeup before the timeout expired, + # even though no actual tasks were scheduled. + # + # - runner.runq=nonempty and events=False: the IO wait hit its + # timeout, but then some code in the host thread rescheduled a task + # before we got here. + # + # So we need to check both. + if idle_primed and not runner.runq and not events: + while runner.waiting_for_idle: + key, task = runner.waiting_for_idle.peekitem(0) + if key[:2] == (cushion, tiebreaker): + del runner.waiting_for_idle[key] + runner.reschedule(task) + else: + break + + # Process all runnable tasks, but only the ones that are already + # runnable now. Anything that becomes runnable during this cycle + # needs to wait until the next pass. This avoids various + # starvation issues by ensuring that there's never an unbounded + # delay between successive checks for I/O. + # + # Also, we randomize the order of each batch to avoid assumptions + # about scheduling order sneaking in. In the long run, I suspect + # we'll either (a) use strict FIFO ordering and document that for + # predictability/determinism, or (b) implement a more + # sophisticated scheduler (e.g. some variant of fair queueing), + # for better behavior under load. For now, this is the worst of + # both worlds - but it keeps our options open. (If we do decide to + # go all in on deterministic scheduling, then there are other + # things that will probably need to change too, like the deadlines + # tie-breaker and the non-deterministic ordering of + # task._notify_queues.) + batch = list(runner.runq) + if _ALLOW_DETERMINISTIC_SCHEDULING: + # We're running under Hypothesis, and pytest-trio has patched + # this in to make the scheduler deterministic and avoid flaky + # tests. It's not worth the (small) performance cost in normal + # operation, since we'll shuffle the list and _r is only + # seeded for tests. + batch.sort(key=lambda t: t._counter) + runner.runq.clear() + _r.shuffle(batch) + while batch: + task = batch.pop() + GLOBAL_RUN_CONTEXT.task = task + + if runner.instruments: + runner.instrument("before_task_step", task) + + next_send_fn = task._next_send_fn + next_send = task._next_send + task._next_send_fn = task._next_send = None + final_outcome = None + try: + # We used to unwrap the Outcome object here and send/throw + # its contents in directly, but it turns out that .throw() + # is buggy, at least on CPython 3.6: + # https://bugs.python.org/issue29587 + # https://bugs.python.org/issue29590 + # So now we send in the Outcome object and unwrap it on the + # other side. + msg = task.context.run(next_send_fn, next_send) + except StopIteration as stop_iteration: + final_outcome = Value(stop_iteration.value) + except BaseException as task_exc: + # Store for later, removing uninteresting top frames: 1 + # frame we always remove, because it's this function + # catching it, and then in addition we remove however many + # more Context.run adds. + tb = task_exc.__traceback__.tb_next + for _ in range(CONTEXT_RUN_TB_FRAMES): + tb = tb.tb_next + final_outcome = Error(task_exc.with_traceback(tb)) + + if final_outcome is not None: + # We can't call this directly inside the except: blocks + # above, because then the exceptions end up attaching + # themselves to other exceptions as __context__ in + # unwanted ways. + runner.task_exited(task, final_outcome) + else: + task._schedule_points += 1 + if msg is CancelShieldedCheckpoint: + runner.reschedule(task) + elif type(msg) is WaitTaskRescheduled: + task._cancel_points += 1 + task._abort_func = msg.abort_func + # KI is "outside" all cancel scopes, so check for it + # before checking for regular cancellation: + if runner.ki_pending and task is runner.main_task: + task._attempt_delivery_of_pending_ki() + task._attempt_delivery_of_any_pending_cancel() + elif type(msg) is PermanentlyDetachCoroutineObject: + # Pretend the task just exited with the given outcome + runner.task_exited(task, msg.final_outcome) + else: + exc = TypeError( + "trio.run received unrecognized yield message {!r}. " + "Are you trying to use a library written for some " + "other framework like asyncio? That won't work " + "without some kind of compatibility shim.".format(msg) + ) + # The foreign library probably doesn't adhere to our + # protocol of unwrapping whatever outcome gets sent in. + # Instead, we'll arrange to throw `exc` in directly, + # which works for at least asyncio and curio. + runner.reschedule(task, exc) + task._next_send_fn = task.coro.throw + + if runner.instruments: + runner.instrument("after_task_step", task) + del GLOBAL_RUN_CONTEXT.task + + except GeneratorExit: + warnings.warn( + RuntimeWarning( + "Trio guest run got abandoned without properly finishing... " + "weird stuff might happen" + ) + ) + except TrioInternalError: + raise + except BaseException as exc: + raise TrioInternalError("internal error in Trio - please file a bug!") from exc + finally: + GLOBAL_RUN_CONTEXT.__dict__.clear() + runner.close() ################################################################ diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index aefdb14ca3..7d0694f9c7 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -57,9 +57,9 @@ def drain(self): except BlockingIOError: pass - def wakeup_on_signals(self, trust_host_loop_to_wake_on_signals=False): + def wakeup_on_signals(self): assert self.old_wakeup_fd is None - if not is_main_thread() or trust_host_loop_to_wake_on_signals: + if not is_main_thread(): return fd = self.write_sock.fileno() if HAVE_WARN_ON_FULL_BUFFER: From b1319582b8e0d1ef9a89ec9b8559f5def61e08ce Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 13:10:52 -0700 Subject: [PATCH 0192/1498] Add a basic test of guest mode --- trio/_core/tests/test_guest_mode.py | 52 +++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 trio/_core/tests/test_guest_mode.py diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py new file mode 100644 index 0000000000..5485ca8be2 --- /dev/null +++ b/trio/_core/tests/test_guest_mode.py @@ -0,0 +1,52 @@ +import pytest +import asyncio + +import trio + + +def test_guest_mode_basic(): + async def aio_main(): + loop = asyncio.get_running_loop() + + trio_done_fut = asyncio.Future() + + def trio_done_callback(main_outcome): + print(f"trio_main finished: {main_outcome!r}") + trio_done_fut.set_result(main_outcome) + + trio.lowlevel.start_guest_run( + trio_main, + run_sync_soon_threadsafe=loop.call_soon_threadsafe, + done_callback=trio_done_callback, + ) + + return (await trio_done_fut).unwrap() + + async def trio_main(): + print("trio_main!") + + to_trio, from_aio = trio.open_memory_channel(float("inf")) + from_trio = asyncio.Queue() + + aio_task = asyncio.create_task(aio_pingpong(from_trio, to_trio)) + + from_trio.put_nowait(0) + + async for n in from_aio: + print(f"trio got: {n}") + from_trio.put_nowait(n + 1) + if n >= 10: + aio_task.cancel() + return "trio-main-done" + + async def aio_pingpong(from_trio, to_trio): + print("aio_pingpong!") + + while True: + n = await from_trio.get() + print(f"aio got: {n}") + to_trio.send_nowait(n + 1) + + loop = asyncio.new_event_loop() + assert loop.run_until_complete(aio_main()) == "trio-main-done" + loop.close() From 91e3554a782c9c831fa9857cb6c4bf75f6659c41 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 13:21:39 -0700 Subject: [PATCH 0193/1498] Try to adapt to py36 asyncio limitations --- trio/_core/tests/test_guest_mode.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 5485ca8be2..8e3cbfbc8c 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -5,9 +5,9 @@ def test_guest_mode_basic(): - async def aio_main(): - loop = asyncio.get_running_loop() + loop = asyncio.new_event_loop() + async def aio_main(): trio_done_fut = asyncio.Future() def trio_done_callback(main_outcome): @@ -47,6 +47,5 @@ async def aio_pingpong(from_trio, to_trio): print(f"aio got: {n}") to_trio.send_nowait(n + 1) - loop = asyncio.new_event_loop() assert loop.run_until_complete(aio_main()) == "trio-main-done" loop.close() From 4340d3a0983658b80c009448a32354cf8a058a4a Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 13:30:04 -0700 Subject: [PATCH 0194/1498] Add missing return --- trio/_core/_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index fbca089ec1..27362c0caf 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1150,6 +1150,7 @@ def guest_tick(self): return except TrioInternalError as exc: self.done_callback(Error(exc)) + return # Optimization: try to skip going into the thread if we can avoid it events_outcome = capture(self.io_manager.get_events, 0) From fd81370369fa190c9966867da2c3ccf1554ccb33 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 13:33:43 -0700 Subject: [PATCH 0195/1498] 3.6 asyncio is super annoying --- trio/_core/tests/test_guest_mode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 8e3cbfbc8c..6925868b9f 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -28,7 +28,7 @@ async def trio_main(): to_trio, from_aio = trio.open_memory_channel(float("inf")) from_trio = asyncio.Queue() - aio_task = asyncio.create_task(aio_pingpong(from_trio, to_trio)) + aio_task = asyncio.ensure_future(aio_pingpong(from_trio, to_trio)) from_trio.put_nowait(0) From 1469d446c80106b7fc3492b483d102a4e81706ba Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 13:52:16 -0700 Subject: [PATCH 0196/1498] missing import --- trio/_core/_wakeup_socketpair.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 7d0694f9c7..392a2c0b2e 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -2,6 +2,7 @@ import sys from contextlib import contextmanager import signal +import warnings from .. import _core from .._util import is_main_thread From cc4d144092fd4c8167d98dcc79a3fbc4d725b47a Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 13:52:25 -0700 Subject: [PATCH 0197/1498] avoid some set_wakeup_fd warnings --- trio/_core/tests/test_guest_mode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 6925868b9f..16d76db819 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -18,6 +18,9 @@ def trio_done_callback(main_outcome): trio_main, run_sync_soon_threadsafe=loop.call_soon_threadsafe, done_callback=trio_done_callback, + # Not all versions of asyncio we test on can actually be trusted, + # but this test doesn't care about signal handling. + trust_host_loop_to_wake_on_signals=True, ) return (await trio_done_fut).unwrap() From b7a21f2a5c367f8c190586c403dfc88b43b48147 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 14:07:36 -0700 Subject: [PATCH 0198/1498] Add missing method that was causing guest mode test to hang on macOS --- trio/_core/_io_kqueue.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index 3eee7d4988..5b3d6f6592 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -43,6 +43,9 @@ def statistics(self): def close(self): self._kqueue.close() + def force_wakeup(self): + self._force_wakeup.wakeup_thread_and_signal_safe() + def get_events(self, timeout): # max_events must be > 0 or kqueue gets cranky # and we generally want this to be strictly larger than the actual From 50322d90958cfec994e8e903c56abc2e0969e24e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 15:48:25 -0700 Subject: [PATCH 0199/1498] Get better debug info from test_guest_mode_basic --- trio/_core/tests/test_guest_mode.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 16d76db819..83bdda6dd0 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -1,5 +1,6 @@ import pytest import asyncio +import traceback import trio @@ -45,10 +46,16 @@ async def trio_main(): async def aio_pingpong(from_trio, to_trio): print("aio_pingpong!") - while True: - n = await from_trio.get() - print(f"aio got: {n}") - to_trio.send_nowait(n + 1) + try: + while True: + n = await from_trio.get() + print(f"aio got: {n}") + to_trio.send_nowait(n + 1) + except asyncio.CancelledError: + raise + except: + traceback.print_exc() + raise assert loop.run_until_complete(aio_main()) == "trio-main-done" loop.close() From 5b7f4ddeba6453e5994c49c2dcc55e2cea0cbf35 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 15:48:40 -0700 Subject: [PATCH 0200/1498] cffi does not implicitly coerce 0 to NULL --- trio/_core/_io_windows.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 18b0876ca5..35fc15e02e 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -391,7 +391,9 @@ def statistics(self): def force_wakeup(self): _check( - kernel32.PostQueuedCompletionStatus(self._iocp, 0, CKeys.FORCE_WAKEUP, 0) + kernel32.PostQueuedCompletionStatus( + self._iocp, 0, CKeys.FORCE_WAKEUP, ffi.NULL + ) ) def get_events(self, timeout): From c5b1c8511137bcdeb765607f66fd688fb1140719 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 15:51:52 -0700 Subject: [PATCH 0201/1498] Temporarily disable testing on python nightly To work around issues like https://github.com/MagicStack/immutables/issues/46 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c6d93471db..01582944db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,9 @@ jobs: - python: 3.7-dev - python: 3.8-dev - python: 3.9-dev - - python: nightly + # Temporarily disabled during the high-churn period of 3.10 + # E.g.: https://github.com/MagicStack/immutables/issues/46 + #- python: nightly script: - ./ci.sh From 3ec60fc36c3493e512a4f48ae4ce19db9e17f8aa Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 17:41:50 -0700 Subject: [PATCH 0202/1498] add some pragma: no covers --- trio/_core/_io_epoll.py | 2 +- trio/_core/_io_kqueue.py | 2 +- trio/_core/tests/test_guest_mode.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index da331b35e8..6ff6467fbb 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -228,7 +228,7 @@ def process_events(self, events): if fd == self._force_wakeup.wakeup_sock.fileno(): self._force_wakeup.drain() continue - else: + else: # pragma: no cover raise # EPOLLONESHOT always clears the flags when an event is delivered waiters.current_flags = 0 diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index 5b3d6f6592..ce85453749 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -72,7 +72,7 @@ def process_events(self, events): if event.ident == self._force_wakeup.wakeup_sock.fileno(): self._force_wakeup.drain() continue - else: + else: # pragma: no cover raise if event.flags & select.KQ_EV_ONESHOT: del self._registered[key] diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 83bdda6dd0..d7d8a6aa8f 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -53,7 +53,7 @@ async def aio_pingpong(from_trio, to_trio): to_trio.send_nowait(n + 1) except asyncio.CancelledError: raise - except: + except: # pragma: no cover traceback.print_exc() raise From 09783ec3d4ec16cac5e21003d4554f35a87aa845 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 17:42:54 -0700 Subject: [PATCH 0203/1498] add missing import --- trio/_core/_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 27362c0caf..327e588430 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -9,6 +9,7 @@ from collections import deque import collections.abc from contextlib import contextmanager, closing +import warnings from contextvars import copy_context from math import inf From f023eeac10686bd785f80ad5a8ff3854761ead16 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 20:36:08 -0700 Subject: [PATCH 0204/1498] TESTS --- trio/_core/_run.py | 145 +++++++------ trio/_core/tests/test_guest_mode.py | 316 +++++++++++++++++++++++++++- 2 files changed, 393 insertions(+), 68 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 327e588430..3972ba434a 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -510,19 +510,13 @@ def _might_change_registered_deadline(self): self._registered_deadline = new runner = GLOBAL_RUN_CONTEXT.runner if runner.is_guest: - try: - (old_next_deadline, _), _ = runner.deadlines.peekitem(0) - except IndexError: - old_next_deadline = None + old_next_deadline = runner.next_deadline() if old != inf: del runner.deadlines[old, id(self)] if new != inf: runner.deadlines[new, id(self)] = self if runner.is_guest: - try: - (new_next_deadline, _), _ = runner.deadlines.peekitem(0) - except IndexError: - new_next_deadline = None + new_next_deadline = runner.next_deadline() if old_next_deadline != new_next_deadline: runner.force_guest_tick_asap() @@ -1106,77 +1100,100 @@ class _RunStatistics: run_sync_soon_queue_size = attr.ib() -@attr.s(eq=False, hash=False) -class Runner: - clock = attr.ib() - instruments = attr.ib() - io_manager = attr.ib() - - # Run-local values, see _local.py - _locals = attr.ib(factory=dict) - - runq = attr.ib(factory=deque) - tasks = attr.ib(factory=set) - - # {(deadline, id(CancelScope)): CancelScope} - # only contains scopes with non-infinite deadlines that are currently - # attached to at least one task - deadlines = attr.ib(factory=SortedDict) - - init_task = attr.ib(default=None) - system_nursery = attr.ib(default=None) - system_context = attr.ib(default=None) - main_task = attr.ib(default=None) - main_task_outcome = attr.ib(default=None) - - entry_queue = attr.ib(factory=EntryQueue) - trio_token = attr.ib(default=None) - - # Guest mode stuff - is_guest = attr.ib(default=False) - run_sync_soon_threadsafe = attr.ib(default=None) - done_callback = attr.ib(default=None) - unrolled_run_gen = attr.ib(default=None) +# This holds all the state that gets trampolined back and forth between +# callbacks when we're running in guest mode. +# +# It has to be a separate object from Runner, and Runner *cannot* have hold +# references to it (directly or indirectly)! +# +# The idea is that we want a chance to detect if our host loop quits and stops +# driving us forward. We detect that by unrolled_run_gen being garbage +# collected, and hitting its 'except GeneratorExit:' block. So this only +# happens if unrolled_run_gen is GCed. +# +# The Runner state is referenced from the global GLOBAL_RUN_CONTEXT. The only +# way it gets *un*referenced is by unrolled_run_gen completing, e.g. by being +# GCed. But if Runner has a direct or indirect reference to it, and the host +# loop has abandoned it, then this will never happen! +# +# So this object can reference Runner, but Runner can't reference it. The only +# references to it are the "in flight" callback chain on the host loop / +# worker thread. +@attr.s(eq=False, hash=False, slots=True) +class GuestState: + runner = attr.ib() + run_sync_soon_threadsafe = attr.ib() + done_callback = attr.ib() + unrolled_run_gen = attr.ib() unrolled_run_next_send = attr.ib(factory=lambda: Value(None)) - guest_tick_scheduled = attr.ib(default=False) def guest_tick(self): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True - assert self.is_guest try: timeout = self.unrolled_run_next_send.send(self.unrolled_run_gen) except StopIteration: # XX if we had KI support, we'd have to do something with it here - self.done_callback(self.main_task_outcome) + self.done_callback(self.runner.main_task_outcome) return except TrioInternalError as exc: self.done_callback(Error(exc)) return # Optimization: try to skip going into the thread if we can avoid it - events_outcome = capture(self.io_manager.get_events, 0) + events_outcome = capture(self.runner.io_manager.get_events, 0) if timeout <= 0 or type(events_outcome) is Error or events_outcome.value: # No need to go into the thread self.unrolled_run_next_send = events_outcome - self.guest_tick_scheduled = True + self.runner.guest_tick_scheduled = True self.run_sync_soon_threadsafe(self.guest_tick) else: # Need to go into the thread and call get_events() there - self.guest_tick_scheduled = False + self.runner.guest_tick_scheduled = False def get_events(): - return self.io_manager.get_events(timeout) + return self.runner.io_manager.get_events(timeout) def deliver(events_outcome): def in_main_thread(): self.unrolled_run_next_send = events_outcome - self.guest_tick_scheduled = True + self.runner.guest_tick_scheduled = True self.guest_tick() self.run_sync_soon_threadsafe(in_main_thread) start_thread_soon(get_events, deliver) + +@attr.s(eq=False, hash=False, slots=True) +class Runner: + clock = attr.ib() + instruments = attr.ib() + io_manager = attr.ib() + + # Run-local values, see _local.py + _locals = attr.ib(factory=dict) + + runq = attr.ib(factory=deque) + tasks = attr.ib(factory=set) + + # {(deadline, id(CancelScope)): CancelScope} + # only contains scopes with non-infinite deadlines that are currently + # attached to at least one task + deadlines = attr.ib(factory=SortedDict) + + init_task = attr.ib(default=None) + system_nursery = attr.ib(default=None) + system_context = attr.ib(default=None) + main_task = attr.ib(default=None) + main_task_outcome = attr.ib(default=None) + + entry_queue = attr.ib(factory=EntryQueue) + trio_token = attr.ib(default=None) + + # Guest mode stuff + is_guest = attr.ib(default=False) + guest_tick_scheduled = attr.ib(default=False) + def force_guest_tick_asap(self): if self.guest_tick_scheduled: return @@ -1189,6 +1206,14 @@ def close(self): if self.instruments: self.instrument("after_run") + def next_deadline(self): + try: + (next_deadline, _), _ = self.deadlines.peekitem(0) + except IndexError: + return inf + else: + return next_deadline + @_public def current_statistics(self): """Returns an object containing run-loop-level debugging information. @@ -1213,11 +1238,7 @@ def current_statistics(self): other attributes vary between backends. """ - if self.deadlines: - next_deadline, _ = self.deadlines.keys()[0] - seconds_to_next_deadline = next_deadline - self.current_time() - else: - seconds_to_next_deadline = float("inf") + seconds_to_next_deadline = self.next_deadline() - self.current_time() return _RunStatistics( tasks_living=len(self.tasks), tasks_runnable=len(self.runq), @@ -1788,16 +1809,20 @@ def start_guest_run( ): runner = setup_runner(clock, instruments) runner.is_guest = True - runner.run_sync_soon_threadsafe = run_sync_soon_threadsafe - runner.done_callback = done_callback - runner.unrolled_run_gen = unrolled_run( + runner.guest_tick_scheduled = True + + guest_state = GuestState( runner, - async_fn, - args, - trust_host_loop_to_wake_on_signals=trust_host_loop_to_wake_on_signals, + run_sync_soon_threadsafe, + done_callback, + unrolled_run( + runner, + async_fn, + args, + trust_host_loop_to_wake_on_signals=trust_host_loop_to_wake_on_signals, + ), ) - runner.guest_tick_scheduled = True - run_sync_soon_threadsafe(runner.guest_tick) + run_sync_soon_threadsafe(guest_state.guest_tick) # 24 hours is arbitrary, but it avoids issues like people setting timeouts of diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index d7d8a6aa8f..ab1fd9530a 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -1,31 +1,323 @@ import pytest import asyncio import traceback +import queue +from functools import partial +from math import inf +import signal +import socket import trio +import trio.testing +from .tutil import gc_collect_harder +# The simplest possible "host" loop. +# Nice features: +# - we can run code "outside" of trio using the schedule function passed to +# our main +# - final result is returned +# - any unhandled exceptions cause an immediate crash +def trivial_guest_run(trio_fn, **start_guest_run_kwargs): + todo = queue.Queue() -def test_guest_mode_basic(): + def run_sync_soon_threadsafe(fn): + todo.put(("run", fn)) + + def done_callback(outcome): + todo.put(("unwrap", outcome)) + + trio.lowlevel.start_guest_run( + trio_fn, + run_sync_soon_threadsafe, + run_sync_soon_threadsafe=run_sync_soon_threadsafe, + done_callback=done_callback, + **start_guest_run_kwargs, + ) + + try: + while True: + op, obj = todo.get() + if op == "run": + obj() + elif op == "unwrap": + return obj.unwrap() + else: # pragma: no cover + assert False + finally: + # Make sure that exceptions raised here don't capture these, so that + # if an exception does cause us to abandon a run then the Trio state + # has a chance to be GC'ed and warn about it. + del todo, run_sync_soon_threadsafe, done_callback + + +def test_guest_trivial(): + async def trio_return(in_host): + await trio.sleep(0) + return "ok" + + assert trivial_guest_run(trio_return) == "ok" + + async def trio_fail(in_host): + raise KeyError("whoopsiedaisy") + + with pytest.raises(KeyError, match="whoopsiedaisy"): + trivial_guest_run(trio_fail) + + +def test_guest_can_do_io(): + async def trio_main(in_host): + record = [] + a, b = trio.socket.socketpair() + with a, b: + async with trio.open_nursery() as nursery: + + async def do_receive(): + record.append(await a.recv(1)) + + nursery.start_soon(do_receive) + await trio.testing.wait_all_tasks_blocked() + + await b.send(b"x") + + assert record == [b"x"] + + trivial_guest_run(trio_main) + + +def test_host_can_directly_wake_trio_task(): + async def trio_main(in_host): + ev = trio.Event() + in_host(ev.set) + await ev.wait() + return "ok" + + assert trivial_guest_run(trio_main) == "ok" + + +def test_host_altering_deadlines_wakes_trio_up(): + def set_deadline(cscope, new_deadline): + cscope.deadline = new_deadline + + async def trio_main(in_host): + with trio.CancelScope() as cscope: + in_host(lambda: set_deadline(cscope, -inf)) + await trio.sleep_forever() + assert cscope.cancelled_caught + + with trio.CancelScope() as cscope: + in_host(lambda: set_deadline(cscope, -inf)) + await trio.sleep(999) + assert cscope.cancelled_caught + + return "ok" + + assert trivial_guest_run(trio_main) == "ok" + + +def test_warn_set_wakeup_fd_overwrite(): + assert signal.set_wakeup_fd(-1) == -1 + + async def trio_main(in_host): + return "ok" + + a, b = socket.socketpair() + with a, b: + a.setblocking(False) + + # Warn if there's already a wakeup fd + signal.set_wakeup_fd(a.fileno()) + try: + with pytest.warns(RuntimeWarning, match="signal handling code.*collided"): + assert trivial_guest_run(trio_main) == "ok" + finally: + assert signal.set_wakeup_fd(-1) == a.fileno() + + signal.set_wakeup_fd(a.fileno()) + try: + with pytest.warns(RuntimeWarning, match="signal handling code.*collided"): + assert ( + trivial_guest_run( + trio_main, trust_host_loop_to_wake_on_signals=False + ) + == "ok" + ) + finally: + assert signal.set_wakeup_fd(-1) == a.fileno() + + # Don't warn if there isn't already a wakeup fd + with pytest.warns(None) as record: + assert trivial_guest_run(trio_main) == "ok" + assert len(record) == 0 + + with pytest.warns(None) as record: + assert ( + trivial_guest_run(trio_main, trust_host_loop_to_wake_on_signals=True) + == "ok" + ) + assert len(record) == 0 + + # If there's already a wakeup fd, but we've been told to trust it, + # then it's left alone and there's no warning + signal.set_wakeup_fd(a.fileno()) + try: + + async def trio_check_wakeup_fd_unaltered(in_host): + fd = signal.set_wakeup_fd(-1) + assert fd == a.fileno() + signal.set_wakeup_fd(fd) + return "ok" + + with pytest.warns(None) as record: + assert ( + trivial_guest_run( + trio_check_wakeup_fd_unaltered, + trust_host_loop_to_wake_on_signals=True, + ) + == "ok" + ) + assert len(record) == 0 + finally: + assert signal.set_wakeup_fd(-1) == a.fileno() + + +def test_host_wakeup_doesnt_trigger_wait_all_tasks_blocked(): + # This is designed to hit the branch in unrolled_run where: + # idle_primed=True + # runner.runq is empty + # events is Truth-y + # ...and confirm that in this case, wait_all_tasks_blocked does not get + # triggered. + def set_deadline(cscope, new_deadline): + print(f"setting deadline {new_deadline}") + cscope.deadline = new_deadline + + async def trio_main(in_host): + async def sit_in_wait_all_tasks_blocked(watb_cscope): + with watb_cscope: + # Overall point of this test is that this + # wait_all_tasks_blocked should *not* return normally, but + # only by cancellation. + await trio.testing.wait_all_tasks_blocked(cushion=9999) + assert False # pragma: no cover + assert watb_cscope.cancelled_caught + + async def get_woken_by_host_deadline(watb_cscope): + with trio.CancelScope() as cscope: + print("scheduling stuff to happen") + # Altering the deadline from the host, to something in the + # future, will cause the run loop to wake up, but then + # discover that there is nothing to do and go back to sleep. + # This should *not* trigger wait_all_tasks_blocked. + # + # So the 'before_io_wait' here will wait until we're blocking + # with the wait_all_tasks_blocked primed, and then schedule a + # deadline change. The critical test is that this should *not* + # wake up 'sit_in_wait_all_tasks_blocked'. + # + # The after we've had a chance to wake up + # 'sit_in_wait_all_tasks_blocked', we want the test to + # actually end. So in after_io_wait we schedule a second host + # call to tear things down. + class InstrumentHelper: + def __init__(self): + self.primed = False + + def before_io_wait(self, timeout): + print(f"before_io_wait({timeout})") + if timeout == 9999: + assert not self.primed + in_host(lambda: set_deadline(cscope, 1e9)) + self.primed = True + + def after_io_wait(self, timeout): + if self.primed: + print("instrument triggered") + in_host(lambda: cscope.cancel()) + trio.lowlevel.remove_instrument(self) + + trio.lowlevel.add_instrument(InstrumentHelper()) + await trio.sleep_forever() + assert cscope.cancelled_caught + watb_cscope.cancel() + + async with trio.open_nursery() as nursery: + watb_cscope = trio.CancelScope() + nursery.start_soon(sit_in_wait_all_tasks_blocked, watb_cscope) + await trio.testing.wait_all_tasks_blocked() + nursery.start_soon(get_woken_by_host_deadline, watb_cscope) + + return "ok" + + assert trivial_guest_run(trio_main) == "ok" + + +def test_guest_warns_if_abandoned(): + # This warning is emitted from the garbage collector. So we have to make + # sure that our abandoned run is garbage. The easiest way to do this is to + # put it into a function, so that we're sure all the local state, + # traceback frames, etc. are garbage once it returns. + def do_abandoned_guest_run(): + async def abandoned_main(in_host): + in_host(lambda: 1 / 0) + while True: + await trio.sleep(0) + + with pytest.raises(ZeroDivisionError): + trivial_guest_run(abandoned_main) + + with pytest.warns(RuntimeWarning, match="Trio guest run got abandoned"): + do_abandoned_guest_run() + gc_collect_harder() + + # If you have problems some day figuring out what's holding onto a + # reference to the unrolled_run generator and making this test fail, + # then this might be useful to help track it down. (It assumes you + # also hack start_guest_run so that it does 'global W; W = + # weakref(unrolled_run_gen)'.) + # + # import gc + # print(trio._core._run.W) + # targets = [trio._core._run.W()] + # for i in range(15): + # new_targets = [] + # for target in targets: + # new_targets += gc.get_referrers(target) + # new_targets.remove(targets) + # print("#####################") + # print(f"depth {i}: {len(new_targets)}") + # print(new_targets) + # targets = new_targets + + with pytest.raises(RuntimeError): + trio.current_time() + + +def aiotrio_run(trio_fn, **start_guest_run_kwargs): loop = asyncio.new_event_loop() async def aio_main(): trio_done_fut = asyncio.Future() def trio_done_callback(main_outcome): - print(f"trio_main finished: {main_outcome!r}") + print(f"trio_fn finished: {main_outcome!r}") trio_done_fut.set_result(main_outcome) trio.lowlevel.start_guest_run( - trio_main, + trio_fn, run_sync_soon_threadsafe=loop.call_soon_threadsafe, done_callback=trio_done_callback, - # Not all versions of asyncio we test on can actually be trusted, - # but this test doesn't care about signal handling. - trust_host_loop_to_wake_on_signals=True, + **start_guest_run_kwargs, ) return (await trio_done_fut).unwrap() + try: + return loop.run_until_complete(aio_main()) + finally: + loop.close() + + +def test_guest_mode_on_asyncio(): async def trio_main(): print("trio_main!") @@ -57,5 +349,13 @@ async def aio_pingpong(from_trio, to_trio): traceback.print_exc() raise - assert loop.run_until_complete(aio_main()) == "trio-main-done" - loop.close() + assert ( + aiotrio_run( + trio_main, + # Not all versions of asyncio we test on can actually be trusted, + # but this test doesn't care about signal handling, and it's + # easier to just avoid the warnings. + trust_host_loop_to_wake_on_signals=True, + ) + == "trio-main-done" + ) From 9c6dff8567be3ab9ca5532f35802399cd3360b71 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 29 May 2020 04:25:48 +0000 Subject: [PATCH 0205/1498] Let eventual_parent_nursery be None for tasks not in the middle of a start() --- newsfragments/1558.feature.rst | 2 +- trio/_core/_run.py | 7 +++---- trio/_core/tests/test_run.py | 8 +++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/newsfragments/1558.feature.rst b/newsfragments/1558.feature.rst index 9b30ecf845..6f8265d232 100644 --- a/newsfragments/1558.feature.rst +++ b/newsfragments/1558.feature.rst @@ -8,4 +8,4 @@ initializing yet. To support this, we added a new attribute For a task spawned with :meth:`~trio.Nursery.start` that hasn't yet called ``started()``, this is the nursery that the task was nominally started in, where it will be running once it finishes starting up. In all other cases, -it hsa the same value as `~trio.lowlevel.Task.parent_nursery`. +it is ``None``. diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 997e77c9ca..d29814cdab 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -661,6 +661,7 @@ def started(self, value=None): self._old_nursery._children = set() for task in tasks: task._parent_nursery = self._new_nursery + task._eventual_parent_nursery = None self._new_nursery._children.add(task) # Move all children of the old nursery's cancel status object @@ -997,9 +998,7 @@ class Task(metaclass=NoPublicConstructor): # For introspection and nursery.start() _child_nurseries = attr.ib(factory=list) - _eventual_parent_nursery = attr.ib( - default=attr.Factory(lambda self: self._parent_nursery, takes_self=True), - ) + _eventual_parent_nursery = attr.ib(default=None) # these are counts of how many cancel/schedule points this task has # executed, for assert{_no,}_checkpoints @@ -1028,7 +1027,7 @@ def eventual_parent_nursery(self): If this task has already called ``started()``, or if it was not spawned using `nursery.start() `, then - `eventual_parent_nursery` is the same as `parent_nursery`. + its `eventual_parent_nursery` is ``None``. """ return self._eventual_parent_nursery diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index d3e7ad36b1..78b46b7adc 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1630,7 +1630,8 @@ async def child1(task_status=_core.TASK_STATUS_IGNORED): assert me.parent_nursery is not nurseries["parent"] assert me.eventual_parent_nursery is nurseries["parent"] task_status.started() - assert me.parent_nursery is me.eventual_parent_nursery is nurseries["parent"] + assert me.parent_nursery is nurseries["parent"] + assert me.eventual_parent_nursery is None # Wait for the start() call to return and close its internal nursery, to # ensure consistent results in child2: @@ -1643,6 +1644,11 @@ async def child1(task_status=_core.TASK_STATUS_IGNORED): async with _core.open_nursery() as nursery: nursery.start_soon(parent) + # There are no pending starts, so no one should have a non-None + # eventual_parent_nursery + for task in tasks.values(): + assert task.eventual_parent_nursery is None + async def test_nursery_closure(): async def child1(nursery): From 1bd024915949459930101295d2575b3906957c33 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 21:37:45 -0700 Subject: [PATCH 0206/1498] Make "assert no RuntimeWarnings" tests more reliable They were failing due to unrelated ResourceWarnings --- trio/_core/tests/test_guest_mode.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index ab1fd9530a..9db7ea2f6a 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -147,14 +147,17 @@ async def trio_main(in_host): # Don't warn if there isn't already a wakeup fd with pytest.warns(None) as record: assert trivial_guest_run(trio_main) == "ok" - assert len(record) == 0 + # Apparently this is how you assert 'there were no RuntimeWarnings' + with pytest.raises(AssertionError): + record.pop(RuntimeWarning) with pytest.warns(None) as record: assert ( trivial_guest_run(trio_main, trust_host_loop_to_wake_on_signals=True) == "ok" ) - assert len(record) == 0 + with pytest.raises(AssertionError): + record.pop(RuntimeWarning) # If there's already a wakeup fd, but we've been told to trust it, # then it's left alone and there's no warning @@ -175,7 +178,8 @@ async def trio_check_wakeup_fd_unaltered(in_host): ) == "ok" ) - assert len(record) == 0 + with pytest.raises(AssertionError): + record.pop(RuntimeWarning) finally: assert signal.set_wakeup_fd(-1) == a.fileno() From ebea1228d5a4b877e47b39ad7d7f111a610b1cdf Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 21:40:50 -0700 Subject: [PATCH 0207/1498] Fix indentation on some asserts --- trio/_core/tests/test_guest_mode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 9db7ea2f6a..41873f5aee 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -102,12 +102,12 @@ async def trio_main(in_host): with trio.CancelScope() as cscope: in_host(lambda: set_deadline(cscope, -inf)) await trio.sleep_forever() - assert cscope.cancelled_caught + assert cscope.cancelled_caught with trio.CancelScope() as cscope: in_host(lambda: set_deadline(cscope, -inf)) await trio.sleep(999) - assert cscope.cancelled_caught + assert cscope.cancelled_caught return "ok" From ef8a85af0745c713627a95f3f7109f4561b6e8ad Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 21:54:20 -0700 Subject: [PATCH 0208/1498] Add test that guest mode properly routes TrioInternalErrors --- trio/_core/tests/test_guest_mode.py | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 41873f5aee..e01200c433 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -6,6 +6,7 @@ from math import inf import signal import socket +import threading import trio import trio.testing @@ -363,3 +364,44 @@ async def aio_pingpong(from_trio, to_trio): ) == "trio-main-done" ) + + +def test_guest_mode_internal_errors(monkeypatch, recwarn): + with monkeypatch.context() as m: + + async def crash_in_run_loop(in_host): + m.setattr("trio._core._run.GLOBAL_RUN_CONTEXT.runner.runq", "HI") + await trio.sleep(1) + + with pytest.raises(trio.TrioInternalError): + trivial_guest_run(crash_in_run_loop) + + with monkeypatch.context() as m: + + async def crash_in_io(in_host): + m.setattr("trio._core._run.TheIOManager.get_events", None) + await trio.sleep(0) + + with pytest.raises(trio.TrioInternalError): + trivial_guest_run(crash_in_io) + + with monkeypatch.context() as m: + + async def crash_in_worker_thread_io(in_host): + t = threading.current_thread() + old_get_events = trio._core._run.TheIOManager.get_events + + def bad_get_events(*args): + if threading.current_thread() is not t: + raise ValueError("oh no!") + else: + return old_get_events(*args) + + m.setattr("trio._core._run.TheIOManager.get_events", bad_get_events) + + await trio.sleep(1) + + with pytest.raises(trio.TrioInternalError): + trivial_guest_run(crash_in_worker_thread_io) + + gc_collect_harder() From cd86726f40a3fa61d61082dd550994ee5b38d593 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 21:55:31 -0700 Subject: [PATCH 0209/1498] Replace some 'type(x) is y' with 'isinstance(x, y)' It turns out the latter is actually faster... --- trio/_core/_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 3972ba434a..d42db22b1f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -821,7 +821,7 @@ def _check_nursery_closed(self): def _child_finished(self, task, outcome): self._children.remove(task) - if type(outcome) is Error: + if isinstance(outcome, Error): self._add_exc(outcome.error) self._check_nursery_closed() @@ -1141,7 +1141,7 @@ def guest_tick(self): # Optimization: try to skip going into the thread if we can avoid it events_outcome = capture(self.runner.io_manager.get_events, 0) - if timeout <= 0 or type(events_outcome) is Error or events_outcome.value: + if timeout <= 0 or isinstance(events_outcome, Error) or events_outcome.value: # No need to go into the thread self.unrolled_run_next_send = events_outcome self.runner.guest_tick_scheduled = True @@ -1786,7 +1786,7 @@ def run( next_send = runner.io_manager.get_events(timeout) # Inlined copy of runner.main_task_outcome.unwrap() to avoid # cluttering every single Trio traceback with an extra frame. - if type(runner.main_task_outcome) is Value: + if isinstance(runner.main_task_outcome, Value): return runner.main_task_outcome.value else: raise runner.main_task_outcome.error From b85176d2f16e27ae3b856c21cddfc03705c27b01 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 28 May 2020 22:01:48 -0700 Subject: [PATCH 0210/1498] Small coverage tweaks --- trio/_core/_io_epoll.py | 14 ++++++-------- trio/_core/tests/test_guest_mode.py | 7 +++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index 6ff6467fbb..665bbe66f7 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -186,9 +186,11 @@ class EpollIOManager: # {fd: EpollWaiters} _registered = attr.ib(factory=lambda: defaultdict(EpollWaiters)) _force_wakeup = attr.ib(factory=WakeupSocketpair) + _force_wakeup_fd = attr.ib(default=None) def __attrs_post_init__(self): self._epoll.register(self._force_wakeup.wakeup_sock, select.EPOLLIN) + self._force_wakeup_fd = self._force_wakeup.wakeup_sock.fileno() def statistics(self): tasks_waiting_read = 0 @@ -222,14 +224,10 @@ def get_events(self, timeout): def process_events(self, events): for fd, flags in events: - try: - waiters = self._registered[fd] - except KeyError: - if fd == self._force_wakeup.wakeup_sock.fileno(): - self._force_wakeup.drain() - continue - else: # pragma: no cover - raise + if fd == self._force_wakeup_fd: + self._force_wakeup.drain() + continue + waiters = self._registered[fd] # EPOLLONESHOT always clears the flags when an event is delivered waiters.current_flags = 0 # Clever hack stolen from selectors.EpollSelector: an event diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index e01200c433..e1caf4c062 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -106,6 +106,9 @@ async def trio_main(in_host): assert cscope.cancelled_caught with trio.CancelScope() as cscope: + # also do a change that doesn't affect the next deadline, just to + # exercise that path + in_host(lambda: set_deadline(cscope, inf)) in_host(lambda: set_deadline(cscope, -inf)) await trio.sleep(999) assert cscope.cancelled_caught @@ -229,13 +232,13 @@ def __init__(self): def before_io_wait(self, timeout): print(f"before_io_wait({timeout})") - if timeout == 9999: + if timeout == 9999: # pragma: no branch assert not self.primed in_host(lambda: set_deadline(cscope, 1e9)) self.primed = True def after_io_wait(self, timeout): - if self.primed: + if self.primed: # pragma: no branch print("instrument triggered") in_host(lambda: cscope.cancel()) trio.lowlevel.remove_instrument(self) From 0376be0f1c82525ec6a9624f19958137e09e8777 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 29 May 2020 05:30:31 -0700 Subject: [PATCH 0211/1498] Fix a bunch of ResourceWarnings due to leaked fds Prompted by this random test failure: https://github.com/python-trio/trio/issues/200#issuecomment-635919605 --- trio/_core/tests/test_multierror.py | 2 +- trio/_highlevel_open_tcp_stream.py | 1 + .../test_highlevel_open_tcp_listeners.py | 15 +- trio/tests/test_highlevel_socket.py | 11 +- trio/tests/test_highlevel_ssl_helpers.py | 90 ++++++----- trio/tests/test_socket.py | 147 ++++++++++-------- trio/tests/test_ssl.py | 51 +++--- 7 files changed, 168 insertions(+), 149 deletions(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index c1444b4c0e..65032bae38 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -600,7 +600,7 @@ def run_script(name, use_ipython=False): print("subprocess PYTHONPATH:", env.get("PYTHONPATH")) if use_ipython: - lines = [script_path.open().read(), "exit()"] + lines = [script_path.read_text(), "exit()"] cmd = [ sys.executable, diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 847fac6a96..99cf8bb1c3 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -222,6 +222,7 @@ async def open_tcp_stream( open_ssl_over_tcp_stream """ + # To keep our public API surface smaller, rule out some cases that # getaddrinfo will accept in some circumstances, but that act weird or # have non-portable behavior or are just plain not useful. diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index ffd708807a..f556a09578 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -115,10 +115,11 @@ async def check_backlog(nominal, required_min, required_max): async def test_open_tcp_listeners_ipv6_v6only(): # Check IPV6_V6ONLY is working properly (ipv6_listener,) = await open_tcp_listeners(0, host="::1") - _, port, *_ = ipv6_listener.socket.getsockname() + async with ipv6_listener: + _, port, *_ = ipv6_listener.socket.getsockname() - with pytest.raises(OSError): - await open_tcp_stream("127.0.0.1", port) + with pytest.raises(OSError): + await open_tcp_stream("127.0.0.1", port) async def test_open_tcp_listeners_rebind(): @@ -127,10 +128,10 @@ async def test_open_tcp_listeners_rebind(): # Plain old rebinding while it's still there should fail, even if we have # SO_REUSEADDR set - probe = stdlib_socket.socket() - probe.setsockopt(stdlib_socket.SOL_SOCKET, stdlib_socket.SO_REUSEADDR, 1) - with pytest.raises(OSError): - probe.bind(sockaddr1) + with stdlib_socket.socket() as probe: + probe.setsockopt(stdlib_socket.SOL_SOCKET, stdlib_socket.SO_REUSEADDR, 1) + with pytest.raises(OSError): + probe.bind(sockaddr1) # Now use the first listener to set up some connections in various states, # and make sure that they don't create any obstacle to rebinding a second diff --git a/trio/tests/test_highlevel_socket.py b/trio/tests/test_highlevel_socket.py index f3570f743e..9dcb834d2c 100644 --- a/trio/tests/test_highlevel_socket.py +++ b/trio/tests/test_highlevel_socket.py @@ -259,8 +259,9 @@ async def accept(self): async def test_socket_stream_works_when_peer_has_already_closed(): sock_a, sock_b = tsocket.socketpair() - await sock_b.send(b"x") - sock_b.close() - stream = SocketStream(sock_a) - assert await stream.receive_some(1) == b"x" - assert await stream.receive_some(1) == b"" + with sock_a, sock_b: + await sock_b.send(b"x") + sock_b.close() + stream = SocketStream(sock_a) + assert await stream.receive_some(1) == b"x" + assert await stream.receive_some(1) == b"" diff --git a/trio/tests/test_highlevel_ssl_helpers.py b/trio/tests/test_highlevel_ssl_helpers.py index 99ee46c9d0..e4afc6d72a 100644 --- a/trio/tests/test_highlevel_ssl_helpers.py +++ b/trio/tests/test_highlevel_ssl_helpers.py @@ -43,53 +43,57 @@ async def getnameinfo(self, *args): # pragma: no cover # This uses serve_ssl_over_tcp, which uses open_ssl_over_tcp_listeners... # noqa is needed because flake8 doesn't understand how pytest fixtures work. -async def test_open_ssl_over_tcp_stream_and_everything_else(client_ctx,): # noqa: F811 +async def test_open_ssl_over_tcp_stream_and_everything_else(client_ctx): # noqa: F811 async with trio.open_nursery() as nursery: (listener,) = await nursery.start( partial(serve_ssl_over_tcp, echo_handler, 0, SERVER_CTX, host="127.0.0.1",) ) - sockaddr = listener.transport_listener.socket.getsockname() - hostname_resolver = FakeHostnameResolver(sockaddr) - trio.socket.set_custom_hostname_resolver(hostname_resolver) - - # We don't have the right trust set up - # (checks that ssl_context=None is doing some validation) - stream = await open_ssl_over_tcp_stream("trio-test-1.example.org", 80) - with pytest.raises(trio.BrokenResourceError): - await stream.do_handshake() - - # We have the trust but not the hostname - # (checks custom ssl_context + hostname checking) - stream = await open_ssl_over_tcp_stream( - "xyzzy.example.org", 80, ssl_context=client_ctx, - ) - with pytest.raises(trio.BrokenResourceError): - await stream.do_handshake() - - # This one should work! - stream = await open_ssl_over_tcp_stream( - "trio-test-1.example.org", 80, ssl_context=client_ctx, - ) - assert isinstance(stream, trio.SSLStream) - assert stream.server_hostname == "trio-test-1.example.org" - await stream.send_all(b"x") - assert await stream.receive_some(1) == b"x" - await stream.aclose() - - # Check https_compatible settings are being passed through - assert not stream._https_compatible - stream = await open_ssl_over_tcp_stream( - "trio-test-1.example.org", - 80, - ssl_context=client_ctx, - https_compatible=True, - # also, smoke test happy_eyeballs_delay - happy_eyeballs_delay=1, - ) - assert stream._https_compatible - - # Stop the echo server - nursery.cancel_scope.cancel() + async with listener: + sockaddr = listener.transport_listener.socket.getsockname() + hostname_resolver = FakeHostnameResolver(sockaddr) + trio.socket.set_custom_hostname_resolver(hostname_resolver) + + # We don't have the right trust set up + # (checks that ssl_context=None is doing some validation) + stream = await open_ssl_over_tcp_stream("trio-test-1.example.org", 80) + async with stream: + with pytest.raises(trio.BrokenResourceError): + await stream.do_handshake() + + # We have the trust but not the hostname + # (checks custom ssl_context + hostname checking) + stream = await open_ssl_over_tcp_stream( + "xyzzy.example.org", 80, ssl_context=client_ctx, + ) + async with stream: + with pytest.raises(trio.BrokenResourceError): + await stream.do_handshake() + + # This one should work! + stream = await open_ssl_over_tcp_stream( + "trio-test-1.example.org", 80, ssl_context=client_ctx, + ) + async with stream: + assert isinstance(stream, trio.SSLStream) + assert stream.server_hostname == "trio-test-1.example.org" + await stream.send_all(b"x") + assert await stream.receive_some(1) == b"x" + + # Check https_compatible settings are being passed through + assert not stream._https_compatible + stream = await open_ssl_over_tcp_stream( + "trio-test-1.example.org", + 80, + ssl_context=client_ctx, + https_compatible=True, + # also, smoke test happy_eyeballs_delay + happy_eyeballs_delay=1, + ) + async with stream: + assert stream._https_compatible + + # Stop the echo server + nursery.cancel_scope.cancel() async def test_open_ssl_over_tcp_listeners(): diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 2a217abdf6..7e4c77ddbe 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -224,9 +224,9 @@ async def test_from_stdlib_socket(): class MySocket(stdlib_socket.socket): pass - mysock = MySocket() - with pytest.raises(TypeError): - tsocket.from_stdlib_socket(mysock) + with MySocket() as mysock: + with pytest.raises(TypeError): + tsocket.from_stdlib_socket(mysock) async def test_from_fd(): @@ -292,12 +292,15 @@ async def test_sniff_sockopts(): # check family / type for correctness: assert tsocket_socket.family == socket.family assert tsocket_socket.type == socket.type + tsocket_socket.detach() # fromfd constructor tsocket_from_fd = tsocket.fromfd(socket.fileno(), AF_INET, SOCK_STREAM) # check family / type for correctness: assert tsocket_from_fd.family == socket.family assert tsocket_from_fd.type == socket.type + tsocket_from_fd.close() + socket.close() @@ -482,73 +485,78 @@ class Addresses: async def test_SocketType_resolve(socket_type, addrs): v6 = socket_type == tsocket.AF_INET6 - # For some reason the stdlib special-cases "" to pass NULL to getaddrinfo - # They also error out on None, but whatever, None is much more consistent, - # so we accept it too. - for null in [None, ""]: - sock = tsocket.socket(family=socket_type) - got = await sock._resolve_local_address((null, 80)) - assert got == (addrs.bind_all, 80, *addrs.extra) - got = await sock._resolve_remote_address((null, 80)) - assert got == (addrs.localhost, 80, *addrs.extra) - - # AI_PASSIVE only affects the wildcard address, so for everything else - # _resolve_local_address and _resolve_remote_address should work the same: - for resolver in ["_resolve_local_address", "_resolve_remote_address"]: - - async def res(*args): - return await getattr(sock, resolver)(*args) - - assert await res((addrs.arbitrary, "http")) == ( - addrs.arbitrary, - 80, - *addrs.extra, - ) - if v6: - assert await res(("1::2", 80, 1)) == ("1::2", 80, 1, 0) - assert await res(("1::2", 80, 1, 2)) == ("1::2", 80, 1, 2) - - # V4 mapped addresses resolved if V6ONLY is False - sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, False) - assert await res(("1.2.3.4", "http")) == ("::ffff:1.2.3.4", 80, 0, 0,) - - # Check the special case, because why not - assert await res(("", 123)) == (addrs.broadcast, 123, *addrs.extra,) - - # But not if it's true (at least on systems where getaddrinfo works - # correctly) - if v6 and not gai_without_v4mapped_is_buggy(): - sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, True) - with pytest.raises(tsocket.gaierror) as excinfo: - await res(("1.2.3.4", 80)) - # Windows, macOS - expected_errnos = {tsocket.EAI_NONAME} - # Linux - if hasattr(tsocket, "EAI_ADDRFAMILY"): - expected_errnos.add(tsocket.EAI_ADDRFAMILY) - assert excinfo.value.errno in expected_errnos - - # A family where we know nothing about the addresses, so should just - # pass them through. This should work on Linux, which is enough to - # smoke test the basic functionality... - try: - netlink_sock = tsocket.socket( - family=tsocket.AF_NETLINK, type=tsocket.SOCK_DGRAM + with tsocket.socket(family=socket_type) as sock: + # For some reason the stdlib special-cases "" to pass NULL to + # getaddrinfo They also error out on None, but whatever, None is much + # more consistent, so we accept it too. + for null in [None, ""]: + got = await sock._resolve_local_address((null, 80)) + assert got == (addrs.bind_all, 80, *addrs.extra) + got = await sock._resolve_remote_address((null, 80)) + assert got == (addrs.localhost, 80, *addrs.extra) + + # AI_PASSIVE only affects the wildcard address, so for everything else + # _resolve_local_address and _resolve_remote_address should work the same: + for resolver in ["_resolve_local_address", "_resolve_remote_address"]: + + async def res(*args): + return await getattr(sock, resolver)(*args) + + assert await res((addrs.arbitrary, "http")) == ( + addrs.arbitrary, + 80, + *addrs.extra, ) - except (AttributeError, OSError): - pass - else: - assert await getattr(netlink_sock, resolver)("asdf") == "asdf" - - with pytest.raises(ValueError): - await res("1.2.3.4") - with pytest.raises(ValueError): - await res(("1.2.3.4",)) - with pytest.raises(ValueError): if v6: - await res(("1.2.3.4", 80, 0, 0, 0)) + assert await res(("1::2", 80, 1)) == ("1::2", 80, 1, 0) + assert await res(("1::2", 80, 1, 2)) == ("1::2", 80, 1, 2) + + # V4 mapped addresses resolved if V6ONLY is False + sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, False) + assert await res(("1.2.3.4", "http")) == ("::ffff:1.2.3.4", 80, 0, 0,) + + # Check the special case, because why not + assert await res(("", 123)) == ( + addrs.broadcast, + 123, + *addrs.extra, + ) + + # But not if it's true (at least on systems where getaddrinfo works + # correctly) + if v6 and not gai_without_v4mapped_is_buggy(): + sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, True) + with pytest.raises(tsocket.gaierror) as excinfo: + await res(("1.2.3.4", 80)) + # Windows, macOS + expected_errnos = {tsocket.EAI_NONAME} + # Linux + if hasattr(tsocket, "EAI_ADDRFAMILY"): + expected_errnos.add(tsocket.EAI_ADDRFAMILY) + assert excinfo.value.errno in expected_errnos + + # A family where we know nothing about the addresses, so should just + # pass them through. This should work on Linux, which is enough to + # smoke test the basic functionality... + try: + netlink_sock = tsocket.socket( + family=tsocket.AF_NETLINK, type=tsocket.SOCK_DGRAM + ) + except (AttributeError, OSError): + pass else: - await res(("1.2.3.4", 80, 0, 0)) + assert await getattr(netlink_sock, resolver)("asdf") == "asdf" + netlink_sock.close() + + with pytest.raises(ValueError): + await res("1.2.3.4") + with pytest.raises(ValueError): + await res(("1.2.3.4",)) + with pytest.raises(ValueError): + if v6: + await res(("1.2.3.4", 80, 0, 0, 0)) + else: + await res(("1.2.3.4", 80, 0, 0)) async def test_SocketType_unresolved_names(): @@ -923,8 +931,9 @@ async def check_AF_UNIX(path): with tsocket.socket(family=tsocket.AF_UNIX) as csock: await csock.connect(path) ssock, _ = await lsock.accept() - await csock.send(b"x") - assert await ssock.recv(1) == b"x" + with ssock: + await csock.send(b"x") + assert await ssock.recv(1) == b"x" # Can't use tmpdir fixture, because we can exceed the maximum AF_UNIX path # length on macOS. diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index dee8e325f7..9b2279c958 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -89,24 +89,25 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): wrapped = SERVER_CTX.wrap_socket( sock, server_side=True, suppress_ragged_eofs=False ) - wrapped.do_handshake() - while True: - data = wrapped.recv(4096) - if not data: - # other side has initiated a graceful shutdown; we try to - # respond in kind but it's legal for them to have already gone - # away. - exceptions = (BrokenPipeError, ssl.SSLZeroReturnError) - # Under unclear conditions, CPython sometimes raises - # SSLWantWriteError here. This is a bug (bpo-32219), but it's - # not our bug, so ignore it. - exceptions += (ssl.SSLWantWriteError,) - try: - wrapped.unwrap() - except exceptions: - pass - return - wrapped.sendall(data) + with wrapped: + wrapped.do_handshake() + while True: + data = wrapped.recv(4096) + if not data: + # other side has initiated a graceful shutdown; we try to + # respond in kind but it's legal for them to have already + # gone away. + exceptions = (BrokenPipeError, ssl.SSLZeroReturnError) + # Under unclear conditions, CPython sometimes raises + # SSLWantWriteError here. This is a bug (bpo-32219), but + # it's not our bug, so ignore it. + exceptions += (ssl.SSLWantWriteError,) + try: + wrapped.unwrap() + except exceptions: + pass + return + wrapped.sendall(data) # This is an obscure workaround for an openssl bug. In server mode, in # some versions, openssl sends some extra data at the end of do_handshake # that it shouldn't send. Normally this is harmless, but, if the other @@ -132,6 +133,8 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): else: if expect_fail: # pragma: no cover raise RuntimeError("failed to fail?") + finally: + sock.close() # Fixture that gives a raw socket connected to a trio-test-1 echo server @@ -428,13 +431,13 @@ async def test_ssl_server_basics(client_ctx): assert server_transport.server_side def client(): - client_sock = client_ctx.wrap_socket( + with client_ctx.wrap_socket( a, server_hostname="trio-test-1.example.org" - ) - client_sock.sendall(b"x") - assert client_sock.recv(1) == b"y" - client_sock.sendall(b"z") - client_sock.unwrap() + ) as client_sock: + client_sock.sendall(b"x") + assert client_sock.recv(1) == b"y" + client_sock.sendall(b"z") + client_sock.unwrap() t = threading.Thread(target=client) t.start() From 2e6463471cff7c1429a946ee1be465f695f6c90c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 29 May 2020 12:54:39 +0000 Subject: [PATCH 0212/1498] Bump pytest-cov from 2.8.1 to 2.9.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.8.1 to 2.9.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.8.1...v2.9.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 88892583a4..0d85e6287f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,7 +43,7 @@ pygments==2.6.1 # via ipython pylint==2.5.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.8.1 # via -r test-requirements.in +pytest-cov==2.9.0 # via -r test-requirements.in pytest==5.4.2 # via -r test-requirements.in, pytest-cov regex==2020.5.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets @@ -52,6 +52,6 @@ sortedcontainers==2.1.0 # via -r test-requirements.in toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in -typed_ast==1.4.1 ; implementation_name == "cpython" # via black +typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black wcwidth==0.1.9 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid From 35595006fce87e98f3c8adc3c978f8b628c0f371 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 29 May 2020 06:02:54 -0700 Subject: [PATCH 0213/1498] Fix test so it exercises what it's supposed to be exercising --- trio/_core/tests/test_guest_mode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index e1caf4c062..f055ca1f0f 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -108,7 +108,7 @@ async def trio_main(in_host): with trio.CancelScope() as cscope: # also do a change that doesn't affect the next deadline, just to # exercise that path - in_host(lambda: set_deadline(cscope, inf)) + in_host(lambda: set_deadline(cscope, 1e6)) in_host(lambda: set_deadline(cscope, -inf)) await trio.sleep(999) assert cscope.cancelled_caught From d4343dc95fd72877cd2fd810fc685e171b1384ae Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 30 May 2020 02:44:02 -0700 Subject: [PATCH 0214/1498] Add run_sync_soon_not_threadsafe= kwarg on start_guest_run This is handy for host loops where the non-threadsafe version is significantly faster (e.g. tkinter). --- trio/_core/_run.py | 18 +++++++++----- trio/_core/tests/test_guest_mode.py | 38 +++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index d42db22b1f..ee1c1a454d 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1123,6 +1123,7 @@ class _RunStatistics: class GuestState: runner = attr.ib() run_sync_soon_threadsafe = attr.ib() + run_sync_soon_not_threadsafe = attr.ib() done_callback = attr.ib() unrolled_run_gen = attr.ib() unrolled_run_next_send = attr.ib(factory=lambda: Value(None)) @@ -1145,7 +1146,7 @@ def guest_tick(self): # No need to go into the thread self.unrolled_run_next_send = events_outcome self.runner.guest_tick_scheduled = True - self.run_sync_soon_threadsafe(self.guest_tick) + self.run_sync_soon_not_threadsafe(self.guest_tick) else: # Need to go into the thread and call get_events() there self.runner.guest_tick_scheduled = False @@ -1803,6 +1804,7 @@ def start_guest_run( *args, run_sync_soon_threadsafe, done_callback, + run_sync_soon_not_threadsafe=None, trust_host_loop_to_wake_on_signals=False, clock=None, instruments=(), @@ -1811,18 +1813,22 @@ def start_guest_run( runner.is_guest = True runner.guest_tick_scheduled = True + if run_sync_soon_not_threadsafe is None: + run_sync_soon_not_threadsafe = run_sync_soon_threadsafe + guest_state = GuestState( - runner, - run_sync_soon_threadsafe, - done_callback, - unrolled_run( + runner=runner, + run_sync_soon_threadsafe=run_sync_soon_threadsafe, + run_sync_soon_not_threadsafe=run_sync_soon_not_threadsafe, + done_callback=done_callback, + unrolled_run_gen=unrolled_run( runner, async_fn, args, trust_host_loop_to_wake_on_signals=trust_host_loop_to_wake_on_signals, ), ) - run_sync_soon_threadsafe(guest_state.guest_tick) + run_sync_soon_not_threadsafe(guest_state.guest_tick) # 24 hours is arbitrary, but it avoids issues like people setting timeouts of diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index f055ca1f0f..c11c2dc70b 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -21,7 +21,22 @@ def trivial_guest_run(trio_fn, **start_guest_run_kwargs): todo = queue.Queue() + host_thread = threading.current_thread() + def run_sync_soon_threadsafe(fn): + if host_thread is threading.current_thread(): # pragma: no cover + crash = partial( + pytest.fail, "run_sync_soon_threadsafe called from host thread" + ) + todo.put(("run", crash)) + todo.put(("run", fn)) + + def run_sync_soon_not_threadsafe(fn): + if host_thread is not threading.current_thread(): # pragma: no cover + crash = partial( + pytest.fail, "run_sync_soon_not_threadsafe called from worker thread" + ) + todo.put(("run", crash)) todo.put(("run", fn)) def done_callback(outcome): @@ -29,8 +44,9 @@ def done_callback(outcome): trio.lowlevel.start_guest_run( trio_fn, - run_sync_soon_threadsafe, + run_sync_soon_not_threadsafe, run_sync_soon_threadsafe=run_sync_soon_threadsafe, + run_sync_soon_not_threadsafe=run_sync_soon_not_threadsafe, done_callback=done_callback, **start_guest_run_kwargs, ) @@ -300,7 +316,7 @@ async def abandoned_main(in_host): trio.current_time() -def aiotrio_run(trio_fn, **start_guest_run_kwargs): +def aiotrio_run(trio_fn, *, pass_not_threadsafe=True, **start_guest_run_kwargs): loop = asyncio.new_event_loop() async def aio_main(): @@ -310,6 +326,9 @@ def trio_done_callback(main_outcome): print(f"trio_fn finished: {main_outcome!r}") trio_done_fut.set_result(main_outcome) + if pass_not_threadsafe: + start_guest_run_kwargs["run_sync_soon_not_threadsafe"] = loop.call_soon + trio.lowlevel.start_guest_run( trio_fn, run_sync_soon_threadsafe=loop.call_soon_threadsafe, @@ -334,6 +353,10 @@ async def trio_main(): aio_task = asyncio.ensure_future(aio_pingpong(from_trio, to_trio)) + # Make sure we have at least one tick where we don't need to go into + # the thread + await trio.sleep(0) + from_trio.put_nowait(0) async for n in from_aio: @@ -368,6 +391,17 @@ async def aio_pingpong(from_trio, to_trio): == "trio-main-done" ) + assert ( + aiotrio_run( + trio_main, + # Also check that passing only call_soon_threadsafe works, via the + # fallback path where we use it for everything. + pass_not_threadsafe=False, + trust_host_loop_to_wake_on_signals=True, + ) + == "trio-main-done" + ) + def test_guest_mode_internal_errors(monkeypatch, recwarn): with monkeypatch.context() as m: From a827cfb308cfc13e8cf5bcbb3074569cd6a903a0 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 31 May 2020 19:40:14 -0700 Subject: [PATCH 0215/1498] First pass a comprehensive docs --- docs/source/design.rst | 3 + docs/source/reference-lowlevel.rst | 381 +++++++++++++++++++++++++++- trio/_core/_run.py | 57 ++++- trio/_core/_wakeup_socketpair.py | 2 +- trio/_core/tests/test_guest_mode.py | 10 +- 5 files changed, 440 insertions(+), 13 deletions(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 55eaf6d652..184e60c110 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -539,6 +539,9 @@ supported systems. Guest mode ---------- +One of Trio's more unusual features is that it supports being run in +"guest mode" on top of some other event loop (the "host"). + XX TODO: document this properly the basic idea of pushing ``get_events`` into a thread diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index b196cd0b43..371f47c365 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -467,7 +467,7 @@ this does serve to illustrate the basic structure of the Task API --------- +======== .. autofunction:: current_root_task() @@ -528,10 +528,387 @@ Task API :func:`wait_task_rescheduled` for details.) +.. _guest-mode: + +Using "guest mode" to run Trio on top of other event loops +========================================================== + +What is "guest mode"? +--------------------- + +An event loop acts as a central coordinator to manage all the IO +happening in your program. Normally, that means that your application +has to pick one event loop, and use it for everything. But what if you +like Trio, but also need to use a framework like `Qt +`__ or `PyGame +`__ that has its own event loop? Then you +need some way to run both event loops at once. + +It is possible to combine event loops, but the standard approaches all +have significant downsides: + +- **Polling:** this is where you use a `busy-loop + `__ to manually check + for IO on both event loops many times per second. This adds latency, + and wastes CPU time and electricity. + +- **Pluggable IO backends:** this is where you reimplement one of the + event loop APIs on top of the other, so you effectively end up with + just one event loop. This requires a significant amount of work for + each pair of event loops you want to integrate, and different + backends inevitably end up with inconsistent behavior, forcing users + to program against the least-common-denominator. And if the two + event loops expose different feature sets, it may not even be + possible to implement one in terms of the other. + +- **Running the two event loops in separate threads:** This works, but + most event loop APIs aren't thread-safe, so in this approach you + need to keep careful track of which code runs on which event loop, + and remember to use explicit inter-thread messaging whenever you + interact with the other loop – or else risk obscure race conditions + and data corruption. + +That's why Trio offers a fourth option: **guest mode**. Guest mode +lets you execute `trio.run` on top of some other "host" event loop, +like Qt. Its advantages are: + +- Efficiency: guest mode is event-driven instead of using a busy-loop, + so it has low latency and doesn't waste electricity. + +- No need to think about threads: your Trio code runs in the same + thread as the host event loop, so you can freely call Trio APIs from + the host, and call host APIs from Trio. For example, if you're + making a GUI app with Qt as the host loop, then making a `cancel + button `__ and connecting + it to a `trio.CancelScope` is as easy as writing:: + + # Trio code can create Qt objects without any special ceremony... + my_cancel_button = QPushButton("Cancel") + # ...and Qt can call back to Trio just as easily + my_cancel_button.clicked.connect(my_cancel_scope.cancel) + +- Consistent behavior: guest mode uses the code as regular Trio: the + same scheduler, same IO code, same everything. So you get the same + features and robustness that you're used to. + +- Simple integration and broad compatibility: pretty much every event + loop offers some threadsafe "schedule a callback" operation, and + that's all you need to use it as a host loop. + + +Really? How is that possible? +----------------------------- + +.. note:: + + You can use guest mode without reading this section. It's included + for those who enjoy understanding how things work. + +All event loops have the same basic structure. They loop through two +operations, over and over: + +1. Wait for the operating system to notify them that something + interesting has happened, like data arriving on a socket or a + timeout passing. They do this by invoking a platform-specific + ``sleep_until_something_happens()`` system call – ``select``, + ``epoll``, ``kqueue``, ``GetQueuedCompletionEvents``, etc. + +2. Run all the user tasks that care about whatever happened, then go + back to step 1. + +The problem here is step 1. Two different event loops on the same +thread can take turns running user tasks in step 2, but when they're +idle and nothing is happening, they can't both invoke their own +``sleep_until_something_happens()`` function at the same time. + +The "polling" and "pluggable backend" strategies solve this by hacking +the loops so both step 1s can run at the same time in the same thread. +Keeping everything in one thread is great for step 2, but the step 1 +hacks create problems. + +The "separate threads" strategy solves this by moving both steps into +separate threads. This makes step 1 work, but the downside is that now +the user tasks in step 2 are running separate threads as well, so +users are forced to deal with inter-thread coordination. + +The idea behind guest mode is to combine the best parts of each +approach: we move Trio's step 1 into a separate worker thread, while +keeping Trio's step 2 in the main host thread. This way, when the +application is idle, both event loops do their +``sleep_until_something_happens()`` at the same time in their own +threads. But when the app wakes up and your code is actually running, +it all happens in a single thread. The threading trickiness is all +handled transparently inside Trio. + +Concretely, we unroll Trio's internal event loop into a chain of +callbacks, and as each callback finishes, it schedules the next +callback onto the host loop or a worker thread as appropriate. So the +only thing the host loop has to provide is a way to schedule a +callback onto the main thread from a worker thread. + +Coordinating between Trio and the host loop does add some overhead. +The main cost is switching in and out of the background thread, since +this requires cross-thread messaging. This is cheap (on the order of a +few microseconds, assuming your host loop is implemented efficiently), +but it's not free. + +But, there's a nice optimization we can make: we only *need* the +thread when our ``sleep_until_something_happens()`` call actually +sleeps, that is, when the Trio part of your program is idle and has +nothing to do. So before we switch into the worker thread, we +double-check whether we're idle, and if not, then we skip the worker +thread and jump directly to step 2. This means that your app only pays +the extra thread-switching penalty at moments when it would otherwise +be sleeping, so it should have minimal effect on your app's overall +performance. + +The total overhead will depend on your host loop, your platform, your +application, etc. But we expect that in most cases, apps running in +guest mode should only be 5-10% slower than the same code using +`trio.run`. If you find that's not true for your app, then please let +us know and we'll see if we can fix it! + + +.. _guest-run-implementation: + +Implementing guest mode for your favorite event loop +---------------------------------------------------- + +Let's walk through what you need to do to integrate Trio's guest mode +with your favorite event loop. Treat this section like a checklist. + +**Getting started:** The first step is to get something basic working. +Here's a minimal example of running Trio on top of asyncio, that you +can use as a model:: + + import asyncio, trio + + # A tiny Trio program + async def trio_main(): + for i in range(5): + print(f"Hello from Trio!") + # This is inside Trio, so we have to use Trio APIs + await trio.sleep(1) + return "trio done!" + + # The code to run it as a guest inside asyncio + async def asyncio_main(): + asyncio_loop = asyncio.get_running_loop() + + def run_sync_soon_threadsafe(fn): + asyncio_loop.call_soon_threadsafe(fn) + + def done_callback(trio_main_outcome): + print(f"Trio program ended with: {trio_main_outcome}") + + # This is where the magic happens: + trio.lowlevel.start_guest_run( + trio_main, + run_sync_soon_threadsafe=run_sync_soon_threadsafe, + done_callback=done_callback, + ) + + # Let the host loop run for a while to give trio_main time to + # finish. (WARNING: This is a hack. See below for better + # approaches.) + # + # This function is in asyncio, so we have to use asyncio APIs. + await asyncio.sleep(10) + + asyncio.run(asyncio_main()) + +You can see we're using asyncio-specific APIs to start up a loop, and +then we call `trio.lowlevel.start_guest_run`. This function is very +similar to `trio.run`, and takes all the same arguments. But it has +two differences: + +First, instead of blocking until ``trio_main`` has finished, it +schedules ``trio_main`` to start running on top of the host loop, and +then returns immediately. So ``trio_main`` is running in the +background – that's why we have to sleep and give it time to finish. + +And second, it requires two extra keyword arguments: +``run_sync_soon_threadsafe``, and ``done_callback``. + +For ``run_sync_soon_threadsafe``, we need a function that takes a +synchronous callback, and schedules it to run on your host loop. And +this function needs to be "threadsafe" in the sense that you can +safely call it from any thread. So you need to figure out how to write +a function that does that using your host loop's API. For asyncio, +this is easy because `~asyncio.loop.call_soon_threadsafe` does exactly +what we need; for your loop, it might be more or less complicated. + +For ``done_callback``, you pass in a function that Trio will +automatically invoke when the Trio run finishes, so you know it's done +and what happened. For this basic starting version, we just print the +result; in the next section we'll discuss better alternatives. + +At this stage you should be able to run a simple Trio program inside +your host loop. Now we'll turn that prototype into something solid. + + +**Loop lifetimes:** One of the trickiest things in most event loops is +shutting down correctly. And having two event loops makes this even +harder! + +If you can, we recommend following this pattern: + +- Start up your host loop +- Immediately call `start_guest_run` to start Trio +- When Trio finishes and your ``done_callback`` is invoked, shut down + the host loop +- Make sure that nothing else shuts down your host loop + +This way, your two event loops have the same lifetime, and your +program automatically exits when your Trio function finishes. + +Here's how we'd extend our asyncio example to implement this pattern: + +.. code-block:: python3 + :emphasize-lines: 8-11,19-22 + + # Improved version, that shuts down properly after Trio finishes + async def asyncio_main(): + asyncio_loop = asyncio.get_running_loop() + + def run_sync_soon_threadsafe(fn): + asyncio_loop.call_soon_threadsafe(fn) + + # Revised 'done' callback: set a Future + done_fut = asyncio.Future() + def done_callback(trio_main_outcome): + done_fut.set_result(trio_main_outcome) + + trio.lowlevel.start_guest_run( + trio_main, + run_sync_soon_threadsafe=run_sync_soon_threadsafe, + done_callback=done_callback, + ) + + # Wait for the guest run to finish + trio_main_outcome = await done_fut + # Pass through the return value or exception from the guest run + return trio_main_outcome.unwrap() + +And then you can encapsulate all this machinery in a utility function +that exposes a `trio.run`-like API, but runs both loops together:: + + def trio_run_with_asyncio(trio_main, *args, **trio_run_kwargs): + async def asyncio_main(): + # same as above + ... + + return asyncio.run(asyncio_main()) + +Technically, it is possible to use other patterns. But there are some +important limitations you have to respect: + +- **You must let the Trio program run to completion.** Many event + loops let you stop the event loop at any point, and any pending + callbacks/tasks/etc. just... don't run. Trio follows a more + structured system, where you can cancel things, but the code always + runs to completion, so ``finally`` blocks run, resources are cleaned + up, etc. If you stop your host loop early, before the + ``done_callback`` is invoked, then that cuts off the Trio run in the + middle without a chance to clean up, and can leave both your program + and Trio itself in an inconsistent state. (For example, + + Some programs need to be able to quit at any time, for example in + response to a GUI window being closed or a user selecting a "Quit" + from a menu. To handle these cases, we recommend wrapping your whole + program in a `trio.CancelScope`, and cancelling it when you want to + quit. + +- Each host loop can only have one `start_guest_run` at a time. If you + try to start a second one, you'll get an error. If you need to run + multiple Trio functions at the same time, then start up a single + Trio run, open a nursery, and then start your functions as child + tasks in that nursery. + +Given these constraints, we think the simplest approach is to always +start and stop the two loops together. + +**Signal management:** `"Signals" +`__ are a low-level +inter-process communication primitive. When you hit control-C to kill +a program, that uses a signal. Signal handling in Python has `a lot of +moving parts +`__. +One of those parts is `signal.set_wakeup_fd`, which event loops use to +make sure that they wake up when a signal arrives so they can respond +to it. (If you've ever had an event loop ignore you when you hit +control-C, it was probably because they weren't using +`signal.set_wakeup_fd` correctly.) + +But, only one event loop can use `signal.set_wakeup_fd` at a time. And +in guest mode that can cause problems: Trio and the host loop might +start fighting over who's using `signal.set_wakeup_fd`. + +Some event loops, like asyncio, won't work correctly unless they win +this fight. Fortunately, Trio is a little less picky: as long as +*someone* makes sure that the program wakes up when a signal arrives, +it should work correctly. So if your host loop wants +`signal.set_wakeup_fd`, then you should disable Trio's +`signal.set_wakeup_fd` support, and then both loops will work +correctly. + +On the other hand, if your host loop doesn't use +`signal.set_wakeup_fd`, then the only way to make everything work +correctly is to *enable* Trio's `signal.set_wakeup_fd` support. + +By default, Trio assumes that your host loop doesn't use +`signal.set_wakeup_fd`. It does try to detect when this creates a +conflict with the host loop, and print a warning – but unfortunately, +by the time it detects it, the damage has already been done. So if +you're getting this warning, then you should disable Trio's +`signal.set_wakeup_fd` support by passing +``host_uses_signal_set_wakeup_fd=True`` to `start_guest_run`. + +If you aren't seeing any warnings with your initial prototype, you're +*probably* fine. But the only way to be certain is to check your host +loop's source. For example, asyncio may or may not use +`signal.set_wakeup_fd` depending on the Python version and operating +system. + + +**Control-C handling** XX FIXME + + +**A small optimization:** Finally, consider a small optimization. Some +event loops offer two versions of their "call this function soon" API: +one that can be used from any thread, and one that can only be used +from the event loop thread, with the latter being cheaper. For +example, asyncio has both `~asyncio.loop.call_soon_threadsafe` and +`~asyncio.loop.call_soon`. + +If you have a loop like this, then you can also pass a +``run_sync_soon_not_threadsafe=...`` kwarg to `start_guest_run`, and +Trio will automatically use it when appropriate. + +If your loop doesn't have a split like this, then don't worry about +it; ``run_sync_soon_not_threadsafe=`` is optional. (If it's not +passed, then Trio will just use your threadsafe version in all cases.) + + + +Limitations +----------- + +only one run + +clock, autojump clock, deadlock detection, ... + + +Reference +--------- + +.. autofunction:: start_guest_run + + .. _live-coroutine-handoff: Handing off live coroutine objects between coroutine runners ------------------------------------------------------------- +============================================================ Internally, Python's async/await syntax is built around the idea of "coroutine objects" and "coroutine runners". A coroutine object diff --git a/trio/_core/_run.py b/trio/_core/_run.py index ee1c1a454d..5be9f27e90 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1103,7 +1103,7 @@ class _RunStatistics: # This holds all the state that gets trampolined back and forth between # callbacks when we're running in guest mode. # -# It has to be a separate object from Runner, and Runner *cannot* have hold +# It has to be a separate object from Runner, and Runner *cannot* hold # references to it (directly or indirectly)! # # The idea is that we want a chance to detect if our host loop quits and stops @@ -1805,10 +1805,57 @@ def start_guest_run( run_sync_soon_threadsafe, done_callback, run_sync_soon_not_threadsafe=None, - trust_host_loop_to_wake_on_signals=False, + host_uses_signal_set_wakeup_fd=False, clock=None, instruments=(), ): + """Start a "guest" run of Trio on top of some other "host" event loop. + + Each host loop can only have one guest run at a time. + + You should always let the Trio run finish before stopping the host loop; + if not, it may leave Trio's internal data structures in an inconsistent + state. You might be able to get away with it if you immediately exit the + program, but it's safest not to go there in the first place. + + So generally, the best way to do this is wrap this in a function that + starts the host loop and then immediately starts the guest run, and then + shuts down the host when the guest run completes. + + Args: + + run_sync_soon_threadsafe: An arbitrary callable, which will be passed a + function as its sole argument:: + + def my_run_sync_soon_threadsafe(fn): + hi() + + This callable should schedule ``fn`` to be run by the host on its + next pass through its loop. **Must support being called from + arbitrary threads.** + + done_callback: An arbitrary callable:: + + def my_done_callback(run_outcome): + hi() + + When the Trio run has finished, Trio will invoke this callback to let + you know. The argument is an `outcome.Outcome`, reporting what would + have been returned or raised by `trio.run`. This function can do + anything you want, but commonly you'll want it to shut down the + host loop, unwrap the outcome, etc. + + run_sync_soon_not_threadsafe: Optional. Like + ``run_sync_soon_threadsafe``, but will only be called from inside the + host loop's main thread. + + host_uses_signal_set_wakeup_fd (bool): Pass `True` if your host loop + uses `signal.set_wakeup_fd`, and `False` otherwise. For more details, + see :ref:`guest-run-implementation`. + + For the meaning of other arguments, see `trio.run`. + + """ runner = setup_runner(clock, instruments) runner.is_guest = True runner.guest_tick_scheduled = True @@ -1825,7 +1872,7 @@ def start_guest_run( runner, async_fn, args, - trust_host_loop_to_wake_on_signals=trust_host_loop_to_wake_on_signals, + host_uses_signal_set_wakeup_fd=host_uses_signal_set_wakeup_fd, ), ) run_sync_soon_not_threadsafe(guest_state.guest_tick) @@ -1840,12 +1887,12 @@ def start_guest_run( # mode", where our core event loop gets unrolled into a series of callbacks on # the host loop. If you're doing a regular trio.run then this gets run # straight through. -def unrolled_run(runner, async_fn, args, trust_host_loop_to_wake_on_signals=False): +def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True __tracebackhide__ = True try: - if not trust_host_loop_to_wake_on_signals: + if not host_uses_signal_set_wakeup_fd: runner.entry_queue.wakeup.wakeup_on_signals() if runner.instruments: diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 392a2c0b2e..80d3090ee9 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -73,7 +73,7 @@ def wakeup_on_signals(self): "It looks like Trio's signal handling code might have " "collided with another library you're using. If you're " "running Trio in guest mode, then this might mean you " - "should set trust_host_loop_to_wake_on_signals=True. " + "should set host_uses_signal_set_wakeup_fd=True. " "Otherwise, file a bug on Trio and we'll help you figure " "out what's going on." ) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index c11c2dc70b..5660f6eddc 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -157,7 +157,7 @@ async def trio_main(in_host): with pytest.warns(RuntimeWarning, match="signal handling code.*collided"): assert ( trivial_guest_run( - trio_main, trust_host_loop_to_wake_on_signals=False + trio_main, host_uses_signal_set_wakeup_fd=False ) == "ok" ) @@ -173,7 +173,7 @@ async def trio_main(in_host): with pytest.warns(None) as record: assert ( - trivial_guest_run(trio_main, trust_host_loop_to_wake_on_signals=True) + trivial_guest_run(trio_main, host_uses_signal_set_wakeup_fd=True) == "ok" ) with pytest.raises(AssertionError): @@ -194,7 +194,7 @@ async def trio_check_wakeup_fd_unaltered(in_host): assert ( trivial_guest_run( trio_check_wakeup_fd_unaltered, - trust_host_loop_to_wake_on_signals=True, + host_uses_signal_set_wakeup_fd=True, ) == "ok" ) @@ -386,7 +386,7 @@ async def aio_pingpong(from_trio, to_trio): # Not all versions of asyncio we test on can actually be trusted, # but this test doesn't care about signal handling, and it's # easier to just avoid the warnings. - trust_host_loop_to_wake_on_signals=True, + host_uses_signal_set_wakeup_fd=True, ) == "trio-main-done" ) @@ -397,7 +397,7 @@ async def aio_pingpong(from_trio, to_trio): # Also check that passing only call_soon_threadsafe works, via the # fallback path where we use it for everything. pass_not_threadsafe=False, - trust_host_loop_to_wake_on_signals=True, + host_uses_signal_set_wakeup_fd=True, ) == "trio-main-done" ) From 8018ceffda3c742c9471c1b7b39301f51c465663 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 31 May 2020 19:43:27 -0700 Subject: [PATCH 0216/1498] newsfragment --- newsfragments/399.feature.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 newsfragments/399.feature.rst diff --git a/newsfragments/399.feature.rst b/newsfragments/399.feature.rst new file mode 100644 index 0000000000..4de52bfb30 --- /dev/null +++ b/newsfragments/399.feature.rst @@ -0,0 +1,3 @@ +If you want to use Trio, but are stuck with some other event loop like +Qt or PyGame, then good news: now you can have both. For details, see: +:ref:`guest-mode`. From eea80105a5356f88d7c564ad276a7ee58ec29102 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 31 May 2020 19:55:45 -0700 Subject: [PATCH 0217/1498] black --- trio/_core/tests/test_guest_mode.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 5660f6eddc..d4efba3449 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -156,9 +156,7 @@ async def trio_main(in_host): try: with pytest.warns(RuntimeWarning, match="signal handling code.*collided"): assert ( - trivial_guest_run( - trio_main, host_uses_signal_set_wakeup_fd=False - ) + trivial_guest_run(trio_main, host_uses_signal_set_wakeup_fd=False) == "ok" ) finally: From 948ef191cea961a0eb268bff0e1007c9b7696c3c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2020 07:09:45 +0000 Subject: [PATCH 0218/1498] Bump ipython from 7.14.0 to 7.15.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.14.0 to 7.15.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.14.0...7.15.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 0d85e6287f..47548117ed 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -20,7 +20,7 @@ flake8==3.8.2 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets -ipython==7.14.0 # via -r test-requirements.in +ipython==7.15.0 # via -r test-requirements.in isort==4.3.21 # via pylint jedi==0.17.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From 7c0744b43b9693b58c522695362a5d73f98079b8 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 01:26:48 -0700 Subject: [PATCH 0219/1498] Doc updates --- docs/source/reference-lowlevel.rst | 17 +++--- trio/_core/_run.py | 83 ++++++++++++++++++++++++++---- 2 files changed, 84 insertions(+), 16 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 371f47c365..9a4ffd6afb 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -587,9 +587,9 @@ like Qt. Its advantages are: # ...and Qt can call back to Trio just as easily my_cancel_button.clicked.connect(my_cancel_scope.cancel) -- Consistent behavior: guest mode uses the code as regular Trio: the - same scheduler, same IO code, same everything. So you get the same - features and robustness that you're used to. +- Consistent behavior: guest mode uses the same code as regular Trio: + the same scheduler, same IO code, same everything. So you get the + full feature set and everything acts the way you expect. - Simple integration and broad compatibility: pretty much every event loop offers some threadsafe "schedule a callback" operation, and @@ -810,12 +810,14 @@ important limitations you have to respect: runs to completion, so ``finally`` blocks run, resources are cleaned up, etc. If you stop your host loop early, before the ``done_callback`` is invoked, then that cuts off the Trio run in the - middle without a chance to clean up, and can leave both your program - and Trio itself in an inconsistent state. (For example, + middle without a chance to clean up. This can leave your code in an + inconsistent state, and will definitely leave Trio's internals in an + inconsistent state, which will cause errors if you try to use Trio + again in that thread. Some programs need to be able to quit at any time, for example in response to a GUI window being closed or a user selecting a "Quit" - from a menu. To handle these cases, we recommend wrapping your whole + from a menu. In these cases, we recommend wrapping your whole program in a `trio.CancelScope`, and cancelling it when you want to quit. @@ -889,6 +891,9 @@ If your loop doesn't have a split like this, then don't worry about it; ``run_sync_soon_not_threadsafe=`` is optional. (If it's not passed, then Trio will just use your threadsafe version in all cases.) +**That's it!** If you've followed all these steps, you should now have +a cleanly-integrated hybrid event loop. Go make some cool +GUIs/games/whatever! Limitations diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 5be9f27e90..ca42e47769 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1666,7 +1666,68 @@ def remove_instrument(self, instrument): ################################################################ # run ################################################################ - +# +# Trio's core task scheduler and coroutine runner is in 'unrolled_run'. It's +# called that because it has an unusual feature: it's actually a generator. +# Whenever it needs to fetch IO events from the OS, it yields, and waits for +# its caller to send the IO events back in. So the loop is "unrolled" into a +# sequence of generator send() calls. +# +# The reason for this unusual design is to support two different modes of +# operation, where the IO is handled differently. +# +# In normal mode using trio.run, the scheduler and IO run in the same thread: +# +# Main thread: +# +# +---------------------------+ +# | Run tasks | +# | (unrolled_run) | +# +---------------------------+ +# | Block waiting for I/O | +# | (io_manager.get_events) | +# +---------------------------+ +# | Run tasks | +# | (unrolled_run) | +# +---------------------------+ +# | Block waiting for I/O | +# | (io_manager.get_events) | +# +---------------------------+ +# : +# +# +# In guest mode using trio.start_guest_run, the scheduler runs on the main +# thread as a host loop callback, but blocking for IO gets pushed into a +# worker thread: +# +# Main thread executing host loop: Trio I/O thread: +# +# +---------------------------+ +# | Run Trio tasks | +# | (unrolled_run) | +# +---------------------------+ --------------+ +# v +# +---------------------------+ +----------------------------+ +# | Host loop does whatever | | Block waiting for Trio I/O | +# | it wants | | (io_manager.get_events) | +# +---------------------------+ +----------------------------+ +# | +# +---------------------------+ <-------------+ +# | Run Trio tasks | +# | (unrolled_run) | +# +---------------------------+ --------------+ +# v +# +---------------------------+ +----------------------------+ +# | Host loop does whatever | | Block waiting for Trio I/O | +# | it wants | | (io_manager.get_events) | +# +---------------------------+ +----------------------------+ +# : : +# +# Most of Trio's internals don't need to care about this difference. The main +# complication it creates is that in guest mode, we might need to wake up not +# just due to OS-reported IO events, but also because of code running on the +# host loop calling reschedule() or changing task deadlines. Search for +# 'is_guest' to see the special cases we need to handle this. def setup_runner(clock, instruments): """Create a Runner object and install it as the GLOBAL_RUN_CONTEXT.""" @@ -1818,9 +1879,9 @@ def start_guest_run( state. You might be able to get away with it if you immediately exit the program, but it's safest not to go there in the first place. - So generally, the best way to do this is wrap this in a function that - starts the host loop and then immediately starts the guest run, and then - shuts down the host when the guest run completes. + Generally, the best way to do this is wrap this in a function that starts + the host loop and then immediately starts the guest run, and then shuts + down the host when the guest run completes. Args: @@ -1828,16 +1889,16 @@ def start_guest_run( function as its sole argument:: def my_run_sync_soon_threadsafe(fn): - hi() + ... - This callable should schedule ``fn`` to be run by the host on its + This callable should schedule ``fn()`` to be run by the host on its next pass through its loop. **Must support being called from arbitrary threads.** done_callback: An arbitrary callable:: def my_done_callback(run_outcome): - hi() + ... When the Trio run has finished, Trio will invoke this callback to let you know. The argument is an `outcome.Outcome`, reporting what would @@ -1845,9 +1906,11 @@ def my_done_callback(run_outcome): anything you want, but commonly you'll want it to shut down the host loop, unwrap the outcome, etc. - run_sync_soon_not_threadsafe: Optional. Like - ``run_sync_soon_threadsafe``, but will only be called from inside the - host loop's main thread. + run_sync_soon_not_threadsafe: Like ``run_sync_soon_threadsafe``, but + will only be called from inside the host loop's main thread. + Optional, but if your host loop allows you to implement this more + efficiently than ``run_sync_soon_threadsafe`` then passing it will + make things a bit faster. host_uses_signal_set_wakeup_fd (bool): Pass `True` if your host loop uses `signal.set_wakeup_fd`, and `False` otherwise. For more details, From 8d9effc1d32b8ef2f7a7a02c1b02a4de3f5f8e3d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 02:08:12 -0700 Subject: [PATCH 0220/1498] Switch KI protection default from "unprotected" to "protected" This is mostly motivated by wanting to allow guest-mode (gh-1551) to use our KI handling, where the host loop will presumably not be using any of our KI protection markers. But I figured I'd split it out into a separate PR rather than mix changes to the tricky KI code together with changes to the tricky core run loop. No newsfragment because I don't think this change is observable by users. --- docs/source/reference-lowlevel.rst | 14 +++++++++----- trio/_core/_ki.py | 2 +- trio/_core/_run.py | 1 - trio/_core/tests/test_ki.py | 13 ++++++++++--- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index b196cd0b43..ef25e6cce4 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -295,11 +295,15 @@ delivered). So that's great, but – how do we know whether we're in one of the sensitive parts of the program or not? -This is determined on a function-by-function basis. By default, a -function is protected if its caller is, and not if its caller isn't; -this is helpful because it means you only need to override the -defaults at places where you transition from protected code to -unprotected code or vice-versa. +This is determined on a function-by-function basis. By default: + +- The top-level function in regular user tasks is unprotected. +- The top-level function in system tasks is protected. +- If a function doesn't specify otherwise, then it inherits the + protection state of its caller. + +This means you only need to override the defaults at places where you +transition from protected code to unprotected code or vice-versa. These transitions are accomplished using two function decorators: diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index d5aa63f5d9..ec22ff5fb6 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -90,7 +90,7 @@ def ki_protection_enabled(frame): if frame.f_code.co_name == "__del__": return True frame = frame.f_back - return False + return True def currently_ki_protected(): diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 4de4e9a2bf..b6c5e19647 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1678,7 +1678,6 @@ def run( system_context=system_context, ) GLOBAL_RUN_CONTEXT.runner = runner - locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True # KI handling goes outside the core try/except/finally to avoid a window # where KeyboardInterrupt would be allowed and converted into an diff --git a/trio/_core/tests/test_ki.py b/trio/_core/tests/test_ki.py index 51b60aebbf..0e4db4af49 100644 --- a/trio/_core/tests/test_ki.py +++ b/trio/_core/tests/test_ki.py @@ -222,8 +222,8 @@ async def agen_unprotected3(): # Test the case where there's no magic local anywhere in the call stack -def test_ki_enabled_out_of_context(): - assert not _core.currently_ki_protected() +def test_ki_disabled_out_of_context(): + assert _core.currently_ki_protected() def test_ki_disabled_in_del(): @@ -234,8 +234,15 @@ def __del__(): assert _core.currently_ki_protected() assert nestedfunction() + @_core.disable_ki_protection + def outerfunction(): + assert not _core.currently_ki_protected() + assert not nestedfunction() + __del__() + __del__() - assert not nestedfunction() + outerfunction() + assert nestedfunction() def test_ki_protection_works(): From be88548e554b00d42596842a5c99380130ab4653 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 01:59:41 -0700 Subject: [PATCH 0221/1498] Add note about custom clocks in guest mode --- docs/source/reference-lowlevel.rst | 14 +++++++++++--- docs/source/reference-testing.rst | 2 ++ trio/_core/_run.py | 1 + 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 9a4ffd6afb..9e6e8d8a21 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -899,9 +899,17 @@ GUIs/games/whatever! Limitations ----------- -only one run - -clock, autojump clock, deadlock detection, ... +In general, almost all Trio features should work in guest mode. The +exception is features which rely on Trio having a complete picture of +everything that your program is doing, since obviously, it can't +control the host loop or see what it's doing. + +Custom clocks can be used in guest mode, but they only affect Trio +timeouts, not host loop timeouts. And the :ref:`autojump clock +` and related `trio.testing.wait_all_tasks_blocked` can +technically be used in guest mode, but they'll only take Trio tasks +into account when decided whether to jump the clock or whether all +tasks are blocked. Reference diff --git a/docs/source/reference-testing.rst b/docs/source/reference-testing.rst index 40a275bbeb..76ecd4a2d4 100644 --- a/docs/source/reference-testing.rst +++ b/docs/source/reference-testing.rst @@ -16,6 +16,8 @@ Test harness integration .. decorator:: trio_test +.. _testing-time: + Time and timeouts ----------------- diff --git a/trio/_core/_run.py b/trio/_core/_run.py index ca42e47769..a0f5083f0a 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1729,6 +1729,7 @@ def remove_instrument(self, instrument): # host loop calling reschedule() or changing task deadlines. Search for # 'is_guest' to see the special cases we need to handle this. + def setup_runner(clock, instruments): """Create a Runner object and install it as the GLOBAL_RUN_CONTEXT.""" # It wouldn't be *hard* to support nested calls to run(), but I can't From b84a715e5140b2242377e59cdd28048cc6c81942 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 13:50:44 -0700 Subject: [PATCH 0222/1498] Fix comment --- trio/_core/_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index a0f5083f0a..edb0369ca8 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1696,8 +1696,8 @@ def remove_instrument(self, instrument): # : # # -# In guest mode using trio.start_guest_run, the scheduler runs on the main -# thread as a host loop callback, but blocking for IO gets pushed into a +# In guest mode using trio.lowlevel.start_guest_run, the scheduler runs on the +# main thread as a host loop callback, but blocking for IO gets pushed into a # worker thread: # # Main thread executing host loop: Trio I/O thread: From c120632ee9b78591144b54305faba6532107dda0 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 14:32:54 -0700 Subject: [PATCH 0223/1498] Enable KI handling in guest mode --- trio/_core/_ki.py | 52 +++++++++++--------- trio/_core/_run.py | 74 +++++++++++++++++------------ trio/_core/tests/test_guest_mode.py | 33 +++++++++++++ 3 files changed, 105 insertions(+), 54 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index ec22ff5fb6..01ce03ce59 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -3,6 +3,7 @@ import sys from contextlib import contextmanager from functools import wraps +import attr import async_generator @@ -170,26 +171,31 @@ def wrapper(*args, **kwargs): disable_ki_protection.__name__ = "disable_ki_protection" -@contextmanager -def ki_manager(deliver_cb, restrict_keyboard_interrupt_to_checkpoints): - if ( - not is_main_thread() - or signal.getsignal(signal.SIGINT) != signal.default_int_handler - ): - yield - return - - def handler(signum, frame): - assert signum == signal.SIGINT - protection_enabled = ki_protection_enabled(frame) - if protection_enabled or restrict_keyboard_interrupt_to_checkpoints: - deliver_cb() - else: - raise KeyboardInterrupt - - signal.signal(signal.SIGINT, handler) - try: - yield - finally: - if signal.getsignal(signal.SIGINT) is handler: - signal.signal(signal.SIGINT, signal.default_int_handler) +@attr.s +class KIManager: + handler = attr.ib(default=None) + + def install(self, deliver_cb, restrict_keyboard_interrupt_to_checkpoints): + assert self.handler is None + if ( + not is_main_thread() + or signal.getsignal(signal.SIGINT) != signal.default_int_handler + ): + return + + def handler(signum, frame): + assert signum == signal.SIGINT + protection_enabled = ki_protection_enabled(frame) + if protection_enabled or restrict_keyboard_interrupt_to_checkpoints: + deliver_cb() + else: + raise KeyboardInterrupt + + self.handler = handler + signal.signal(signal.SIGINT, handler) + + def close(self): + if self.handler is not None: + if signal.getsignal(signal.SIGINT) is self.handler: + signal.signal(signal.SIGINT, signal.default_int_handler) + self.handler = None diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 6b16a4a9d2..a10d6cb57f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -27,7 +27,7 @@ from ._exceptions import TrioInternalError, RunFinishedError, Cancelled from ._ki import ( LOCALS_KEY_KI_PROTECTION_ENABLED, - ki_manager, + KIManager, enable_ki_protection, ) from ._multierror import MultiError @@ -1148,11 +1148,9 @@ class GuestState: unrolled_run_next_send = attr.ib(factory=lambda: Value(None)) def guest_tick(self): - locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: timeout = self.unrolled_run_next_send.send(self.unrolled_run_gen) except StopIteration: - # XX if we had KI support, we'd have to do something with it here self.done_callback(self.runner.main_task_outcome) return except TrioInternalError as exc: @@ -1189,6 +1187,7 @@ class Runner: clock = attr.ib() instruments = attr.ib() io_manager = attr.ib() + ki_manager = attr.ib() # Run-local values, see _local.py _locals = attr.ib(factory=dict) @@ -1225,6 +1224,8 @@ def close(self): self.entry_queue.close() if self.instruments: self.instrument("after_run") + # This is where KI protection gets disabled, so we do it last + self.ki_manager.close() def next_deadline(self): try: @@ -1749,7 +1750,7 @@ def remove_instrument(self, instrument): # 'is_guest' to see the special cases we need to handle this. -def setup_runner(clock, instruments): +def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints): """Create a Runner object and install it as the GLOBAL_RUN_CONTEXT.""" # It wouldn't be *hard* to support nested calls to run(), but I can't # think of a single good reason for it, so let's be conservative for @@ -1763,12 +1764,20 @@ def setup_runner(clock, instruments): io_manager = TheIOManager() system_context = copy_context() system_context.run(current_async_library_cvar.set, "trio") + ki_manager = KIManager() + runner = Runner( clock=clock, instruments=instruments, io_manager=io_manager, system_context=system_context, + ki_manager=ki_manager, ) + + # This is where KI protection gets enabled, so we want to do it early - in + # particular before we start modifying global state like GLOBAL_RUN_CONTEXT + ki_manager.install(runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints) + GLOBAL_RUN_CONTEXT.runner = runner return runner @@ -1850,33 +1859,24 @@ def run( __tracebackhide__ = True - runner = setup_runner(clock, instruments) + runner = setup_runner( + clock, instruments, restrict_keyboard_interrupt_to_checkpoints + ) - # KI handling goes outside unrolled_run to avoid an interval where - # KeyboardInterrupt would be allowed and converted into an - # TrioInternalError: - try: - with ki_manager(runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints): - gen = unrolled_run(runner, async_fn, args) - next_send = None - while True: - try: - timeout = gen.send(next_send) - except StopIteration: - break - next_send = runner.io_manager.get_events(timeout) - # Inlined copy of runner.main_task_outcome.unwrap() to avoid - # cluttering every single Trio traceback with an extra frame. - if isinstance(runner.main_task_outcome, Value): - return runner.main_task_outcome.value - else: - raise runner.main_task_outcome.error - finally: - # To guarantee that we never swallow a KeyboardInterrupt, we have to - # check for pending ones once more after leaving the context manager: - if runner.ki_pending: - # Implicitly chains with any exception from outcome.unwrap(): - raise KeyboardInterrupt + gen = unrolled_run(runner, async_fn, args) + next_send = None + while True: + try: + timeout = gen.send(next_send) + except StopIteration: + break + next_send = runner.io_manager.get_events(timeout) + # Inlined copy of runner.main_task_outcome.unwrap() to avoid + # cluttering every single Trio traceback with an extra frame. + if isinstance(runner.main_task_outcome, Value): + return runner.main_task_outcome.value + else: + raise runner.main_task_outcome.error def start_guest_run( @@ -1888,6 +1888,7 @@ def start_guest_run( host_uses_signal_set_wakeup_fd=False, clock=None, instruments=(), + restrict_keyboard_interrupt_to_checkpoints=False, ): """Start a "guest" run of Trio on top of some other "host" event loop. @@ -1938,7 +1939,9 @@ def my_done_callback(run_outcome): For the meaning of other arguments, see `trio.run`. """ - runner = setup_runner(clock, instruments) + runner = setup_runner( + clock, instruments, restrict_keyboard_interrupt_to_checkpoints + ) runner.is_guest = True runner.guest_tick_scheduled = True @@ -2154,6 +2157,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): del GLOBAL_RUN_CONTEXT.task except GeneratorExit: + # The run-loop generator has been garbage collected without finishing warnings.warn( RuntimeWarning( "Trio guest run got abandoned without properly finishing... " @@ -2167,6 +2171,14 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): finally: GLOBAL_RUN_CONTEXT.__dict__.clear() runner.close() + # Have to do this after runner.close() has disabled KI protection, + # because otherwise there's a race where ki_pending could get set + # after we check it. + if runner.ki_pending: + ki = KeyboardInterrupt() + if isinstance(runner.main_task_outcome, Error): + ki.__context__ = runner.main_task_outcome.error + runner.main_task_outcome = Error(ki) ################################################################ diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index d4efba3449..46e741e392 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -11,6 +11,7 @@ import trio import trio.testing from .tutil import gc_collect_harder +from ..._util import signal_raise # The simplest possible "host" loop. # Nice features: @@ -440,3 +441,35 @@ def bad_get_events(*args): trivial_guest_run(crash_in_worker_thread_io) gc_collect_harder() + + +def test_guest_mode_ki(): + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + + # Check SIGINT in Trio func and in host func + async def trio_main(in_host): + with pytest.raises(KeyboardInterrupt): + signal_raise(signal.SIGINT) + + # Host SIGINT should get injected into Trio + in_host(partial(signal_raise, signal.SIGINT)) + await trio.sleep(10) + + with pytest.raises(KeyboardInterrupt) as excinfo: + trivial_guest_run(trio_main) + assert excinfo.value.__context__ is None + # Signal handler should be restored properly on exit + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + + # Also check chaining in the case where KI is injected after main exits + final_exc = KeyError("whoa") + + async def trio_main_raising(in_host): + in_host(partial(signal_raise, signal.SIGINT)) + raise final_exc + + with pytest.raises(KeyboardInterrupt) as excinfo: + trivial_guest_run(trio_main_raising) + assert excinfo.value.__context__ is final_exc + + assert signal.getsignal(signal.SIGINT) is signal.default_int_handler From 7c9ce288ee5d8e5f6452eed4be4d34c2ee44f3f8 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 14:39:39 -0700 Subject: [PATCH 0224/1498] Add note about guest mode KI handling to docs --- docs/source/reference-lowlevel.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 92f0e1dab3..77f3a83170 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -833,6 +833,15 @@ important limitations you have to respect: Trio run, open a nursery, and then start your functions as child tasks in that nursery. +- Unless you or your host loop register a handler for `signal.SIGINT` + before starting Trio (this is not common), then Trio will take over + delivery of `KeyboardInterrupt`\s. And since Trio can't tell which + host code is safe to interrupt, it will only deliver + `KeyboardInterrupt` into the Trio part of your code. This is fine if + your program is set up to exit when the Trio part exits, because the + `KeyboardInterrupt` will propagate out of Trio and then trigger the + shutdown of your host loop, which is just what you want. + Given these constraints, we think the simplest approach is to always start and stop the two loops together. From 19882c73577880bd359763bcc6d73dda2283ed1c Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 16:42:06 -0700 Subject: [PATCH 0225/1498] Clarify that you can't magically share async code between host and guest --- docs/source/reference-lowlevel.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 77f3a83170..b72a5fe457 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -582,17 +582,21 @@ like Qt. Its advantages are: so it has low latency and doesn't waste electricity. - No need to think about threads: your Trio code runs in the same - thread as the host event loop, so you can freely call Trio APIs from - the host, and call host APIs from Trio. For example, if you're - making a GUI app with Qt as the host loop, then making a `cancel - button `__ and connecting - it to a `trio.CancelScope` is as easy as writing:: + thread as the host event loop, so you can freely call sync Trio APIs + from the host, and call sync host APIs from Trio. For example, if + you're making a GUI app with Qt as the host loop, then making a + `cancel button `__ and + connecting it to a `trio.CancelScope` is as easy as writing:: # Trio code can create Qt objects without any special ceremony... my_cancel_button = QPushButton("Cancel") # ...and Qt can call back to Trio just as easily my_cancel_button.clicked.connect(my_cancel_scope.cancel) + (For async APIs, it's not that simple, but you can build on this + make explicit bridges between the two worlds, e.g. by passing async + functions through queues.) + - Consistent behavior: guest mode uses the same code as regular Trio: the same scheduler, same IO code, same everything. So you get the full feature set and everything acts the way you expect. From 7207cde2956874236e062175f8a38c3e3ecbb28b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 17:58:18 -0700 Subject: [PATCH 0226/1498] Remove defunct draft docs --- docs/source/design.rst | 74 ------------------------------------------ 1 file changed, 74 deletions(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 184e60c110..6251f22cdb 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -534,77 +534,3 @@ there is no provision for "pluggable" backends. The intuition here is that we'd rather focus our energy on making one set of solid, official backends that provide a high-quality experience out-of-the-box on all supported systems. - - -Guest mode ----------- - -One of Trio's more unusual features is that it supports being run in -"guest mode" on top of some other event loop (the "host"). - -XX TODO: document this properly - -the basic idea of pushing ``get_events`` into a thread - -actual core logic is identical whether running in regular mode or -guest mode; alternate between waiting for I/O+timeout vs running trio tasks - -one extra wrinkle is: normally tasks can only become runnable, and -deadline can only change, if trio task is running. In guest mode, -that's no longer true. So reschedule() and deadline changes need to -potentially trigger the scheduler, or at least update the I/O -deadline. Do this in the simplest possible way: force the I/O thread -to return immediately via the normal path. - -subtlety around wait_all_tasks_blocked and 'events' semantics - -diagram:: - - - Normal mode - - Main thread executing trio.run: - - +---------------------------+ - | wait for I/O+timeout | - +---------------------------+ - | run trio tasks | - +---------------------------+ - | wait for I/O+timeout | - +---------------------------+ - | run trio tasks | - +---------------------------+ - | wait for I/O+timeout | - +---------------------------+ - | run trio tasks | - +---------------------------+ - . - . - . - - - Guest mode - - Main thread executing host loop: Trio I/O thread: - - +---------------------------+ - | host loop does its thing | +---------------------------+ - | | | wait for trio I/O+timeout | - +---------------------------+ +---------------------------+ - / - +---------------------------+ <---------------/ - | run trio tasks | - +---------------------------+ ----------------\ - \ - +---------------------------+ v - | host loop does its thing | +---------------------------+ - | | | wait for trio I/O+timeout | - +---------------------------+ +---------------------------+ - / - +---------------------------+ <---------------/ - | run trio tasks | - +---------------------------+ ----------------\ - \ - . - . - . From c6697dbe3e5865f48a41b7e13952f3065f1d2d45 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 17:58:40 -0700 Subject: [PATCH 0227/1498] Remove obsolete placeholder --- docs/source/reference-lowlevel.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index b72a5fe457..bf5666d2b0 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -892,9 +892,6 @@ loop's source. For example, asyncio may or may not use system. -**Control-C handling** XX FIXME - - **A small optimization:** Finally, consider a small optimization. Some event loops offer two versions of their "call this function soon" API: one that can be used from any thread, and one that can only be used From 66967ce57e91ce5fefc2c5db3e98ac1b33ca0a52 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 18:03:40 -0700 Subject: [PATCH 0228/1498] Make kqueue code more similar to epoll code --- trio/_core/_io_kqueue.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index ce85453749..d2d80a7341 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -23,12 +23,14 @@ class KqueueIOManager: # {(ident, filter): Task or UnboundedQueue} _registered = attr.ib(factory=dict) _force_wakeup = attr.ib(factory=WakeupSocketpair) + _force_wakeup_fd = attr.ib(default=None) def __attrs_post_init__(self): force_wakeup_event = select.kevent( self._force_wakeup.wakeup_sock, select.KQ_FILTER_READ, select.KQ_EV_ADD ) self._kqueue.control([force_wakeup_event], 0) + self._force_wakeup_fd = self._force_wakeup.wakeup_sock.fileno() def statistics(self): tasks_waiting = 0 @@ -66,14 +68,10 @@ def get_events(self, timeout): def process_events(self, events): for event in events: key = (event.ident, event.filter) - try: - receiver = self._registered[key] - except KeyError: - if event.ident == self._force_wakeup.wakeup_sock.fileno(): - self._force_wakeup.drain() - continue - else: # pragma: no cover - raise + if event.ident == self._force_wakeup_fd: + self._force_wakeup.drain() + continue + receiver = self._registered[key] if event.flags & select.KQ_EV_ONESHOT: del self._registered[key] if type(receiver) is _core.Task: From 3083de56427725eb5c32a821a7d9ca8cebea671b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 18:06:50 -0700 Subject: [PATCH 0229/1498] Improve wording in docs --- docs/source/reference-lowlevel.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index bf5666d2b0..36b37e3e37 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -593,9 +593,9 @@ like Qt. Its advantages are: # ...and Qt can call back to Trio just as easily my_cancel_button.clicked.connect(my_cancel_scope.cancel) - (For async APIs, it's not that simple, but you can build on this - make explicit bridges between the two worlds, e.g. by passing async - functions through queues.) + (For async APIs, it's not that simple, but you can use sync APIs to + build explicit bridges between the two worlds, e.g. by passing async + functions and their results back and forth through queues.) - Consistent behavior: guest mode uses the same code as regular Trio: the same scheduler, same IO code, same everything. So you get the From 5cfce026f87cc11f1eda540ea96ae55e8ab4fb09 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 23:50:07 -0700 Subject: [PATCH 0230/1498] Make test more specific to prevent flakiness Sometimes ResourceWarnings decide to print during this test, and that used to cause it to fail every once in a while. --- trio/tests/test_threads.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 632ce13656..a9059fef79 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -270,7 +270,8 @@ async def child(): # Make sure we don't have a "Exception in thread ..." dump to the console: out, err = capfd.readouterr() - assert not out and not err + assert "Exception in thread" not in out + assert "Exception in thread" not in err @pytest.mark.parametrize("MAX", [3, 5, 10]) From 315f34defccb3b49c5311608e3db3f24071c5b2d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2020 06:15:00 +0000 Subject: [PATCH 0231/1498] Bump pytest from 5.4.2 to 5.4.3 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.2 to 5.4.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.2...5.4.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 47548117ed..90a09f8d97 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -44,7 +44,7 @@ pylint==2.5.2 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.9.0 # via -r test-requirements.in -pytest==5.4.2 # via -r test-requirements.in, pytest-cov +pytest==5.4.3 # via -r test-requirements.in, pytest-cov regex==2020.5.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in From fb4590a0910202eb35435d37720dc691af15c0fd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2020 06:15:30 +0000 Subject: [PATCH 0232/1498] Bump wcwidth from 0.1.9 to 0.2.3 Bumps [wcwidth](https://github.com/jquast/wcwidth) from 0.1.9 to 0.2.3. - [Release notes](https://github.com/jquast/wcwidth/releases) - [Commits](https://github.com/jquast/wcwidth/compare/0.1.9...0.2.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 47548117ed..4769f5376b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,5 +53,5 @@ toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black -wcwidth==0.1.9 # via prompt-toolkit, pytest +wcwidth==0.2.3 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid From 70f7a661fad63086b514215c5b6de2e1cf8c23a3 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Thu, 4 Jun 2020 11:00:08 -0300 Subject: [PATCH 0233/1498] Added a helpful error message if an async function is passed to trio.to_thread_run_sync --- newsfragments/1573.bugfix.rst | 1 + trio/_threads.py | 12 +++++++++++- trio/tests/test_threads.py | 9 +++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 newsfragments/1573.bugfix.rst diff --git a/newsfragments/1573.bugfix.rst b/newsfragments/1573.bugfix.rst new file mode 100644 index 0000000000..1995ea15ab --- /dev/null +++ b/newsfragments/1573.bugfix.rst @@ -0,0 +1 @@ +Added a helpful error message if an async function is passed to `trio.to_thread_run_sync`. \ No newline at end of file diff --git a/trio/_threads.py b/trio/_threads.py index 8867388bd7..0ce2a7e3d8 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -179,7 +179,17 @@ def do_release_then_return_result(): def worker_fn(): TOKEN_LOCAL.token = current_trio_token try: - return sync_fn(*args) + ret = sync_fn(*args) + + if inspect.iscoroutine(ret): + # Manually close coroutine to avoid RuntimeWarnings + ret.close() + raise TypeError( + "Trio expected a sync function, but {!r} appears to be " + "asynchronous".format(getattr(sync_fn, "__qualname__", sync_fn)) + ) + + return ret finally: del TOKEN_LOCAL.token diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index a9059fef79..976220cc32 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -457,6 +457,15 @@ def thread_fn(): assert callee_token == caller_token +async def test_trio_to_thread_run_sync(): + # Test correct error when passed async function + async def async_fn(): # pragma: no cover + pass + + with pytest.raises(TypeError, match="expected a sync function"): + await to_thread_run_sync(async_fn) + + async def test_trio_from_thread_run_sync(): # Test that to_thread_run_sync correctly "hands off" the trio token to # trio.from_thread.run_sync() From 20aff0dac3cdabae172b40dfdcf118cc2c7c3c80 Mon Sep 17 00:00:00 2001 From: Frank Smit Date: Fri, 5 Jun 2020 11:36:18 +0200 Subject: [PATCH 0234/1498] Add trio_redis to Trio libraries list. --- docs/source/awesome-trio-libraries.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index da3213330c..5e1f828e7c 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -42,6 +42,7 @@ Database * `trio-mysql `__ - Pure Python MySQL Client. * `sqlalchemy_aio `__ - Add asyncio and Trio support to SQLAlchemy core, derived from alchimia. * `redio `__ - Redis client, pure Python and Trio. +* `trio_redis `__ - A Redis client for Trio. Depends on hiredis-py. IOT From 475a84618e00c72bc49ea8ba96d8a4f159215c4a Mon Sep 17 00:00:00 2001 From: palkeo Date: Mon, 8 Jun 2020 16:50:35 +1200 Subject: [PATCH 0235/1498] Don't shuffle each batch, instead reverse them half the time. I consistently see batch shuffling taking 1% CPU on a real-world application, and this is a simple fix to make this overhead disappear. Suggested by @njsmith. --- trio/_core/_run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index a10d6cb57f..56c617372f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2084,7 +2084,10 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # seeded for tests. batch.sort(key=lambda t: t._counter) runner.runq.clear() - _r.shuffle(batch) + # 50% chance of reversing the batch, this way each task + # can appear before/after any other task. + if _r.random() < 0.5: + batch = batch[::-1] while batch: task = batch.pop() GLOBAL_RUN_CONTEXT.task = task From c8128f3c2f07c71ad2e5add401a3f6efb64a0a23 Mon Sep 17 00:00:00 2001 From: palkeo Date: Mon, 8 Jun 2020 17:20:34 +1200 Subject: [PATCH 0236/1498] Make test_scheduler_determinist much less likely to fail. The scheduler is still non-deterministic, but exhibit much less variety of behavior, making the test a lot more likely to fail. So we increase the size of the traces. --- trio/tests/test_scheduler_determinism.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_scheduler_determinism.py b/trio/tests/test_scheduler_determinism.py index 67b2447f0a..e2d3167e45 100644 --- a/trio/tests/test_scheduler_determinism.py +++ b/trio/tests/test_scheduler_determinism.py @@ -6,7 +6,7 @@ async def scheduler_trace(): trace = [] async def tracer(name): - for i in range(10): + for i in range(50): trace.append((name, i)) await trio.sleep(0) From a53ce0ad1ffd3fc1bebfd15729fa44cae8aa8414 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 8 Jun 2020 06:28:30 +0000 Subject: [PATCH 0237/1498] Ignore ResourceWarnings in test_deprecate --- trio/tests/test_deprecate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trio/tests/test_deprecate.py b/trio/tests/test_deprecate.py index a11e9b8d1f..421278f0d3 100644 --- a/trio/tests/test_deprecate.py +++ b/trio/tests/test_deprecate.py @@ -16,6 +16,9 @@ @pytest.fixture def recwarn_always(recwarn): warnings.simplefilter("always") + # ResourceWarnings about unclosed sockets can occur nondeterministically + # (during GC) which throws off the tests in this file + warnings.simplefilter("ignore", ResourceWarning) return recwarn From f093598d78d1d459c0d970f448e4084026f2e852 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 06:37:58 +0000 Subject: [PATCH 0238/1498] Bump sortedcontainers from 2.1.0 to 2.2.2 Bumps [sortedcontainers](https://github.com/grantjenks/python-sortedcontainers) from 2.1.0 to 2.2.2. - [Release notes](https://github.com/grantjenks/python-sortedcontainers/releases) - [Changelog](https://github.com/grantjenks/python-sortedcontainers/blob/master/HISTORY.rst) - [Commits](https://github.com/grantjenks/python-sortedcontainers/compare/v2.1.0...v2.2.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f4956a8ea6..7f73b4aaa0 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -27,7 +27,7 @@ requests==2.23.0 # via sphinx six==1.15.0 # via packaging sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx -sortedcontainers==2.1.0 # via -r docs-requirements.in +sortedcontainers==2.2.2 # via -r docs-requirements.in sphinx-rtd-theme==0.4.3 # via -r docs-requirements.in sphinx==3.0.4 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index 179efe6cc1..7c12096f70 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -48,7 +48,7 @@ pytest==5.4.3 # via -r test-requirements.in, pytest-cov regex==2020.5.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in -sortedcontainers==2.1.0 # via -r test-requirements.in +sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in From 3640fe052c4b96fb689d40ab9925b5549cf984f7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 06:38:32 +0000 Subject: [PATCH 0239/1498] Bump certifi from 2020.4.5.1 to 2020.4.5.2 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.4.5.1 to 2020.4.5.2. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.04.05.1...2020.04.05.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f4956a8ea6..28c9f76733 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in attrs==19.3.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx -certifi==2020.4.5.1 # via requests +certifi==2020.4.5.2 # via requests chardet==3.0.4 # via requests click==7.1.2 # via towncrier docutils==0.16 # via sphinx From 47f23b002cfdd8b26df2bc1e1b271e6b65b70fad Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 06:39:25 +0000 Subject: [PATCH 0240/1498] Bump wcwidth from 0.2.3 to 0.2.4 Bumps [wcwidth](https://github.com/jquast/wcwidth) from 0.2.3 to 0.2.4. - [Release notes](https://github.com/jquast/wcwidth/releases) - [Commits](https://github.com/jquast/wcwidth/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 179efe6cc1..f38fe779a8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,5 +53,5 @@ toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black -wcwidth==0.2.3 # via prompt-toolkit, pytest +wcwidth==0.2.4 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid From 438ddb631c02af3d0a692cd95532077d2f212395 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 06:50:39 +0000 Subject: [PATCH 0241/1498] Bump regex from 2020.5.14 to 2020.6.8 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.5.14 to 2020.6.8. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7c12096f70..80aefcca48 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.9.0 # via -r test-requirements.in pytest==5.4.3 # via -r test-requirements.in, pytest-cov -regex==2020.5.14 # via black +regex==2020.6.8 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in From a67aaf6db0b6925c0afd2ae2d80a9ffefd61a66f Mon Sep 17 00:00:00 2001 From: palkeo Date: Mon, 8 Jun 2020 23:29:53 +1200 Subject: [PATCH 0242/1498] Keep the old shuffle() behavior on hypothesis & reverse in-place. --- trio/_core/_run.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 56c617372f..465aebe6f5 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2076,6 +2076,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # tie-breaker and the non-deterministic ordering of # task._notify_queues.) batch = list(runner.runq) + runner.runq.clear() if _ALLOW_DETERMINISTIC_SCHEDULING: # We're running under Hypothesis, and pytest-trio has patched # this in to make the scheduler deterministic and avoid flaky @@ -2083,11 +2084,12 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # operation, since we'll shuffle the list and _r is only # seeded for tests. batch.sort(key=lambda t: t._counter) - runner.runq.clear() - # 50% chance of reversing the batch, this way each task - # can appear before/after any other task. - if _r.random() < 0.5: - batch = batch[::-1] + _r.shuffle(batch) + else: + # 50% chance of reversing the batch, this way each task + # can appear before/after any other task. + if _r.random() < 0.5: + batch.reverse() while batch: task = batch.pop() GLOBAL_RUN_CONTEXT.task = task From a52d6a3cf081d6afa636c1391288649545c44867 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 8 Jun 2020 20:39:53 -0700 Subject: [PATCH 0243/1498] Move MockClock into _core. No functional changes. --- trio/_core/__init__.py | 2 + trio/{testing => _core}/_mock_clock.py | 0 trio/_core/tests/test_mock_clock.py | 155 +++++++++++++++++++++++++ trio/testing/__init__.py | 4 +- trio/tests/test_testing.py | 152 ------------------------ 5 files changed, 158 insertions(+), 155 deletions(-) rename trio/{testing => _core}/_mock_clock.py (100%) create mode 100644 trio/_core/tests/test_mock_clock.py diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index e22485149d..b2fc7d4300 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -71,6 +71,8 @@ from ._thread_cache import start_thread_soon +from ._mock_clock import MockClock + # Kqueue imports try: from ._run import current_kqueue, monitor_kevent, wait_kevent diff --git a/trio/testing/_mock_clock.py b/trio/_core/_mock_clock.py similarity index 100% rename from trio/testing/_mock_clock.py rename to trio/_core/_mock_clock.py diff --git a/trio/_core/tests/test_mock_clock.py b/trio/_core/tests/test_mock_clock.py new file mode 100644 index 0000000000..d8ab81d88b --- /dev/null +++ b/trio/_core/tests/test_mock_clock.py @@ -0,0 +1,155 @@ +from math import inf +import time + +import pytest + +from trio import sleep +from ... import _core +from .. import wait_all_tasks_blocked +from .._mock_clock import MockClock + +def test_mock_clock(): + REAL_NOW = 123.0 + c = MockClock() + c._real_clock = lambda: REAL_NOW + repr(c) # smoke test + assert c.rate == 0 + assert c.current_time() == 0 + c.jump(1.2) + assert c.current_time() == 1.2 + with pytest.raises(ValueError): + c.jump(-1) + assert c.current_time() == 1.2 + assert c.deadline_to_sleep_time(1.1) == 0 + assert c.deadline_to_sleep_time(1.2) == 0 + assert c.deadline_to_sleep_time(1.3) > 999999 + + with pytest.raises(ValueError): + c.rate = -1 + assert c.rate == 0 + + c.rate = 2 + assert c.current_time() == 1.2 + REAL_NOW += 1 + assert c.current_time() == 3.2 + assert c.deadline_to_sleep_time(3.1) == 0 + assert c.deadline_to_sleep_time(3.2) == 0 + assert c.deadline_to_sleep_time(4.2) == 0.5 + + c.rate = 0.5 + assert c.current_time() == 3.2 + assert c.deadline_to_sleep_time(3.1) == 0 + assert c.deadline_to_sleep_time(3.2) == 0 + assert c.deadline_to_sleep_time(4.2) == 2.0 + + c.jump(0.8) + assert c.current_time() == 4.0 + REAL_NOW += 1 + assert c.current_time() == 4.5 + + c2 = MockClock(rate=3) + assert c2.rate == 3 + assert c2.current_time() < 10 + + +async def test_mock_clock_autojump(mock_clock): + assert mock_clock.autojump_threshold == inf + + mock_clock.autojump_threshold = 0 + assert mock_clock.autojump_threshold == 0 + + real_start = time.perf_counter() + + virtual_start = _core.current_time() + for i in range(10): + print("sleeping {} seconds".format(10 * i)) + await sleep(10 * i) + print("woke up!") + assert virtual_start + 10 * i == _core.current_time() + virtual_start = _core.current_time() + + real_duration = time.perf_counter() - real_start + print("Slept {} seconds in {} seconds".format(10 * sum(range(10)), real_duration)) + assert real_duration < 1 + + mock_clock.autojump_threshold = 0.02 + t = _core.current_time() + # this should wake up before the autojump threshold triggers, so time + # shouldn't change + await wait_all_tasks_blocked() + assert t == _core.current_time() + # this should too + await wait_all_tasks_blocked(0.01) + assert t == _core.current_time() + + # This should wake up at the same time as the autojump_threshold, and + # confuse things. There is no deadline, so it shouldn't actually jump + # the clock. But does it handle the situation gracefully? + await wait_all_tasks_blocked(cushion=0.02, tiebreaker=float("inf")) + # And again with threshold=0, because that has some special + # busy-wait-avoidance logic: + mock_clock.autojump_threshold = 0 + await wait_all_tasks_blocked(tiebreaker=float("inf")) + + # set up a situation where the autojump task is blocked for a long long + # time, to make sure that cancel-and-adjust-threshold logic is working + mock_clock.autojump_threshold = 10000 + await wait_all_tasks_blocked() + mock_clock.autojump_threshold = 0 + # if the above line didn't take affect immediately, then this would be + # bad: + await sleep(100000) + + +async def test_mock_clock_autojump_interference(mock_clock): + mock_clock.autojump_threshold = 0.02 + + mock_clock2 = MockClock() + # messing with the autojump threshold of a clock that isn't actually + # installed in the run loop shouldn't do anything. + mock_clock2.autojump_threshold = 0.01 + + # if the autojump_threshold of 0.01 were in effect, then the next line + # would block forever, as the autojump task kept waking up to try to + # jump the clock. + await wait_all_tasks_blocked(0.015) + + # but the 0.02 limit does apply + await sleep(100000) + + +def test_mock_clock_autojump_preset(): + # Check that we can set the autojump_threshold before the clock is + # actually in use, and it gets picked up + mock_clock = MockClock(autojump_threshold=0.1) + mock_clock.autojump_threshold = 0.01 + real_start = time.perf_counter() + _core.run(sleep, 10000, clock=mock_clock) + assert time.perf_counter() - real_start < 1 + + +async def test_mock_clock_autojump_0_and_wait_all_tasks_blocked(mock_clock): + # Checks that autojump_threshold=0 doesn't interfere with + # calling wait_all_tasks_blocked with the default cushion=0 and arbitrary + # tiebreakers. + + mock_clock.autojump_threshold = 0 + + record = [] + + async def sleeper(): + await sleep(100) + record.append("yawn") + + async def waiter(): + for i in range(10): + await wait_all_tasks_blocked(tiebreaker=i) + record.append(i) + await sleep(1000) + record.append("waiter done") + + async with _core.open_nursery() as nursery: + nursery.start_soon(sleeper) + nursery.start_soon(waiter) + + assert record == list(range(10)) + ["yawn", "waiter done"] diff --git a/trio/testing/__init__.py b/trio/testing/__init__.py index 8c730ffeb5..aa15c4743e 100644 --- a/trio/testing/__init__.py +++ b/trio/testing/__init__.py @@ -1,9 +1,7 @@ -from .._core import wait_all_tasks_blocked +from .._core import wait_all_tasks_blocked, MockClock from ._trio_test import trio_test -from ._mock_clock import MockClock - from ._checkpoints import assert_checkpoints, assert_no_checkpoints from ._sequencer import Sequencer diff --git a/trio/tests/test_testing.py b/trio/tests/test_testing.py index 460200b28c..304f18cb88 100644 --- a/trio/tests/test_testing.py +++ b/trio/tests/test_testing.py @@ -1,7 +1,5 @@ # XX this should get broken up, like testing.py did -import time -from math import inf import tempfile import pytest @@ -260,156 +258,6 @@ async def child(i): ################################################################ -def test_mock_clock(): - REAL_NOW = 123.0 - c = MockClock() - c._real_clock = lambda: REAL_NOW - repr(c) # smoke test - assert c.rate == 0 - assert c.current_time() == 0 - c.jump(1.2) - assert c.current_time() == 1.2 - with pytest.raises(ValueError): - c.jump(-1) - assert c.current_time() == 1.2 - assert c.deadline_to_sleep_time(1.1) == 0 - assert c.deadline_to_sleep_time(1.2) == 0 - assert c.deadline_to_sleep_time(1.3) > 999999 - - with pytest.raises(ValueError): - c.rate = -1 - assert c.rate == 0 - - c.rate = 2 - assert c.current_time() == 1.2 - REAL_NOW += 1 - assert c.current_time() == 3.2 - assert c.deadline_to_sleep_time(3.1) == 0 - assert c.deadline_to_sleep_time(3.2) == 0 - assert c.deadline_to_sleep_time(4.2) == 0.5 - - c.rate = 0.5 - assert c.current_time() == 3.2 - assert c.deadline_to_sleep_time(3.1) == 0 - assert c.deadline_to_sleep_time(3.2) == 0 - assert c.deadline_to_sleep_time(4.2) == 2.0 - - c.jump(0.8) - assert c.current_time() == 4.0 - REAL_NOW += 1 - assert c.current_time() == 4.5 - - c2 = MockClock(rate=3) - assert c2.rate == 3 - assert c2.current_time() < 10 - - -async def test_mock_clock_autojump(mock_clock): - assert mock_clock.autojump_threshold == inf - - mock_clock.autojump_threshold = 0 - assert mock_clock.autojump_threshold == 0 - - real_start = time.perf_counter() - - virtual_start = _core.current_time() - for i in range(10): - print("sleeping {} seconds".format(10 * i)) - await sleep(10 * i) - print("woke up!") - assert virtual_start + 10 * i == _core.current_time() - virtual_start = _core.current_time() - - real_duration = time.perf_counter() - real_start - print("Slept {} seconds in {} seconds".format(10 * sum(range(10)), real_duration)) - assert real_duration < 1 - - mock_clock.autojump_threshold = 0.02 - t = _core.current_time() - # this should wake up before the autojump threshold triggers, so time - # shouldn't change - await wait_all_tasks_blocked() - assert t == _core.current_time() - # this should too - await wait_all_tasks_blocked(0.01) - assert t == _core.current_time() - - # This should wake up at the same time as the autojump_threshold, and - # confuse things. There is no deadline, so it shouldn't actually jump - # the clock. But does it handle the situation gracefully? - await wait_all_tasks_blocked(cushion=0.02, tiebreaker=float("inf")) - # And again with threshold=0, because that has some special - # busy-wait-avoidance logic: - mock_clock.autojump_threshold = 0 - await wait_all_tasks_blocked(tiebreaker=float("inf")) - - # set up a situation where the autojump task is blocked for a long long - # time, to make sure that cancel-and-adjust-threshold logic is working - mock_clock.autojump_threshold = 10000 - await wait_all_tasks_blocked() - mock_clock.autojump_threshold = 0 - # if the above line didn't take affect immediately, then this would be - # bad: - await sleep(100000) - - -async def test_mock_clock_autojump_interference(mock_clock): - mock_clock.autojump_threshold = 0.02 - - mock_clock2 = MockClock() - # messing with the autojump threshold of a clock that isn't actually - # installed in the run loop shouldn't do anything. - mock_clock2.autojump_threshold = 0.01 - - # if the autojump_threshold of 0.01 were in effect, then the next line - # would block forever, as the autojump task kept waking up to try to - # jump the clock. - await wait_all_tasks_blocked(0.015) - - # but the 0.02 limit does apply - await sleep(100000) - - -def test_mock_clock_autojump_preset(): - # Check that we can set the autojump_threshold before the clock is - # actually in use, and it gets picked up - mock_clock = MockClock(autojump_threshold=0.1) - mock_clock.autojump_threshold = 0.01 - real_start = time.perf_counter() - _core.run(sleep, 10000, clock=mock_clock) - assert time.perf_counter() - real_start < 1 - - -async def test_mock_clock_autojump_0_and_wait_all_tasks_blocked(mock_clock): - # Checks that autojump_threshold=0 doesn't interfere with - # calling wait_all_tasks_blocked with the default cushion=0 and arbitrary - # tiebreakers. - - mock_clock.autojump_threshold = 0 - - record = [] - - async def sleeper(): - await sleep(100) - record.append("yawn") - - async def waiter(): - for i in range(10): - await wait_all_tasks_blocked(tiebreaker=i) - record.append(i) - await sleep(1000) - record.append("waiter done") - - async with _core.open_nursery() as nursery: - nursery.start_soon(sleeper) - nursery.start_soon(waiter) - - assert record == list(range(10)) + ["yawn", "waiter done"] - - -################################################################ - - async def test__assert_raises(): with pytest.raises(AssertionError): with _assert_raises(RuntimeError): From 0513af1ab6473728e7727ee0c03b9efe6aada667 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 8 Jun 2020 21:17:13 -0700 Subject: [PATCH 0244/1498] Rewrite autojump functionality to avoid using a system task --- trio/_core/_mock_clock.py | 68 +++++++++++------------------ trio/_core/_run.py | 39 +++++++++++++---- trio/_core/tests/test_mock_clock.py | 1 + 3 files changed, 56 insertions(+), 52 deletions(-) diff --git a/trio/_core/_mock_clock.py b/trio/_core/_mock_clock.py index 843f51197f..e5931a05e2 100644 --- a/trio/_core/_mock_clock.py +++ b/trio/_core/_mock_clock.py @@ -2,6 +2,7 @@ from math import inf from .. import _core +from ._run import GLOBAL_RUN_CONTEXT from .._abc import Clock from .._util import SubclassingDeprecatedIn_v0_15_0 @@ -124,47 +125,29 @@ def autojump_threshold(self): @autojump_threshold.setter def autojump_threshold(self, new_autojump_threshold): self._autojump_threshold = float(new_autojump_threshold) - self._maybe_spawn_autojump_task() - if self._autojump_cancel_scope is not None: - # Task is running and currently blocked on the old setting, wake - # it up so it picks up the new setting - self._autojump_cancel_scope.cancel() - - async def _autojumper(self): - while True: - with _core.CancelScope() as cancel_scope: - self._autojump_cancel_scope = cancel_scope - try: - # If the autojump_threshold changes, then the setter does - # cancel_scope.cancel(), which causes the next line here - # to raise Cancelled, which is absorbed by the cancel - # scope above, and effectively just causes us to skip back - # to the start the loop, like a 'continue'. - await _core.wait_all_tasks_blocked(self._autojump_threshold, inf) - statistics = _core.current_statistics() - jump = statistics.seconds_to_next_deadline - if 0 < jump < inf: - self.jump(jump) - else: - # There are no deadlines, nothing is going to happen - # until some actual I/O arrives (or maybe another - # wait_all_tasks_blocked task wakes up). That's fine, - # but if our threshold is zero then this will become a - # busy-wait -- so insert a small-but-non-zero _sleep to - # avoid that. - if self._autojump_threshold == 0: - await _core.wait_all_tasks_blocked(0.01) - finally: - self._autojump_cancel_scope = None - - def _maybe_spawn_autojump_task(self): - if self._autojump_threshold < inf and self._autojump_task is None: - try: - clock = _core.current_clock() - except RuntimeError: - return - if clock is self: - self._autojump_task = _core.spawn_system_task(self._autojumper) + self._try_resync_autojump_threshold() + + # runner.clock_autojump_threshold is an internal API that isn't easily + # usable by custom third-party Clock objects. If you need access to this + # functionality, let us know, and we'll figure out how to make a public + # API. Discussion: + # + # https://github.com/python-trio/trio/issues/1587 + def _try_resync_autojump_threshold(self): + try: + runner = GLOBAL_RUN_CONTEXT.runner + except AttributeError: + pass + else: + runner.clock_autojump_threshold = self._autojump_threshold + + # Invoked by the run loop when runner.clock_autojump_threshold is + # exceeded. + def _autojump(self): + statistics = _core.current_statistics() + jump = statistics.seconds_to_next_deadline + if 0 < jump < inf: + self.jump(jump) def _real_to_virtual(self, real): real_offset = real - self._real_base @@ -172,8 +155,7 @@ def _real_to_virtual(self, real): return self._virtual_base + virtual_offset def start_clock(self): - token = _core.current_trio_token() - token.run_sync_soon(self._maybe_spawn_autojump_task) + self._try_resync_autojump_threshold() def current_time(self): return self._real_to_virtual(self._real_clock()) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 465aebe6f5..1441361dd4 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -12,6 +12,7 @@ import collections.abc from contextlib import contextmanager, closing import warnings +import enum from contextvars import copy_context from math import inf @@ -112,6 +113,11 @@ def deadline_to_sleep_time(self, deadline): return deadline - self.current_time() +class IdlePrimedTypes(enum.Enum): + WAITING_FOR_IDLE = 1 + AUTOJUMP_CLOCK = 2 + + ################################################################ # CancelScope and friends ################################################################ @@ -1209,6 +1215,9 @@ class Runner: entry_queue = attr.ib(factory=EntryQueue) trio_token = attr.ib(default=None) + # If everything goes idle for this long, we call clock._autojump() + clock_autojump_threshold = attr.ib(default=inf) + # Guest mode stuff is_guest = attr.ib(default=False) guest_tick_scheduled = attr.ib(default=False) @@ -2005,6 +2014,14 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): if cushion < timeout: timeout = cushion idle_primed = True + idle_prime_type = IdlePrimedTypes.WAITING_FOR_IDLE + # We use 'elif' here because if there are tasks in + # wait_all_tasks_blocked, then those tasks will wake up without + # jumping the clock, so we don't need to autojump. + elif runner.clock_autojump_threshold < timeout: + timeout = runner.clock_autojump_threshold + idle_primed = True + idle_prime_type = IdlePrimedTypes.AUTOJUMP_CLOCK if runner.instruments: runner.instrument("before_io_wait", timeout) @@ -2030,8 +2047,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # idle_primed=True means: if the IO wait hit the timeout, and still # nothing is happening, then we should start waking up - # wait_all_tasks_blocked tasks. But there are some subtleties in - # defining "nothing is happening". + # wait_all_tasks_blocked tasks or autojump the clock. But there + # are some subtleties in defining "nothing is happening". # # 'not runner.runq' means that no tasks are currently runnable. # 'not events' means that the last IO wait call hit its full @@ -2050,13 +2067,17 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # # So we need to check both. if idle_primed and not runner.runq and not events: - while runner.waiting_for_idle: - key, task = runner.waiting_for_idle.peekitem(0) - if key[:2] == (cushion, tiebreaker): - del runner.waiting_for_idle[key] - runner.reschedule(task) - else: - break + if idle_prime_type is IdlePrimedTypes.WAITING_FOR_IDLE: + while runner.waiting_for_idle: + key, task = runner.waiting_for_idle.peekitem(0) + if key[:2] == (cushion, tiebreaker): + del runner.waiting_for_idle[key] + runner.reschedule(task) + else: + break + else: + assert idle_prime_type is IdlePrimedTypes.AUTOJUMP_CLOCK + runner.clock._autojump() # Process all runnable tasks, but only the ones that are already # runnable now. Anything that becomes runnable during this cycle diff --git a/trio/_core/tests/test_mock_clock.py b/trio/_core/tests/test_mock_clock.py index d8ab81d88b..2473fbfaee 100644 --- a/trio/_core/tests/test_mock_clock.py +++ b/trio/_core/tests/test_mock_clock.py @@ -8,6 +8,7 @@ from .. import wait_all_tasks_blocked from .._mock_clock import MockClock + def test_mock_clock(): REAL_NOW = 123.0 c = MockClock() From 70f25434703a4ffc114138b4acb76953bc0e68c2 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 8 Jun 2020 21:39:57 -0700 Subject: [PATCH 0245/1498] Update docs to account for the new autojump clock implementation --- newsfragments/1587.feature.rst | 7 +++++++ trio/_core/_mock_clock.py | 33 +++++++-------------------------- 2 files changed, 14 insertions(+), 26 deletions(-) create mode 100644 newsfragments/1587.feature.rst diff --git a/newsfragments/1587.feature.rst b/newsfragments/1587.feature.rst new file mode 100644 index 0000000000..856e85a933 --- /dev/null +++ b/newsfragments/1587.feature.rst @@ -0,0 +1,7 @@ +We refactored `trio.testing.MockClock` so that it no longer needs to +run an internal task to manage autojumping. This should be mostly +invisible to users, but there is one semantic change: the interaction +between `trio.testing.wait_all_tasks_blocked` and the autojump clock +was fixed. Now, the autojump will always wait until after all +`wait_all_tasks_blocked` calls have finished before firing, instead of +it depending on which threshold values you passed. diff --git a/trio/_core/_mock_clock.py b/trio/_core/_mock_clock.py index e5931a05e2..56d66388ff 100644 --- a/trio/_core/_mock_clock.py +++ b/trio/_core/_mock_clock.py @@ -52,32 +52,13 @@ class MockClock(Clock, metaclass=SubclassingDeprecatedIn_v0_15_0): above) then just set it to zero, and the clock will jump whenever all tasks are blocked. - .. warning:: - - If you're using :func:`wait_all_tasks_blocked` and - :attr:`autojump_threshold` together, then you have to be - careful. Setting :attr:`autojump_threshold` acts like a background - task calling:: - - while True: - await wait_all_tasks_blocked( - cushion=clock.autojump_threshold, tiebreaker=float("inf")) - - This means that if you call :func:`wait_all_tasks_blocked` with a - cushion *larger* than your autojump threshold, then your call to - :func:`wait_all_tasks_blocked` will never return, because the - autojump task will keep waking up before your task does, and each - time it does it'll reset your task's timer. However, if your cushion - and the autojump threshold are the *same*, then the autojump's - tiebreaker will prevent them from interfering (unless you also set - your tiebreaker to infinity for some reason. Don't do that). As an - important special case: this means that if you set an autojump - threshold of zero and use :func:`wait_all_tasks_blocked` with the - default zero cushion, then everything will work fine. - - **Summary**: you should set :attr:`autojump_threshold` to be at - least as large as the largest cushion you plan to pass to - :func:`wait_all_tasks_blocked`. + .. note:: If you use ``autojump_threshold`` and + `wait_all_tasks_blocked` at the same time, then you might wonder how + they interact, since they both cause things to happen after the run + loop goes idle for some time. The answer is: + `wait_all_tasks_blocked` takes priority. If there's a task blocked + in `wait_all_tasks_blocked`, then the autojump feature treats that + as active task and does *not* jump the clock. """ From 80089eb878407e18013243a0407c4f1bf1a3787e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 8 Jun 2020 21:46:42 -0700 Subject: [PATCH 0246/1498] fix guest-mode/autojump-clock interaction --- trio/_core/_mock_clock.py | 2 ++ trio/_core/tests/test_guest_mode.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/trio/_core/_mock_clock.py b/trio/_core/_mock_clock.py index 56d66388ff..467707c5f4 100644 --- a/trio/_core/_mock_clock.py +++ b/trio/_core/_mock_clock.py @@ -117,6 +117,8 @@ def autojump_threshold(self, new_autojump_threshold): def _try_resync_autojump_threshold(self): try: runner = GLOBAL_RUN_CONTEXT.runner + if runner.is_guest: + runner.force_guest_tick_asap() except AttributeError: pass else: diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 46e741e392..a5490c99bb 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -7,6 +7,7 @@ import signal import socket import threading +import time import trio import trio.testing @@ -473,3 +474,24 @@ async def trio_main_raising(in_host): assert excinfo.value.__context__ is final_exc assert signal.getsignal(signal.SIGINT) is signal.default_int_handler + + +def test_guest_mode_autojump_clock_threshold_changing(): + # This is super obscure and probably no-one will ever notice, but + # technically mutating the MockClock.autojump_threshold from the host + # should wake up the guest, so let's test it. + + clock = trio.testing.MockClock() + + DURATION = 9999999 + + async def trio_main(in_host): + assert trio.current_time() == 0 + in_host(lambda: setattr(clock, "autojump_threshold", 0)) + await trio.sleep(DURATION) + assert trio.current_time() == DURATION + + start = time.monotonic() + trivial_guest_run(trio_main, clock=clock) + end = time.monotonic() + assert end - start < DURATION / 10 From 21770735f91dccda9e85feea06a67cd585d8fabb Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 8 Jun 2020 21:54:57 -0700 Subject: [PATCH 0247/1498] Add test for new autojump + wait_all_tasks_blocked semantics --- trio/_core/tests/test_mock_clock.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/test_mock_clock.py b/trio/_core/tests/test_mock_clock.py index 2473fbfaee..aef4cef498 100644 --- a/trio/_core/tests/test_mock_clock.py +++ b/trio/_core/tests/test_mock_clock.py @@ -7,6 +7,7 @@ from ... import _core from .. import wait_all_tasks_blocked from .._mock_clock import MockClock +from .tutil import slow def test_mock_clock(): @@ -129,7 +130,7 @@ def test_mock_clock_autojump_preset(): assert time.perf_counter() - real_start < 1 -async def test_mock_clock_autojump_0_and_wait_all_tasks_blocked(mock_clock): +async def test_mock_clock_autojump_0_and_wait_all_tasks_blocked_0(mock_clock): # Checks that autojump_threshold=0 doesn't interfere with # calling wait_all_tasks_blocked with the default cushion=0 and arbitrary # tiebreakers. @@ -154,3 +155,27 @@ async def waiter(): nursery.start_soon(waiter) assert record == list(range(10)) + ["yawn", "waiter done"] + + +@slow +async def test_mock_clock_autojump_0_and_wait_all_tasks_blocked_nonzero(mock_clock): + # Checks that autojump_threshold=0 doesn't interfere with + # calling wait_all_tasks_blocked with a non-zero cushion. + + mock_clock.autojump_threshold = 0 + + record = [] + + async def sleeper(): + await sleep(100) + record.append("yawn") + + async def waiter(): + await wait_all_tasks_blocked(1) + record.append("waiter done") + + async with _core.open_nursery() as nursery: + nursery.start_soon(sleeper) + nursery.start_soon(waiter) + + assert record == ["waiter done", "yawn"] From 9ed488e6269b75f7978f2e15b66f6bded14afff7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2020 06:23:11 +0000 Subject: [PATCH 0248/1498] Bump astroid from 2.4.1 to 2.4.2 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.4.1 to 2.4.2. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/astroid-2.4.1...astroid-2.4.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8bb4e4304b..0149468083 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.4.1 # via pylint +astroid==2.4.2 # via pylint async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, black, outcome, pytest backcall==0.1.0 # via ipython From 391694c83fc105d5d73bd5eb8b419b91fd26d9a8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2020 06:24:23 +0000 Subject: [PATCH 0249/1498] Bump flake8 from 3.8.2 to 3.8.3 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.8.2 to 3.8.3. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.8.2...3.8.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8bb4e4304b..596d2bf280 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -16,7 +16,7 @@ click==7.1.2 # via black coverage==5.1 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets -flake8==3.8.2 # via -r test-requirements.in +flake8==3.8.3 # via -r test-requirements.in idna==2.9 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets From 6066ffbf62e67aacfd8949bb93241aec4afc6ad1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2020 06:40:33 +0000 Subject: [PATCH 0250/1498] Bump pylint from 2.5.2 to 2.5.3 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.5.2 to 2.5.3. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.5.2...pylint-2.5.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 0149468083..3963cca07f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -40,7 +40,7 @@ pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 pygments==2.6.1 # via ipython -pylint==2.5.2 # via -r test-requirements.in +pylint==2.5.3 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.9.0 # via -r test-requirements.in From 1de6f81a85d1cb77049b4a3f080a91e37b124d63 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 4 Jun 2020 10:20:50 +0400 Subject: [PATCH 0251/1498] Remove deprecated BlockingTrioPortal --- trio/__init__.py | 5 ----- trio/_threads.py | 19 +----------------- trio/tests/test_threads.py | 40 +++----------------------------------- 3 files changed, 4 insertions(+), 60 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index 5339d107eb..50734c71ae 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -54,8 +54,6 @@ Condition, ) -from ._threads import BlockingTrioPortal as _BlockingTrioPortal - from ._highlevel_generic import aclose_forcefully, StapledStream from ._channel import ( @@ -134,9 +132,6 @@ "current_default_worker_thread_limiter": _deprecate.DeprecatedAttribute( to_thread.current_default_thread_limiter, "0.12.0", issue=810, ), - "BlockingTrioPortal": _deprecate.DeprecatedAttribute( - _BlockingTrioPortal, "0.12.0", issue=810, instead=from_thread, - ), # NOTE: when you remove this, you should also remove the file # trio/hazmat.py. For details on why we have both, see: # diff --git a/trio/_threads.py b/trio/_threads.py index 8867388bd7..62c22041f0 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -21,20 +21,6 @@ # Global due to Threading API, thread local storage for trio token TOKEN_LOCAL = threading.local() - -class BlockingTrioPortal: - def __init__(self, trio_token=None): - if trio_token is None: - trio_token = trio.lowlevel.current_trio_token() - self._trio_token = trio_token - - def run(self, afn, *args): - return from_thread_run(afn, *args, trio_token=self._trio_token) - - def run_sync(self, fn, *args): - return from_thread_run_sync(fn, *args, trio_token=self._trio_token) - - _limiter_local = RunVar("limiter") # I pulled this number out of the air; it isn't based on anything. Probably we # should make some kind of measurements to pick a good value. @@ -227,10 +213,7 @@ def _run_fn_as_system_task(cb, fn, *args, trio_token=None): "this thread wasn't created by Trio, pass kwarg trio_token=..." ) - # TODO: This is only necessary for compatibility with BlockingTrioPortal. - # once that is deprecated, this check should no longer be necessary because - # thread local storage (or the absence of) is sufficient to check if trio - # is running in a thread or not. + # Avoid deadlock by making sure we're not called from Trio thread try: trio.lowlevel.current_task() except RuntimeError: diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index a9059fef79..1cb3956def 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -12,7 +12,6 @@ current_default_thread_limiter, from_thread_run, from_thread_run_sync, - BlockingTrioPortal, ) from .._core.tests.test_ki import ki_self @@ -539,40 +538,7 @@ def test_run_fn_as_system_task_catched_badly_typed_token(): from_thread_run_sync(_core.current_time, trio_token="Not TrioTokentype") -async def test_do_in_trio_thread_from_trio_thread_legacy(): - # This check specifically confirms that a RuntimeError will be raised if - # the old BlockingTrIoPortal API calls into a trio loop while already - # running inside of one. - portal = BlockingTrioPortal() - +async def test_from_thread_inside_trio_thread(): + trio_token = _core.current_trio_token() with pytest.raises(RuntimeError): - portal.run_sync(lambda: None) # pragma: no branch - - async def foo(): # pragma: no cover - pass - - with pytest.raises(RuntimeError): - portal.run(foo) - - -async def test_BlockingTrioPortal_with_explicit_TrioToken(): - # This tests the deprecated BlockingTrioPortal with a token passed in to - # confirm that both methods of making a portal are supported by - # trio.from_thread - token = _core.current_trio_token() - - def worker_thread(token): - with pytest.raises(RuntimeError): - BlockingTrioPortal() - portal = BlockingTrioPortal(token) - return portal.run_sync(threading.current_thread) - - t = await to_thread_run_sync(worker_thread, token) - assert t == threading.current_thread() - - -def test_BlockingTrioPortal_deprecated_export(recwarn): - import trio - - btp = trio.BlockingTrioPortal - assert btp is BlockingTrioPortal + from_thread_run_sync(lambda: None, trio_token=trio_token) From ddf2483bb831f83327c28dbc09932dcd8c7d67ae Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 4 Jun 2020 21:51:28 +0400 Subject: [PATCH 0252/1498] Exclude deprecated exports checks from coverage We don't necessarily always have deprecated exports. --- trio/tests/test_exports.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 1d49f4765d..510a0987b2 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -33,7 +33,8 @@ def test_core_is_properly_reexported(): def public_modules(module): yield module for name, class_ in module.__dict__.items(): - if name.startswith("_"): + # Deprecated classes are exported with a leading underscore + if name.startswith("_"): # pragma: nocover continue if not isinstance(class_, types.ModuleType): continue @@ -118,7 +119,8 @@ def test_classes_are_final(): for name, class_ in module.__dict__.items(): if not isinstance(class_, type): continue - if name.startswith("_"): + # Deprecated classes are exported with a leading underscore + if name.startswith("_"): # pragma: nocover continue # Abstract classes can be subclassed, because that's the whole From 781531feca222ffe82f11a4fc41811c367a690cd Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 9 Jun 2020 12:01:52 +0400 Subject: [PATCH 0253/1498] Fix coverage by not passing lambda Otherwise, coverage complains that we did not run the lambda. --- trio/tests/test_threads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 1cb3956def..d7f337e6f6 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -541,4 +541,4 @@ def test_run_fn_as_system_task_catched_badly_typed_token(): async def test_from_thread_inside_trio_thread(): trio_token = _core.current_trio_token() with pytest.raises(RuntimeError): - from_thread_run_sync(lambda: None, trio_token=trio_token) + from_thread_run_sync(fn=None, trio_token=trio_token) From f5a219131d8ba61cea6b4d028576a81349d8f5af Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 9 Jun 2020 12:07:29 +0400 Subject: [PATCH 0254/1498] Add newsfragment --- newsfragments/1574.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1574.removal.rst diff --git a/newsfragments/1574.removal.rst b/newsfragments/1574.removal.rst new file mode 100644 index 0000000000..175e607319 --- /dev/null +++ b/newsfragments/1574.removal.rst @@ -0,0 +1 @@ +Remove ``BlockingTrioPortal``: it was deprecated in 0.12.0. From e749a883a80e87564a414dcb2d55c836f8af9c15 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 01:29:01 -0700 Subject: [PATCH 0255/1498] Minor overhaul of our newsfragment categories In particular, splits features into "regular features" and "headline features", and split "removal" into "breaking changes without deprecations" and "deprecations". --- newsfragments/{399.feature.rst => 399.headline.rst} | 0 newsfragments/README.rst | 12 +++++++++--- pyproject.toml | 13 ++++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) rename newsfragments/{399.feature.rst => 399.headline.rst} (100%) diff --git a/newsfragments/399.feature.rst b/newsfragments/399.headline.rst similarity index 100% rename from newsfragments/399.feature.rst rename to newsfragments/399.headline.rst diff --git a/newsfragments/README.rst b/newsfragments/README.rst index d01e930f24..951998f5ba 100644 --- a/newsfragments/README.rst +++ b/newsfragments/README.rst @@ -8,13 +8,19 @@ relevant to people working on the code itself.) Each file should be named like ``..rst``, where ```` is an issue numbers, and ```` is one of: -* ``feature`` +* ``headline``: a major new feature we want to highlight for users +* ``breaking``: any breaking changes that happen without a proper + deprecation period (note: deprecations, and removal of previously + deprecated features after an appropriate time, go in the + ``deprecated`` category instead) +* ``feature``: any new feature that doesn't qualify for ``headline`` * ``bugfix`` * ``doc`` -* ``removal`` +* ``deprecated`` * ``misc`` -So for example: ``123.feature.rst``, ``456.bugfix.rst`` +So for example: ``123.headline.rst``, ``456.bugfix.rst``, +``789.deprecated.rst`` If your PR fixes an issue, use that number here. If there is no issue, then after you submit the PR and get the PR number you can add a diff --git a/pyproject.toml b/pyproject.toml index e16fa5c401..c23b1f8933 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,15 @@ directory = "newsfragments" underlines = ["-", "~", "^"] issue_format = "`#{issue} `__" -# Unfortunately there's no way to simply override -# tool.towncrier.type.misc.showcontent +[[tool.towncrier.type]] +directory = "headline" +name = "Headline features" +showcontent = true + +[[tool.towncrier.type]] +directory = "breakin" +name = "Breaking changes" +showcontent = true [[tool.towncrier.type]] directory = "feature" @@ -34,7 +41,7 @@ name = "Improved Documentation" showcontent = true [[tool.towncrier.type]] -directory = "removal" +directory = "deprecated" name = "Deprecations and Removals" showcontent = true From de2ba3cc3212c6c371562249526d1031f0a049eb Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 01:31:23 -0700 Subject: [PATCH 0256/1498] Make this a misc newsfragments instead of a feature --- newsfragments/{1587.feature.rst => 1587.misc.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{1587.feature.rst => 1587.misc.rst} (100%) diff --git a/newsfragments/1587.feature.rst b/newsfragments/1587.misc.rst similarity index 100% rename from newsfragments/1587.feature.rst rename to newsfragments/1587.misc.rst From 486932fff23fb1cab1def66ea34472dfe2b29288 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 01:31:47 -0700 Subject: [PATCH 0257/1498] Fix bad sphinx reference in newfragment --- newsfragments/1587.misc.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newsfragments/1587.misc.rst b/newsfragments/1587.misc.rst index 856e85a933..8152340ba2 100644 --- a/newsfragments/1587.misc.rst +++ b/newsfragments/1587.misc.rst @@ -3,5 +3,5 @@ run an internal task to manage autojumping. This should be mostly invisible to users, but there is one semantic change: the interaction between `trio.testing.wait_all_tasks_blocked` and the autojump clock was fixed. Now, the autojump will always wait until after all -`wait_all_tasks_blocked` calls have finished before firing, instead of -it depending on which threshold values you passed. +`~trio.testing.wait_all_tasks_blocked` calls have finished before +firing, instead of it depending on which threshold values you passed. From 6866ae048531b96cfaa29307b35222384b25a2e6 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 9 Jun 2020 13:24:33 +0400 Subject: [PATCH 0258/1498] Avoid passing None function for future mypy checks --- trio/tests/test_threads.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index d7f337e6f6..8b95eff030 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -539,6 +539,9 @@ def test_run_fn_as_system_task_catched_badly_typed_token(): async def test_from_thread_inside_trio_thread(): + def not_called(): # pragma: no cover + assert False + trio_token = _core.current_trio_token() with pytest.raises(RuntimeError): - from_thread_run_sync(fn=None, trio_token=trio_token) + from_thread_run_sync(not_called, trio_token=trio_token) From b362f069fe19bacd6020288b047ac734c3290842 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 02:33:23 -0700 Subject: [PATCH 0259/1498] fix tyop --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c23b1f8933..a29eab8560 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ name = "Headline features" showcontent = true [[tool.towncrier.type]] -directory = "breakin" +directory = "breaking" name = "Breaking changes" showcontent = true From cf012e3255cf9e46a8f4fe8e647921b8b7e4d929 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 9 Jun 2020 15:37:13 +0400 Subject: [PATCH 0260/1498] Remove deprecated ssl/subprocess reexports --- trio/__init__.py | 26 ---- trio/_deprecated_ssl_reexports.py | 157 ----------------------- trio/_deprecated_subprocess_reexports.py | 45 ------- 3 files changed, 228 deletions(-) delete mode 100644 trio/_deprecated_ssl_reexports.py delete mode 100644 trio/_deprecated_subprocess_reexports.py diff --git a/trio/__init__.py b/trio/__init__.py index 50734c71ae..2e8b440316 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -102,30 +102,8 @@ if False: from . import testing -from . import _deprecated_ssl_reexports -from . import _deprecated_subprocess_reexports - _deprecate.enable_attribute_deprecations(__name__) __deprecated_attributes__ = { - "ssl": _deprecate.DeprecatedAttribute( - _deprecated_ssl_reexports, - "0.11.0", - issue=852, - instead=( - "trio.SSLStream, trio.SSLListener, trio.NeedHandshakeError, " - "and the standard library 'ssl' module (minus SSLSocket and " - "wrap_socket())" - ), - ), - "subprocess": _deprecate.DeprecatedAttribute( - _deprecated_subprocess_reexports, - "0.11.0", - issue=852, - instead=( - "trio.Process and the constants in the standard " - "library 'subprocess' module" - ), - ), "run_sync_in_worker_thread": _deprecate.DeprecatedAttribute( to_thread.run_sync, "0.12.0", issue=810, ), @@ -171,8 +149,4 @@ fixup_module_metadata(abc.__name__, abc.__dict__) fixup_module_metadata(from_thread.__name__, from_thread.__dict__) fixup_module_metadata(to_thread.__name__, to_thread.__dict__) -fixup_module_metadata(__name__ + ".ssl", _deprecated_ssl_reexports.__dict__) -fixup_module_metadata( - __name__ + ".subprocess", _deprecated_subprocess_reexports.__dict__ -) del fixup_module_metadata diff --git a/trio/_deprecated_ssl_reexports.py b/trio/_deprecated_ssl_reexports.py deleted file mode 100644 index 31d5fdf9f6..0000000000 --- a/trio/_deprecated_ssl_reexports.py +++ /dev/null @@ -1,157 +0,0 @@ -# This is a public namespace, so we don't want to expose any non-underscored -# attributes that aren't actually part of our public API. But it's very -# annoying to carefully always use underscored names for module-level -# temporaries, imports, etc. when implementing the module. So we put the -# implementation in an underscored module, and then re-export the public parts -# here. - -# Trio-specific symbols: -from ._ssl import SSLStream, SSLListener, NeedHandshakeError - -# Symbols re-exported from the stdlib ssl module: - -# Always available -from ssl import ( - cert_time_to_seconds, - CertificateError, - create_default_context, - DER_cert_to_PEM_cert, - get_default_verify_paths, - match_hostname, - PEM_cert_to_DER_cert, - Purpose, - SSLEOFError, - SSLError, - SSLSyscallError, - SSLZeroReturnError, - AlertDescription, - SSLErrorNumber, - SSLSession, - VerifyFlags, - VerifyMode, - Options, -) - -# Added in python 3.7 -try: - from ssl import SSLCertVerificationError, TLSVersion # noqa -except ImportError: - pass - -# Windows-only -try: - from ssl import enum_certificates, enum_crls # noqa -except ImportError: - pass - -# Fake import to enable static analysis tools to catch the names -# (Real import is below) -try: - from ssl import ( - AF_INET, - ALERT_DESCRIPTION_ACCESS_DENIED, - ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE, - ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE, - ALERT_DESCRIPTION_BAD_CERTIFICATE, - ALERT_DESCRIPTION_BAD_RECORD_MAC, - ALERT_DESCRIPTION_CERTIFICATE_EXPIRED, - ALERT_DESCRIPTION_CERTIFICATE_REVOKED, - ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN, - ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE, - ALERT_DESCRIPTION_CLOSE_NOTIFY, - ALERT_DESCRIPTION_DECODE_ERROR, - ALERT_DESCRIPTION_DECOMPRESSION_FAILURE, - ALERT_DESCRIPTION_DECRYPT_ERROR, - ALERT_DESCRIPTION_HANDSHAKE_FAILURE, - ALERT_DESCRIPTION_ILLEGAL_PARAMETER, - ALERT_DESCRIPTION_INSUFFICIENT_SECURITY, - ALERT_DESCRIPTION_INTERNAL_ERROR, - ALERT_DESCRIPTION_NO_RENEGOTIATION, - ALERT_DESCRIPTION_PROTOCOL_VERSION, - ALERT_DESCRIPTION_RECORD_OVERFLOW, - ALERT_DESCRIPTION_UNEXPECTED_MESSAGE, - ALERT_DESCRIPTION_UNKNOWN_CA, - ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY, - ALERT_DESCRIPTION_UNRECOGNIZED_NAME, - ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE, - ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION, - ALERT_DESCRIPTION_USER_CANCELLED, - CERT_NONE, - CERT_OPTIONAL, - CERT_REQUIRED, - CHANNEL_BINDING_TYPES, - HAS_ALPN, - HAS_ECDH, - HAS_NEVER_CHECK_COMMON_NAME, - HAS_NPN, - HAS_SNI, - OP_ALL, - OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION, - OP_COOKIE_EXCHANGE, - OP_DONT_INSERT_EMPTY_FRAGMENTS, - OP_EPHEMERAL_RSA, - OP_LEGACY_SERVER_CONNECT, - OP_MICROSOFT_BIG_SSLV3_BUFFER, - OP_MICROSOFT_SESS_ID_BUG, - OP_MSIE_SSLV2_RSA_PADDING, - OP_NETSCAPE_CA_DN_BUG, - OP_NETSCAPE_CHALLENGE_BUG, - OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG, - OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG, - OP_NO_QUERY_MTU, - OP_PKCS1_CHECK_1, - OP_PKCS1_CHECK_2, - OP_SSLEAY_080_CLIENT_DH_BUG, - OP_SSLREF2_REUSE_CERT_TYPE_BUG, - OP_TLS_BLOCK_PADDING_BUG, - OP_TLS_D5_BUG, - OP_TLS_ROLLBACK_BUG, - SSL_ERROR_NONE, - SSL_ERROR_NO_SOCKET, - OP_CIPHER_SERVER_PREFERENCE, - OP_NO_COMPRESSION, - OP_NO_RENEGOTIATION, - OP_NO_TICKET, - OP_SINGLE_DH_USE, - OP_SINGLE_ECDH_USE, - OPENSSL_VERSION_INFO, - OPENSSL_VERSION_NUMBER, - OPENSSL_VERSION, - PEM_FOOTER, - PEM_HEADER, - PROTOCOL_TLS_CLIENT, - PROTOCOL_TLS_SERVER, - PROTOCOL_TLS, - SO_TYPE, - SOCK_STREAM, - SOL_SOCKET, - SSL_ERROR_EOF, - SSL_ERROR_INVALID_ERROR_CODE, - SSL_ERROR_SSL, - SSL_ERROR_SYSCALL, - SSL_ERROR_WANT_CONNECT, - SSL_ERROR_WANT_READ, - SSL_ERROR_WANT_WRITE, - SSL_ERROR_WANT_X509_LOOKUP, - SSL_ERROR_ZERO_RETURN, - VERIFY_CRL_CHECK_CHAIN, - VERIFY_CRL_CHECK_LEAF, - VERIFY_DEFAULT, - VERIFY_X509_STRICT, - VERIFY_X509_TRUSTED_FIRST, - OP_ENABLE_MIDDLEBOX_COMPAT, - ) -except ImportError: - pass - -# Dynamically re-export whatever constants this particular Python happens to -# have: -import ssl as _stdlib_ssl - -globals().update( - { - _name: getattr(_stdlib_ssl, _name) - for _name in _stdlib_ssl.__dict__.keys() - if _name.isupper() and not _name.startswith("_") - } -) diff --git a/trio/_deprecated_subprocess_reexports.py b/trio/_deprecated_subprocess_reexports.py deleted file mode 100644 index 2d1e4eed25..0000000000 --- a/trio/_deprecated_subprocess_reexports.py +++ /dev/null @@ -1,45 +0,0 @@ -from ._subprocess import Process - -# Reexport constants and exceptions from the stdlib subprocess module -from subprocess import ( - PIPE, - STDOUT, - DEVNULL, - CalledProcessError, - SubprocessError, - TimeoutExpired, - CompletedProcess, -) - -# Windows only -try: - from subprocess import ( - STARTUPINFO, - STD_INPUT_HANDLE, - STD_OUTPUT_HANDLE, - STD_ERROR_HANDLE, - SW_HIDE, - STARTF_USESTDHANDLES, - STARTF_USESHOWWINDOW, - CREATE_NEW_CONSOLE, - CREATE_NEW_PROCESS_GROUP, - ) -except ImportError: - pass - -# Windows 3.7+ only -try: - from subprocess import ( - ABOVE_NORMAL_PRIORITY_CLASS, - BELOW_NORMAL_PRIORITY_CLASS, - HIGH_PRIORITY_CLASS, - IDLE_PRIORITY_CLASS, - NORMAL_PRIORITY_CLASS, - REALTIME_PRIORITY_CLASS, - CREATE_NO_WINDOW, - DETACHED_PROCESS, - CREATE_DEFAULT_ERROR_MODE, - CREATE_BREAKAWAY_FROM_JOB, - ) -except ImportError: - pass From 31c933647747e2771326f594d392edcb1524e265 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 12:50:05 -0700 Subject: [PATCH 0261/1498] Consistent capitalization --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a29eab8560..8597a4f545 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,12 +37,12 @@ showcontent = true [[tool.towncrier.type]] directory = "doc" -name = "Improved Documentation" +name = "Improved documentation" showcontent = true [[tool.towncrier.type]] directory = "deprecated" -name = "Deprecations and Removals" +name = "Deprecations and removals" showcontent = true [[tool.towncrier.type]] From f52c0d2d77a83899c15045b31863fd7c398018bb Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 12:51:49 -0700 Subject: [PATCH 0262/1498] Rename newsfragment for 1574 to use new style --- newsfragments/{1574.removal.rst => 1574.deprecated.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{1574.removal.rst => 1574.deprecated.rst} (100%) diff --git a/newsfragments/1574.removal.rst b/newsfragments/1574.deprecated.rst similarity index 100% rename from newsfragments/1574.removal.rst rename to newsfragments/1574.deprecated.rst From 4d58d7417fe3658230f61001f42092e8e70e7cf3 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 12:56:58 -0700 Subject: [PATCH 0263/1498] Update instructions on how to write newsfragments --- newsfragments/README.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/newsfragments/README.rst b/newsfragments/README.rst index 951998f5ba..51dc92290b 100644 --- a/newsfragments/README.rst +++ b/newsfragments/README.rst @@ -26,8 +26,10 @@ If your PR fixes an issue, use that number here. If there is no issue, then after you submit the PR and get the PR number you can add a newsfragment using that instead. -Note that the ``towncrier`` tool will automatically -reflow your text, so don't try to do any fancy formatting. You can -install ``towncrier`` and then run ``towncrier --draft`` if you want -to get a preview of how your change will look in the final release -notes. +Your text can use all the same markup that we use in our Sphinx docs. +For example, you can use double-backticks to mark code snippets, or +single-backticks to link to a function/class/module. + +To check how your formatting looks, the easiest way is to make the PR, +and then after the CI checks run, click on the "Read the Docs build" +details link, and navigate to the release history page. From e9994e15b0dd8b1cf457cb22f4dacd0316236c4b Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 10 Jun 2020 00:23:40 +0400 Subject: [PATCH 0264/1498] Remove remaining aliases deprecated in 0.12.0 --- trio/__init__.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index 2e8b440316..1a17c5ee67 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -104,12 +104,6 @@ _deprecate.enable_attribute_deprecations(__name__) __deprecated_attributes__ = { - "run_sync_in_worker_thread": _deprecate.DeprecatedAttribute( - to_thread.run_sync, "0.12.0", issue=810, - ), - "current_default_worker_thread_limiter": _deprecate.DeprecatedAttribute( - to_thread.current_default_thread_limiter, "0.12.0", issue=810, - ), # NOTE: when you remove this, you should also remove the file # trio/hazmat.py. For details on why we have both, see: # @@ -119,21 +113,6 @@ ), } -_deprecate.enable_attribute_deprecations(lowlevel.__name__) -lowlevel.__deprecated_attributes__ = { - "wait_socket_readable": _deprecate.DeprecatedAttribute( - lowlevel.wait_readable, "0.12.0", issue=878, - ), - "wait_socket_writable": _deprecate.DeprecatedAttribute( - lowlevel.wait_writable, "0.12.0", issue=878, - ), - "notify_socket_close": _deprecate.DeprecatedAttribute( - lowlevel.notify_closing, "0.12.0", issue=878, - ), - "notify_fd_close": _deprecate.DeprecatedAttribute( - lowlevel.notify_closing, "0.12.0", issue=878, - ), -} # Having the public path in .__module__ attributes is important for: # - exception names in printed tracebacks From c95f6f9d1d844ca53221cf276d5262a455c51641 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 15:41:57 -0700 Subject: [PATCH 0265/1498] Make sleep duration less over-the-top --- trio/_core/tests/test_guest_mode.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index a5490c99bb..e58ec3ebc2 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -483,7 +483,7 @@ def test_guest_mode_autojump_clock_threshold_changing(): clock = trio.testing.MockClock() - DURATION = 9999999 + DURATION = 120 async def trio_main(in_host): assert trio.current_time() == 0 @@ -494,4 +494,6 @@ async def trio_main(in_host): start = time.monotonic() trivial_guest_run(trio_main, clock=clock) end = time.monotonic() - assert end - start < DURATION / 10 + # Should be basically instantaneous, but we'll leave a generous buffer to + # account for any CI weirdness + assert end - start < DURATION / 2 From e71f69621b742071c474878c27f4f46a4c7440f0 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 15:42:22 -0700 Subject: [PATCH 0266/1498] Remove some dead code --- trio/_core/_mock_clock.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/trio/_core/_mock_clock.py b/trio/_core/_mock_clock.py index 467707c5f4..6d9332fd23 100644 --- a/trio/_core/_mock_clock.py +++ b/trio/_core/_mock_clock.py @@ -70,8 +70,6 @@ def __init__(self, rate=0.0, autojump_threshold=inf): self._virtual_base = 0.0 self._rate = 0.0 self._autojump_threshold = 0.0 - self._autojump_task = None - self._autojump_cancel_scope = None # kept as an attribute so that our tests can monkeypatch it self._real_clock = time.perf_counter From 3c27ee7841c86ca7ee9d3cb19c11f2bd80ca3357 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Tue, 9 Jun 2020 19:42:31 -0300 Subject: [PATCH 0267/1498] Fixed function name in newsfragment, and new test name. --- newsfragments/1573.bugfix.rst | 2 +- trio/tests/test_threads.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newsfragments/1573.bugfix.rst b/newsfragments/1573.bugfix.rst index 1995ea15ab..26fdec2be3 100644 --- a/newsfragments/1573.bugfix.rst +++ b/newsfragments/1573.bugfix.rst @@ -1 +1 @@ -Added a helpful error message if an async function is passed to `trio.to_thread_run_sync`. \ No newline at end of file +Added a helpful error message if an async function is passed to `trio.to_thread.run_sync`. \ No newline at end of file diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 976220cc32..55f01eeaaa 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -457,7 +457,7 @@ def thread_fn(): assert callee_token == caller_token -async def test_trio_to_thread_run_sync(): +async def test_trio_to_thread_run_sync_expected_error(): # Test correct error when passed async function async def async_fn(): # pragma: no cover pass From 6b4341c0172a091290e52c5a585ad6720f4ba4e4 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 15:43:05 -0700 Subject: [PATCH 0268/1498] Track idle primed state in one variable, instead of two --- trio/_core/_run.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 1441361dd4..8b112051e5 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2008,20 +2008,18 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): timeout = _MAX_TIMEOUT timeout = min(max(0, timeout), _MAX_TIMEOUT) - idle_primed = False + idle_primed = None if runner.waiting_for_idle: cushion, tiebreaker, _ = runner.waiting_for_idle.keys()[0] if cushion < timeout: timeout = cushion - idle_primed = True - idle_prime_type = IdlePrimedTypes.WAITING_FOR_IDLE + idle_primed = IdlePrimedTypes.WAITING_FOR_IDLE # We use 'elif' here because if there are tasks in # wait_all_tasks_blocked, then those tasks will wake up without # jumping the clock, so we don't need to autojump. elif runner.clock_autojump_threshold < timeout: timeout = runner.clock_autojump_threshold - idle_primed = True - idle_prime_type = IdlePrimedTypes.AUTOJUMP_CLOCK + idle_primed = IdlePrimedTypes.AUTOJUMP_CLOCK if runner.instruments: runner.instrument("before_io_wait", timeout) @@ -2041,18 +2039,18 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): if deadline <= now: # This removes the given scope from runner.deadlines: cancel_scope.cancel() - idle_primed = False + idle_primed = None else: break - # idle_primed=True means: if the IO wait hit the timeout, and still - # nothing is happening, then we should start waking up + # idle_primed != None means: if the IO wait hit the timeout, and + # still nothing is happening, then we should start waking up # wait_all_tasks_blocked tasks or autojump the clock. But there # are some subtleties in defining "nothing is happening". # # 'not runner.runq' means that no tasks are currently runnable. # 'not events' means that the last IO wait call hit its full - # timeout. These are very similar, and if idle_primed=True and + # timeout. These are very similar, and if idle_primed != None and # we're running in regular mode then they always go together. But, # in *guest* mode, they can happen independently, even when # idle_primed=True: @@ -2066,8 +2064,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # before we got here. # # So we need to check both. - if idle_primed and not runner.runq and not events: - if idle_prime_type is IdlePrimedTypes.WAITING_FOR_IDLE: + if idle_primed is not None and not runner.runq and not events: + if idle_primed is IdlePrimedTypes.WAITING_FOR_IDLE: while runner.waiting_for_idle: key, task = runner.waiting_for_idle.peekitem(0) if key[:2] == (cushion, tiebreaker): @@ -2076,7 +2074,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): else: break else: - assert idle_prime_type is IdlePrimedTypes.AUTOJUMP_CLOCK + assert idle_primed is IdlePrimedTypes.AUTOJUMP_CLOCK runner.clock._autojump() # Process all runnable tasks, but only the ones that are already From 27989c9e2a64c1a8aa3bbbd8cc4093ccd297ef73 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 17:36:36 -0700 Subject: [PATCH 0269/1498] Deprecate the tiebreaker= argument to wait_all_tasks_blocked It was only added for the autojump clock, it wasn't very good at that, and since gh-1588, the autojump clock doesn't even use it anyway. --- newsfragments/1587.deprecated.rst | 4 ++++ trio/_core/_generated_run.py | 6 ++---- trio/_core/_run.py | 18 +++++++++++++----- trio/_core/tests/test_mock_clock.py | 19 ++++--------------- trio/tests/test_testing.py | 3 ++- 5 files changed, 25 insertions(+), 25 deletions(-) create mode 100644 newsfragments/1587.deprecated.rst diff --git a/newsfragments/1587.deprecated.rst b/newsfragments/1587.deprecated.rst new file mode 100644 index 0000000000..03c0677c80 --- /dev/null +++ b/newsfragments/1587.deprecated.rst @@ -0,0 +1,4 @@ +The ``tiebreaker`` argument to `trio.testing.wait_all_tasks_blocked` +has been deprecated. This is a highly obscure feature that was +probably never used by anyone except `trio.testing.MockClock`, and +`~trio.testing.MockClock` doesn't need it anymore. diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index edf46fd741..49e2d6acd9 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -161,7 +161,7 @@ def current_trio_token(): raise RuntimeError("must be called from async context") -async def wait_all_tasks_blocked(cushion=0.0, tiebreaker=0): +async def wait_all_tasks_blocked(cushion=0.0, tiebreaker='deprecated'): """Block until there are no runnable tasks. This is useful in testing code when you want to give other tasks a @@ -179,9 +179,7 @@ async def wait_all_tasks_blocked(cushion=0.0, tiebreaker=0): then the one with the shortest ``cushion`` is the one woken (and this task becoming unblocked resets the timers for the remaining tasks). If there are multiple tasks that have exactly the same - ``cushion``, then the one with the lowest ``tiebreaker`` value is - woken first. And if there are multiple tasks with the same ``cushion`` - and the same ``tiebreaker``, then all are woken. + ``cushion``, then all are woken. You should also consider :class:`trio.testing.Sequencer`, which provides a more explicit way to control execution ordering within a diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 8b112051e5..8e8b1e90b4 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -41,7 +41,7 @@ ) from ._thread_cache import start_thread_soon from .. import _core -from .._deprecate import deprecated +from .._deprecate import deprecated, warn_deprecated from .._util import Final, NoPublicConstructor, coroutine_or_error _NO_SEND = object() @@ -1563,7 +1563,7 @@ def _deliver_ki_cb(self): waiting_for_idle = attr.ib(factory=SortedDict) @_public - async def wait_all_tasks_blocked(self, cushion=0.0, tiebreaker=0): + async def wait_all_tasks_blocked(self, cushion=0.0, tiebreaker="deprecated"): """Block until there are no runnable tasks. This is useful in testing code when you want to give other tasks a @@ -1581,9 +1581,7 @@ async def wait_all_tasks_blocked(self, cushion=0.0, tiebreaker=0): then the one with the shortest ``cushion`` is the one woken (and this task becoming unblocked resets the timers for the remaining tasks). If there are multiple tasks that have exactly the same - ``cushion``, then the one with the lowest ``tiebreaker`` value is - woken first. And if there are multiple tasks with the same ``cushion`` - and the same ``tiebreaker``, then all are woken. + ``cushion``, then all are woken. You should also consider :class:`trio.testing.Sequencer`, which provides a more explicit way to control execution ordering within a @@ -1623,6 +1621,16 @@ async def test_lock_fairness(): print("FAIL") """ + if tiebreaker == "deprecated": + tiebreaker = 0 + else: + warn_deprecated( + "the 'tiebreaker' argument to wait_all_tasks_blocked", + "v0.16.0", + issue=1558, + instead=None, + ) + task = current_task() key = (cushion, tiebreaker, id(task)) self.waiting_for_idle[key] = task diff --git a/trio/_core/tests/test_mock_clock.py b/trio/_core/tests/test_mock_clock.py index aef4cef498..bea9509686 100644 --- a/trio/_core/tests/test_mock_clock.py +++ b/trio/_core/tests/test_mock_clock.py @@ -84,15 +84,6 @@ async def test_mock_clock_autojump(mock_clock): await wait_all_tasks_blocked(0.01) assert t == _core.current_time() - # This should wake up at the same time as the autojump_threshold, and - # confuse things. There is no deadline, so it shouldn't actually jump - # the clock. But does it handle the situation gracefully? - await wait_all_tasks_blocked(cushion=0.02, tiebreaker=float("inf")) - # And again with threshold=0, because that has some special - # busy-wait-avoidance logic: - mock_clock.autojump_threshold = 0 - await wait_all_tasks_blocked(tiebreaker=float("inf")) - # set up a situation where the autojump task is blocked for a long long # time, to make sure that cancel-and-adjust-threshold logic is working mock_clock.autojump_threshold = 10000 @@ -132,8 +123,7 @@ def test_mock_clock_autojump_preset(): async def test_mock_clock_autojump_0_and_wait_all_tasks_blocked_0(mock_clock): # Checks that autojump_threshold=0 doesn't interfere with - # calling wait_all_tasks_blocked with the default cushion=0 and arbitrary - # tiebreakers. + # calling wait_all_tasks_blocked with the default cushion=0. mock_clock.autojump_threshold = 0 @@ -144,9 +134,8 @@ async def sleeper(): record.append("yawn") async def waiter(): - for i in range(10): - await wait_all_tasks_blocked(tiebreaker=i) - record.append(i) + await wait_all_tasks_blocked() + record.append("waiter woke") await sleep(1000) record.append("waiter done") @@ -154,7 +143,7 @@ async def waiter(): nursery.start_soon(sleeper) nursery.start_soon(waiter) - assert record == list(range(10)) + ["yawn", "waiter done"] + assert record == ["waiter woke", "yawn", "waiter done"] @slow diff --git a/trio/tests/test_testing.py b/trio/tests/test_testing.py index 304f18cb88..706d0a5a13 100644 --- a/trio/tests/test_testing.py +++ b/trio/tests/test_testing.py @@ -103,7 +103,8 @@ async def wait_big_cushion(): ] -async def test_wait_all_tasks_blocked_with_tiebreaker(): +# This test can be deleted after tiebreaker= is removed +async def test_wait_all_tasks_blocked_with_tiebreaker(recwarn): record = [] async def do_wait(cushion, tiebreaker): From 82246c9416a79f573817081945ffe72e5db2379e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 20:47:35 -0700 Subject: [PATCH 0270/1498] Give up on measuring TCP socket backlog It is just way too flaky --- .../test_highlevel_open_tcp_listeners.py | 86 ++++++------------- 1 file changed, 27 insertions(+), 59 deletions(-) diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index f556a09578..ee9672caf8 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -52,65 +52,6 @@ async def test_open_tcp_listeners_specific_port_specific_host(): assert listener.socket.getsockname() == (host, port) -# Warning: this sleeps, and needs to use a real sleep -- MockClock won't -# work. -# -# Also, this measurement technique often works, but not always: sometimes SYN -# cookies get triggered, and then the backlog measured this way effectively -# becomes infinite. (In particular, this has been observed happening on -# Travis-CI.) To avoid this blowing up and eating all FDs / ephemeral ports, -# we put an upper limit on the number of connections we attempt, and if we hit -# it then we return the magic string "lots". Then -# test_open_tcp_listeners_backlog uses a special path to handle this, treating -# it as a success -- but at least we'll see in coverage if none of our test -# runs are actually running the test properly. -async def measure_backlog(listener, limit): - client_streams = [] - try: - while True: - # Generally the response to the listen buffer being full is that - # the SYN gets dropped, and the client retries after 1 second. So - # we assume that any connect() call to localhost that takes >0.5 - # seconds indicates a dropped SYN. - # - # Exception: on FreeBSD when the backlog is exhausted, connect - # has been observed to sometimes raise ConnectionResetError. - with trio.move_on_after(0.5) as cancel_scope: - try: - client_stream = await open_stream_to_socket_listener(listener) - except ConnectionResetError: # pragma: no cover - break - client_streams.append(client_stream) - if cancel_scope.cancelled_caught: - break - if len(client_streams) >= limit: # pragma: no cover - return "lots" - finally: - # The need for "no cover" here is subtle: see - # https://github.com/python-trio/trio/issues/522 - for client_stream in client_streams: # pragma: no cover - await client_stream.aclose() - - return len(client_streams) - - -@slow -async def test_open_tcp_listeners_backlog(): - # Operating systems don't necessarily use the exact backlog you pass - async def check_backlog(nominal, required_min, required_max): - listeners = await open_tcp_listeners(0, backlog=nominal) - actual = await measure_backlog(listeners[0], required_max + 10) - for listener in listeners: - await listener.aclose() - print("nominal", nominal, "actual", actual) - if actual == "lots": # pragma: no cover - return - assert required_min <= actual <= required_max - - await check_backlog(nominal=1, required_min=1, required_max=10) - await check_backlog(nominal=11, required_min=11, required_max=20) - - @binds_ipv6 async def test_open_tcp_listeners_ipv6_v6only(): # Check IPV6_V6ONLY is working properly @@ -175,6 +116,7 @@ class FakeSocket(tsocket.SocketType): closed = attr.ib(default=False) poison_listen = attr.ib(default=False) + backlog = attr.ib(default=None) def getsockopt(self, level, option): if (level, option) == (tsocket.SOL_SOCKET, tsocket.SO_ACCEPTCONN): @@ -188,6 +130,9 @@ async def bind(self, sockaddr): pass def listen(self, backlog): + assert self.backlog is None + assert backlog is not None + self.backlog = backlog if self.poison_listen: raise FakeOSError("whoops") @@ -325,3 +270,26 @@ async def test_open_tcp_listeners_socket_fails_not_afnosupport(): assert exc_info.value.errno == errno.EINVAL assert exc_info.value.__cause__ is None assert "nope" in str(exc_info.value) + + +# We used to have an elaborate test that opened a real TCP listening socket +# and then tried to measure its backlog by making connections to it. And most +# of the time, it worked. But no matter what we tried, it was always fragile, +# because it had to do things like use timeouts to guess when the listening +# queue was full, sometimes the CI hosts go into SYN-cookie mode (where there +# effectively is no backlog), sometimes the host might not be enough resources +# to give us the full requested backlog... it was a mess. So now we just check +# that the backlog argument is passed through correctly. +async def test_open_tcp_listeners_backlog(): + fsf = FakeSocketFactory(99) + tsocket.set_custom_socket_factory(fsf) + for (given, expected) in [ + (None, 0xFFFF), + (99999999, 0xFFFF), + (10, 10), + (1, 1), + ]: + listeners = await open_tcp_listeners(0, backlog=given) + assert listeners + for listener in listeners: + assert listener.socket.backlog == expected From c067244352e04e12228c14e7000e7f15ac171a2a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 10 Jun 2020 08:58:48 +0400 Subject: [PATCH 0271/1498] Add newsfragment for ssl/subprocess removals --- newsfragments/1594.deprecated.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1594.deprecated.rst diff --git a/newsfragments/1594.deprecated.rst b/newsfragments/1594.deprecated.rst new file mode 100644 index 0000000000..6706ecb7ca --- /dev/null +++ b/newsfragments/1594.deprecated.rst @@ -0,0 +1 @@ +Remove the deprecated ``trio.ssl`` and ``trio.subprocess`` modules. From 7ef6f09f79ec3ce3ba8c3658cfa79457c55952ed Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 10 Jun 2020 09:18:35 +0400 Subject: [PATCH 0272/1498] Release 0.16.0 --- docs/source/history.rst | 62 +++++++++++++++++++++++++++++++ newsfragments/1558.feature.rst | 11 ------ newsfragments/1573.bugfix.rst | 1 - newsfragments/1574.deprecated.rst | 1 - newsfragments/1587.deprecated.rst | 4 -- newsfragments/1587.misc.rst | 7 ---- newsfragments/1594.deprecated.rst | 1 - newsfragments/399.headline.rst | 3 -- newsfragments/6.feature.rst | 6 --- trio/_version.py | 2 +- 10 files changed, 63 insertions(+), 35 deletions(-) delete mode 100644 newsfragments/1558.feature.rst delete mode 100644 newsfragments/1573.bugfix.rst delete mode 100644 newsfragments/1574.deprecated.rst delete mode 100644 newsfragments/1587.deprecated.rst delete mode 100644 newsfragments/1587.misc.rst delete mode 100644 newsfragments/1594.deprecated.rst delete mode 100644 newsfragments/399.headline.rst delete mode 100644 newsfragments/6.feature.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index d6967217c0..33f72d8681 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,68 @@ Release history .. towncrier release notes start +Trio 0.16.0 (2020-06-10) +------------------------ + +Headline features +~~~~~~~~~~~~~~~~~ + +- If you want to use Trio, but are stuck with some other event loop like + Qt or PyGame, then good news: now you can have both. For details, see: + :ref:`guest-mode`. (`#399 `__) + + +Features +~~~~~~~~ + +- To speed up `trio.to_thread.run_sync`, Trio now caches and re-uses + worker threads. + + And in case you have some exotic use case where you need to spawn + threads manually, but want to take advantage of Trio's cache, you can + do that using the new `trio.lowlevel.start_thread_soon`. (`#6 `__) +- Tasks spawned with `nursery.start() ` aren't treated as + direct children of their nursery until they call ``task_status.started()``. + This is visible through the task tree introspection attributes such as + `Task.parent_nursery `. Sometimes, though, + you want to know where the task is going to wind up, even if it hasn't finished + initializing yet. To support this, we added a new attribute + `Task.eventual_parent_nursery `. + For a task spawned with :meth:`~trio.Nursery.start` that hasn't yet called + ``started()``, this is the nursery that the task was nominally started in, + where it will be running once it finishes starting up. In all other cases, + it is ``None``. (`#1558 `__) + + +Bugfixes +~~~~~~~~ + +- Added a helpful error message if an async function is passed to `trio.to_thread.run_sync`. (`#1573 `__) + + +Deprecations and removals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Remove ``BlockingTrioPortal``: it was deprecated in 0.12.0. (`#1574 `__) +- The ``tiebreaker`` argument to `trio.testing.wait_all_tasks_blocked` + has been deprecated. This is a highly obscure feature that was + probably never used by anyone except `trio.testing.MockClock`, and + `~trio.testing.MockClock` doesn't need it anymore. (`#1587 `__) +- Remove the deprecated ``trio.ssl`` and ``trio.subprocess`` modules. (`#1594 `__) + + +Miscellaneous internal changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- We refactored `trio.testing.MockClock` so that it no longer needs to + run an internal task to manage autojumping. This should be mostly + invisible to users, but there is one semantic change: the interaction + between `trio.testing.wait_all_tasks_blocked` and the autojump clock + was fixed. Now, the autojump will always wait until after all + `~trio.testing.wait_all_tasks_blocked` calls have finished before + firing, instead of it depending on which threshold values you passed. (`#1587 `__) + + Trio 0.15.1 (2020-05-22) ------------------------ diff --git a/newsfragments/1558.feature.rst b/newsfragments/1558.feature.rst deleted file mode 100644 index 6f8265d232..0000000000 --- a/newsfragments/1558.feature.rst +++ /dev/null @@ -1,11 +0,0 @@ -Tasks spawned with `nursery.start() ` aren't treated as -direct children of their nursery until they call ``task_status.started()``. -This is visible through the task tree introspection attributes such as -`Task.parent_nursery `. Sometimes, though, -you want to know where the task is going to wind up, even if it hasn't finished -initializing yet. To support this, we added a new attribute -`Task.eventual_parent_nursery `. -For a task spawned with :meth:`~trio.Nursery.start` that hasn't yet called -``started()``, this is the nursery that the task was nominally started in, -where it will be running once it finishes starting up. In all other cases, -it is ``None``. diff --git a/newsfragments/1573.bugfix.rst b/newsfragments/1573.bugfix.rst deleted file mode 100644 index 26fdec2be3..0000000000 --- a/newsfragments/1573.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Added a helpful error message if an async function is passed to `trio.to_thread.run_sync`. \ No newline at end of file diff --git a/newsfragments/1574.deprecated.rst b/newsfragments/1574.deprecated.rst deleted file mode 100644 index 175e607319..0000000000 --- a/newsfragments/1574.deprecated.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``BlockingTrioPortal``: it was deprecated in 0.12.0. diff --git a/newsfragments/1587.deprecated.rst b/newsfragments/1587.deprecated.rst deleted file mode 100644 index 03c0677c80..0000000000 --- a/newsfragments/1587.deprecated.rst +++ /dev/null @@ -1,4 +0,0 @@ -The ``tiebreaker`` argument to `trio.testing.wait_all_tasks_blocked` -has been deprecated. This is a highly obscure feature that was -probably never used by anyone except `trio.testing.MockClock`, and -`~trio.testing.MockClock` doesn't need it anymore. diff --git a/newsfragments/1587.misc.rst b/newsfragments/1587.misc.rst deleted file mode 100644 index 8152340ba2..0000000000 --- a/newsfragments/1587.misc.rst +++ /dev/null @@ -1,7 +0,0 @@ -We refactored `trio.testing.MockClock` so that it no longer needs to -run an internal task to manage autojumping. This should be mostly -invisible to users, but there is one semantic change: the interaction -between `trio.testing.wait_all_tasks_blocked` and the autojump clock -was fixed. Now, the autojump will always wait until after all -`~trio.testing.wait_all_tasks_blocked` calls have finished before -firing, instead of it depending on which threshold values you passed. diff --git a/newsfragments/1594.deprecated.rst b/newsfragments/1594.deprecated.rst deleted file mode 100644 index 6706ecb7ca..0000000000 --- a/newsfragments/1594.deprecated.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the deprecated ``trio.ssl`` and ``trio.subprocess`` modules. diff --git a/newsfragments/399.headline.rst b/newsfragments/399.headline.rst deleted file mode 100644 index 4de52bfb30..0000000000 --- a/newsfragments/399.headline.rst +++ /dev/null @@ -1,3 +0,0 @@ -If you want to use Trio, but are stuck with some other event loop like -Qt or PyGame, then good news: now you can have both. For details, see: -:ref:`guest-mode`. diff --git a/newsfragments/6.feature.rst b/newsfragments/6.feature.rst deleted file mode 100644 index 8baa3a27d9..0000000000 --- a/newsfragments/6.feature.rst +++ /dev/null @@ -1,6 +0,0 @@ -To speed up `trio.to_thread.run_sync`, Trio now caches and re-uses -worker threads. - -And in case you have some exotic use case where you need to spawn -threads manually, but want to take advantage of Trio's cache, you can -do that using the new `trio.lowlevel.start_thread_soon`. diff --git a/trio/_version.py b/trio/_version.py index 4d8550b19f..5d09687ce8 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.15.1+dev" +__version__ = "0.16.0" From 153dd7a9c89260635a83d818fbf736af930b4d35 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 10 Jun 2020 10:00:19 +0400 Subject: [PATCH 0273/1498] Bump version to v0.16.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index 5d09687ce8..4d1671b9ec 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.16.0" +__version__ = "0.16.0+dev" From 236118e24f5583e64916b94b02908ba9dfc5893d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 9 Jun 2020 23:05:41 -0700 Subject: [PATCH 0274/1498] Stop leaking a thread in test_race_between_idle_exit_and_job_assignment It can lead to confusion, e.g.: https://github.com/python-trio/trio/issues/1604 --- trio/_core/tests/test_thread_cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/trio/_core/tests/test_thread_cache.py b/trio/_core/tests/test_thread_cache.py index 895c1c2e5f..f19ac1d4cc 100644 --- a/trio/_core/tests/test_thread_cache.py +++ b/trio/_core/tests/test_thread_cache.py @@ -2,6 +2,7 @@ import threading from queue import Queue import time +import sys from .tutil import slow from .. import _thread_cache @@ -118,3 +119,9 @@ def release(self): done = threading.Event() tc.start_thread_soon(lambda: None, lambda _: done.set()) done.wait() + # Let's kill the thread we started, so it doesn't hang around until the + # test suite finishes. Doesn't really do any harm, but it can be confusing + # to see it in debug output. This is hacky, and leaves our ThreadCache + # object in an inconsistent state... but it doesn't matter, because we're + # not going to use it again anyway. + tc.start_thread_soon(lambda: None, lambda _: sys.exit()) From 341fbbfa879a7e4538ff806c0f8602108265033d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2020 06:26:21 +0000 Subject: [PATCH 0275/1498] Bump backcall from 0.1.0 to 0.2.0 Bumps [backcall](https://github.com/takluyver/backcall) from 0.1.0 to 0.2.0. - [Release notes](https://github.com/takluyver/backcall/releases) - [Commits](https://github.com/takluyver/backcall/compare/0.1...0.2.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2c5d9ac0c4..4f39cc066d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,7 +9,7 @@ astor==0.8.1 # via -r test-requirements.in astroid==2.4.2 # via pylint async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, black, outcome, pytest -backcall==0.1.0 # via ipython +backcall==0.2.0 # via ipython black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.0 # via cryptography click==7.1.2 # via black From 26f1aeae1e42cef682b829b98b36cb0fc5f52b8d Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 10 Jun 2020 08:03:20 +0000 Subject: [PATCH 0276/1498] Async generator hooks, simpler approach --- trio/_core/_entry_queue.py | 14 ++- trio/_core/_run.py | 193 +++++++++++++++++++++++++++--- trio/_core/tests/test_asyncgen.py | 115 ++++++++++++++++++ trio/_core/tests/test_run.py | 1 - trio/_util.py | 19 +++ 5 files changed, 319 insertions(+), 23 deletions(-) create mode 100644 trio/_core/tests/test_asyncgen.py diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index 791ab8ca6e..ac71405e4c 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -56,7 +56,15 @@ def run_cb(job): async def kill_everything(exc): raise exc - _core.spawn_system_task(kill_everything, exc) + try: + _core.spawn_system_task(kill_everything, exc) + except RuntimeError: + # We're quite late in the shutdown process and + # the system nursery is already closed. + _core.current_task().parent_nursery.start_soon( + kill_everything, exc + ) + return True # This has to be carefully written to be safe in the face of new items @@ -102,10 +110,6 @@ def close(self): def size(self): return len(self.queue) + len(self.idempotent_queue) - def spawn(self): - name = "" - _core.spawn_system_task(self.task, name=name) - def run_sync_soon(self, sync_fn, *args, idempotent=False): with self.lock: if self.done: diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 8b112051e5..24cdf9166d 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -9,9 +9,11 @@ import sys import threading from collections import deque +from functools import partial import collections.abc from contextlib import contextmanager, closing import warnings +import weakref import enum from contextvars import copy_context @@ -42,7 +44,7 @@ from ._thread_cache import start_thread_soon from .. import _core from .._deprecate import deprecated -from .._util import Final, NoPublicConstructor, coroutine_or_error +from .._util import Final, NoPublicConstructor, coroutine_or_error, name_asyncgen _NO_SEND = object() @@ -61,8 +63,9 @@ def _public(fn): _ALLOW_DETERMINISTIC_SCHEDULING = False _r = random.Random() -# Used to log exceptions in instruments +# Used to log exceptions in instruments and async generator finalizers INSTRUMENT_LOGGER = logging.getLogger("trio.abc.Instrument") +ASYNCGEN_LOGGER = logging.getLogger("trio.async_generator_errors") # On 3.7+, Context.run() is implemented in C and doesn't show up in @@ -958,7 +961,7 @@ async def async_fn(arg1, arg2, \*, task_status=trio.TASK_STATUS_IGNORED): self._pending_starts += 1 async with open_nursery() as old_nursery: task_status = _TaskStatus(old_nursery, self) - thunk = functools.partial(async_fn, task_status=task_status) + thunk = partial(async_fn, task_status=task_status) task = GLOBAL_RUN_CONTEXT.runner.spawn_impl( thunk, args, old_nursery, name ) @@ -1222,6 +1225,14 @@ class Runner: is_guest = attr.ib(default=False) guest_tick_scheduled = attr.ib(default=False) + # Async generators are added to this set when first iterated. Any + # left after the main task exits will be closed before trio.run() + # returns. During the execution of the main task, this is a + # WeakSet so GC works. During shutdown, it's a regular set so we + # don't have to deal with GC firing at unexpected times. + asyncgens = attr.ib(factory=weakref.WeakSet) + prev_asyncgen_hooks = attr.ib(default=None) + def force_guest_tick_asap(self): if self.guest_tick_scheduled: return @@ -1231,6 +1242,8 @@ def force_guest_tick_asap(self): def close(self): self.io_manager.close() self.entry_queue.close() + if self.prev_asyncgen_hooks is not None: + sys.set_asyncgen_hooks(*self.prev_asyncgen_hooks) if self.instruments: self.instrument("after_run") # This is where KI protection gets disabled, so we do it last @@ -1366,7 +1379,7 @@ def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): if name is None: name = async_fn - if isinstance(name, functools.partial): + if isinstance(name, partial): name = name.func if not isinstance(name, str): try: @@ -1432,11 +1445,7 @@ def task_exited(self, task, outcome): task._activate_cancel_status(None) self.tasks.remove(task) - if task is self.main_task: - self.main_task_outcome = outcome - self.system_nursery.cancel_scope.cancel() - self.system_nursery._child_finished(task, Value(None)) - elif task is self.init_task: + if task is self.init_task: # If the init task crashed, then something is very wrong and we # let the error propagate. (It'll eventually be wrapped in a # TrioInternalError.) @@ -1446,11 +1455,120 @@ def task_exited(self, task, outcome): if self.tasks: # pragma: no cover raise TrioInternalError else: + if task is self.main_task: + self.main_task_outcome = outcome + outcome = Value(None) task._parent_nursery._child_finished(task, outcome) if self.instruments: self.instrument("task_exited", task) + ################ + # Async generator finalization support + ################ + + async def finalize_asyncgen(self, agen, name, *, check_running): + if check_running and agen.ag_running: + # Another async generator is iterating this one, which is + # suspended at an event loop trap. Add it back to the + # asyncgens set and we'll get it on the next round. Note + # that this is only possible during end-of-run + # finalization; in GC-directed finalization, no one has a + # reference to agen anymore, so no one can be iterating it. + # + # This field is only reliable on 3.8+ due to + # ttps://bugs.python.org/issue32526. Pythons below + # 3.8 use a workaround in finalize_remaining_asyncgens. + self.asyncgens.add(agen) + return + + try: + # This shield ensures that finalize_asyncgen never exits + # with an exception, not even a Cancelled. The inside + # is cancelled so there's no deadlock risk. + with CancelScope(shield=True) as cancel_scope: + cancel_scope.cancel() + await agen.aclose() + except BaseException as exc: + ASYNCGEN_LOGGER.exception( + "Exception ignored during finalization of async generator %r -- " + "surround your use of the generator in 'async with aclosing(...):' " + "to raise exceptions like this in the context where they're generated", + name, + ) + + async def finalize_remaining_asyncgens(self): + # At the time this function is called, there are exactly two + # tasks running: init and the run_sync_soon task. (And we've + # shut down the system nursery, so no more can appear.) + # Neither one uses async generators, so every async generator + # must be suspended at a yield point -- there's no one to be + # doing the iteration. However, once we start aclose() of one + # async generator, it might start fetching the next value from + # another, thus preventing us from closing that other. + # + # On 3.8+, we can detect this condition by looking at + # ag_running. On earlier versions, ag_running doesn't provide + # useful information. We could look at ag_await, but that + # would fail in case of shenanigans like + # https://github.com/python-trio/async_generator/pull/16. + # It's easier to just not parallelize the shutdowns. + finalize_in_parallel = sys.version_info >= (3, 8) + + # It's possible that that cleanup code will itself create + # more async generators, so we iterate repeatedly until + # all are gone. + while self.asyncgens: + batch = self.asyncgens + self.asyncgens = set() + + if finalize_in_parallel: + async with open_nursery() as kill_them_all: + # This shield is needed to avoid the checkpoint + # in Nursery.__aexit__ raising Cancelled if we're + # in a cancelled scope. (Which can happen if + # a run_sync_soon callback raises an exception.) + kill_them_all.cancel_scope.shield = True + for agen in batch: + name = name_asyncgen(agen) + kill_them_all.start_soon( + partial(self.finalize_asyncgen, agen, name, check_running=True), + name="close asyncgen {} (outlived run)".format(name), + ) + + if self.asyncgens == batch: # pragma: no cover + # Something about the running-detection seems + # to have failed; fall back to one-at-a-time mode + # instead of looping forever + finalize_in_parallel = False + else: + for agen in batch: + await self.finalize_asyncgen(agen, name_asyncgen(agen), check_running=False) + + def setup_asyncgen_hooks(self): + def firstiter(agen): + self.asyncgens.add(agen) + + def finalizer(agen): + agen_name = name_asyncgen(agen) + warnings.warn( + f"Async generator {agen_name!r} was garbage collected before it had " + f"been exhausted. Surround its use in 'async with aclosing(...):' " + f"to ensure that it gets cleaned up as soon as you're done using it.", + ResourceWarning, + stacklevel=2, + ) + self.entry_queue.run_sync_soon( + partial( + self.spawn_system_task, + partial(self.finalize_asyncgen, agen, agen_name, check_running=False), + name=f"close asyncgen {agen_name} (abandoned)", + ), + ) + + self.prev_asyncgen_hooks = sys.get_asyncgen_hooks() + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer=finalizer) + ################ # System tasks and init ################ @@ -1500,14 +1618,51 @@ def spawn_system_task(self, async_fn, *args, name=None): ) async def init(self, async_fn, args): - async with open_nursery() as system_nursery: - self.system_nursery = system_nursery - try: - self.main_task = self.spawn_impl(async_fn, args, system_nursery, None) - except BaseException as exc: - self.main_task_outcome = Error(exc) - system_nursery.cancel_scope.cancel() - self.entry_queue.spawn() + # run_sync_soon task runs here: + async with open_nursery() as run_sync_soon_nursery: + # All other system tasks run here: + async with open_nursery() as self.system_nursery: + # Only the main task runs here: + async with open_nursery() as main_task_nursery: + try: + self.main_task = self.spawn_impl( + async_fn, args, main_task_nursery, None + ) + except BaseException as exc: + self.main_task_outcome = Error(exc) + return + self.spawn_impl( + self.entry_queue.task, + (), + run_sync_soon_nursery, + "", + system_task=True, + ) + + # Main task is done. We should be exiting soon, so + # we're going to shut down GC-mediated async generator + # finalization by turning the asyncgens WeakSet into a + # regular set. We must do that before closing the system + # nursery, since finalization spawns a new system tasks. + self.asyncgens = set(self.asyncgens) + + # Process all pending run_sync_soon callbacks, in case one of + # them was an asyncgen finalizer. + self.entry_queue.run_sync_soon(self.reschedule, self.init_task) + await wait_task_rescheduled(lambda _: Abort.FAILED) + + # Now it's safe to proceed with shutting down system tasks + self.system_nursery.cancel_scope.cancel() + + # System tasks are gone and no more will be appearing. + # The only async-colored user code left to run is the + # finalizers for the async generators that remain alive. + await self.finalize_remaining_asyncgens() + + # There are no more asyncgens, which means no more user-provided + # code except possibly run_sync_soon callbacks. It's finally safe + # to stop the run_sync_soon task and exit run(). + run_sync_soon_nursery.cancel_scope.cancel() ################ # Outside context problems @@ -1989,6 +2144,10 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): if not host_uses_signal_set_wakeup_fd: runner.entry_queue.wakeup.wakeup_on_signals() + # Do this before before_run in case before_run wants to override + # our hooks + runner.setup_asyncgen_hooks() + if runner.instruments: runner.instrument("before_run") runner.clock.start_clock() diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py new file mode 100644 index 0000000000..ecc4c4ba54 --- /dev/null +++ b/trio/_core/tests/test_asyncgen.py @@ -0,0 +1,115 @@ +import sys +import pytest +from math import inf +from functools import partial +from async_generator import aclosing +from ... import _core +from .tutil import gc_collect_harder + + +def test_asyncgen_basics(): + collected = [] + + async def example(cause): + try: + try: + yield 42 + except GeneratorExit: + pass + await _core.checkpoint() + except _core.Cancelled: + assert "example" in _core.current_task().name + assert "exhausted" not in cause + task_name = _core.current_task().name + assert cause in task_name or task_name == "" + assert _core.current_effective_deadline() == -inf + with pytest.raises(_core.Cancelled): + await _core.checkpoint() + collected.append(cause) + else: + assert "async_main" in _core.current_task().name + assert "exhausted" in cause + assert _core.current_effective_deadline() == inf + await _core.checkpoint() + collected.append(cause) + + saved = [] + + async def async_main(): + # GC'ed before exhausted + with pytest.warns( + ResourceWarning, match="Async generator.*collected before.*exhausted", + ): + async for val in example("abandoned"): + assert val == 42 + break + gc_collect_harder() + await _core.wait_all_tasks_blocked() + assert collected.pop() == "abandoned" + + # aclosing() ensures it's cleaned up at point of use + async with aclosing(example("exhausted 1")) as aiter: + async for val in aiter: + assert val == 42 + break + assert collected.pop() == "exhausted 1" + + # Also fine if you exhaust it at point of use + async for val in example("exhausted 2"): + assert val == 42 + assert collected.pop() == "exhausted 2" + + gc_collect_harder() + + # No problems saving the geniter when using either of these patterns + async with aclosing(example("exhausted 3")) as aiter: + saved.append(aiter) + async for val in aiter: + assert val == 42 + break + assert collected.pop() == "exhausted 3" + + # Also fine if you exhaust it at point of use + saved.append(example("exhausted 4")) + async for val in saved[-1]: + assert val == 42 + assert collected.pop() == "exhausted 4" + + # Leave one referenced-but-unexhausted and make sure it gets cleaned up + saved.append(example("outlived run")) + async for val in saved[-1]: + assert val == 42 + break + assert collected == [] + + _core.run(async_main) + assert collected.pop() == "outlived run" + for agen in saved: + assert agen.ag_frame is None # all should now be exhausted + + +def test_firstiter_after_closing(): + saved = [] + record = [] + + async def funky_agen(): + try: + yield 1 + except GeneratorExit: + record.append("cleanup 1") + raise + try: + yield 2 + finally: + record.append("cleanup 2") + async for _ in funky_agen(): + break + + async def async_main(): + aiter = funky_agen() + saved.append(aiter) + assert 1 == await aiter.asend(None) + assert 2 == await aiter.asend(None) + + _core.run(async_main) + assert record == ["cleanup 2", "cleanup 1"] diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 78b46b7adc..1ddab60cd2 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -403,7 +403,6 @@ async def main(): + [("before", task), ("schedule", task), ("after", task)] * 5 + [("before", task), ("after", task), ("after_run",)] ) - assert len(r1.record) > len(r2.record) > len(r3.record) assert r1.record == r2.record + r3.record assert list(r1.filter_tasks([task])) == expected diff --git a/trio/_util.py b/trio/_util.py index 03b79065e2..939eaad332 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -1,3 +1,5 @@ +# coding: utf-8 + # Little utilities we use internally from abc import ABCMeta @@ -349,3 +351,20 @@ def __call__(self, *args, **kwargs): def _create(self, *args, **kwargs): return super().__call__(*args, **kwargs) + + +def name_asyncgen(agen): + """Return the fully-qualified name of the async generator function + that produced the async generator iterator *agen*. + """ + if not hasattr(agen, "ag_code"): + return repr(agen) + try: + module = agen.ag_frame.f_globals["__name__"] + except (AttributeError, KeyError): + module = "<{}>".format(agen.ag_code.co_filename) + try: + qualname = agen.__qualname__ + except AttributeError: + qualname = agen.ag_code.co_name + return f"{module}.{qualname}" From ac3e46d113e7b0f9eb1ad0708bd955a1a7dacc77 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 10 Jun 2020 08:21:30 +0000 Subject: [PATCH 0277/1498] blacken --- trio/_core/_entry_queue.py | 4 +--- trio/_core/_run.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index ac71405e4c..357ea2cca9 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -61,9 +61,7 @@ async def kill_everything(exc): except RuntimeError: # We're quite late in the shutdown process and # the system nursery is already closed. - _core.current_task().parent_nursery.start_soon( - kill_everything, exc - ) + _core.current_task().parent_nursery.start_soon(kill_everything, exc) return True diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 24cdf9166d..f891fda7b3 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1532,7 +1532,9 @@ async def finalize_remaining_asyncgens(self): for agen in batch: name = name_asyncgen(agen) kill_them_all.start_soon( - partial(self.finalize_asyncgen, agen, name, check_running=True), + partial( + self.finalize_asyncgen, agen, name, check_running=True + ), name="close asyncgen {} (outlived run)".format(name), ) @@ -1543,7 +1545,9 @@ async def finalize_remaining_asyncgens(self): finalize_in_parallel = False else: for agen in batch: - await self.finalize_asyncgen(agen, name_asyncgen(agen), check_running=False) + await self.finalize_asyncgen( + agen, name_asyncgen(agen), check_running=False + ) def setup_asyncgen_hooks(self): def firstiter(agen): @@ -1561,7 +1565,9 @@ def finalizer(agen): self.entry_queue.run_sync_soon( partial( self.spawn_system_task, - partial(self.finalize_asyncgen, agen, agen_name, check_running=False), + partial( + self.finalize_asyncgen, agen, agen_name, check_running=False + ), name=f"close asyncgen {agen_name} (abandoned)", ), ) From 0f5903cd7c3b0113db4182a1abf21f385746c333 Mon Sep 17 00:00:00 2001 From: Alex Chamberlain Date: Wed, 10 Jun 2020 22:25:27 +0100 Subject: [PATCH 0278/1498] Remove unused imports --- trio/_core/_ki.py | 1 - trio/_core/_run.py | 5 ++--- trio/_core/_thread_cache.py | 1 - trio/_core/_wakeup_socketpair.py | 1 - 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index 01ce03ce59..36aacecd96 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -1,7 +1,6 @@ import inspect import signal import sys -from contextlib import contextmanager from functools import wraps import attr diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 8e8b1e90b4..d576670767 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -9,8 +9,7 @@ import sys import threading from collections import deque -import collections.abc -from contextlib import contextmanager, closing +from contextlib import contextmanager import warnings import enum @@ -41,7 +40,7 @@ ) from ._thread_cache import start_thread_soon from .. import _core -from .._deprecate import deprecated, warn_deprecated +from .._deprecate import warn_deprecated from .._util import Final, NoPublicConstructor, coroutine_or_error _NO_SEND = object() diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index c71a9a37ea..8faf554322 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -1,5 +1,4 @@ from threading import Thread, Lock -import sys import outcome from itertools import count diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 80d3090ee9..86eaeb6355 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -1,6 +1,5 @@ import socket import sys -from contextlib import contextmanager import signal import warnings From b0ed1e4a149aa01e7b16f5e2fda7c8fbcc719152 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 12 Jun 2020 11:12:28 +0400 Subject: [PATCH 0279/1498] Enforce mypy-cleanliness on trio._core --- check.sh | 6 ++++++ test-requirements.in | 1 + test-requirements.txt | 3 +++ trio/_core/__init__.py | 20 +++++++++++++------- trio/_core/_io_epoll.py | 7 +++++-- trio/_core/_io_kqueue.py | 2 +- trio/_core/_multierror.py | 6 +++--- trio/_core/_run.py | 15 ++++++++++----- trio/_core/tests/test_multierror.py | 2 +- 9 files changed, 43 insertions(+), 19 deletions(-) diff --git a/check.sh b/check.sh index c1e81986ae..62ee6ab848 100755 --- a/check.sh +++ b/check.sh @@ -23,6 +23,12 @@ flake8 trio/ \ --ignore=D,E,W,F401,F403,F405,F821,F822\ || EXIT_STATUS=$? +# Run mypy +# We specify Linux so that we get the same results on all platforms. Without +# that, mypy would complain on macOS that epoll does not exist, and we can't +# ignore it as it would cause a mypy error on Linux +mypy -p trio._core --platform linux || EXIT_STATUS=$? + # Finally, leave a really clear warning of any issues and exit if [ $EXIT_STATUS -ne 0 ]; then cat < Date: Fri, 12 Jun 2020 11:57:48 +0400 Subject: [PATCH 0280/1498] Do not try installing mypy on PyPy It depends on typed-ast. --- test-requirements.in | 4 +++- test-requirements.txt | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index ec5992744b..9f5d6ec99f 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -9,12 +9,14 @@ jedi # for jedi code completion tests # Tools black; implementation_name == "cpython" +mypy; implementation_name == "cpython" flake8 astor # code generation -mypy # https://github.com/python-trio/trio/pull/654#issuecomment-420518745 typed_ast; implementation_name == "cpython" +mypy-extensions; implementation_name == "cpython" +typing-extensions; implementation_name == "cpython" # Trio's own dependencies cffi; os_name == "nt" diff --git a/test-requirements.txt b/test-requirements.txt index 001b66e2d1..1e5c9e2018 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -26,8 +26,8 @@ jedi==0.17.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint more-itertools==8.3.0 # via pytest -mypy-extensions==0.4.3 # via mypy -mypy==0.780 # via -r test-requirements.in +mypy-extensions==0.4.3 ; implementation_name == "cpython" # via mypy +mypy==0.780 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest parso==0.7.0 # via jedi @@ -55,6 +55,6 @@ toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black -typing-extensions==3.7.4.2 # via mypy +typing-extensions==3.7.4.2; implementation_name == "cpython" # via mypy wcwidth==0.2.4 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid From 1c7cdc2263e1dad1c4544d189e8a5c8fe5672e02 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 12 Jun 2020 13:13:34 +0400 Subject: [PATCH 0281/1498] Fix missing branch coverage "if not TYPE_CHECKING" is always True when running the code. --- trio/_core/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 5873d1ae2a..9ba611dcce 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -77,11 +77,7 @@ from ._mock_clock import MockClock # Kqueue imports -if ( - sys.platform.startswith("freebsd") - or sys.platform.startswith("darwin") - or not _t.TYPE_CHECKING -): +if not _t.TYPE_CHECKING: # pragma: no branch try: from ._run import current_kqueue, monitor_kevent, wait_kevent except ImportError: From 4e63bda88d17ac9b8a889d476c3f3e6be2e0bda1 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 12 Jun 2020 15:03:30 +0400 Subject: [PATCH 0282/1498] Run GitHub Actions in forks too This is the same configuration that we have in Travis: it allows testing the CI before sending a pull request. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04a96c495c..963f9e1531 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,8 @@ name: CI on: push: - branches: - - master + branches-ignore: + - "dependabot/*" pull_request: jobs: From e59050e8d6dda1262a804920c3b903e532eae9ad Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 13 Jun 2020 19:34:41 -0700 Subject: [PATCH 0283/1498] Make trio.lowlevel.checkpoint much faster This is a bit of a kluge, and will hopefully be cleaned up in the future when we overhaul KeyboardInterrupt (gh-733) and/or the cancellation checking APIs (gh-961). But 'checkpoint()' is a common operation, and this speeds it up by ~7x, so... not bad for a ~4 line change. Before this change: - `await checkpoint()`: ~24k/second - `await cancel_shielded_checkpoint()`: ~180k/second After this change: - `await checkpoint()`: ~170k/second - `await cancel_shielded_checkpoint()`: ~180k/second Benchmark script: ```python import time import trio LOOP_SIZE = 1_000_000 async def main(): start = time.monotonic() for _ in range(LOOP_SIZE): await trio.lowlevel.checkpoint() #await trio.lowlevel.cancel_shielded_checkpoint() end = time.monotonic() print(f"{LOOP_SIZE / (end - start):.2f} schedules/second") trio.run(main) ``` --- newsfragments/1613.feature.rst | 1 + trio/_core/_run.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 newsfragments/1613.feature.rst diff --git a/newsfragments/1613.feature.rst b/newsfragments/1613.feature.rst new file mode 100644 index 0000000000..2971bb054d --- /dev/null +++ b/newsfragments/1613.feature.rst @@ -0,0 +1 @@ +`trio.lowlevel.checkpoint` is now much faster. diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 8e8b1e90b4..643ae3e76c 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -35,6 +35,7 @@ from ._traps import ( Abort, wait_task_rescheduled, + cancel_shielded_checkpoint, CancelShieldedCheckpoint, PermanentlyDetachCoroutineObject, WaitTaskRescheduled, @@ -2284,8 +2285,17 @@ async def checkpoint(): :func:`checkpoint`.) """ - with CancelScope(deadline=-inf): - await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) + # The scheduler is what checks timeouts and converts them into + # cancellations. So by doing the schedule point first, we ensure that the + # cancel point has the most up-to-date info. + await cancel_shielded_checkpoint() + task = current_task() + task._cancel_points += 1 + if task._cancel_status.effectively_cancelled or ( + task is task._runner.main_task and task._runner.ki_pending + ): + with CancelScope(deadline=-inf): + await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) async def checkpoint_if_cancelled(): From 9f9d704c1b9dd3f4a2f287cad77d2ad81e4eb7ec Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 14 Jun 2020 21:04:34 -0400 Subject: [PATCH 0284/1498] Remove duplicated 'is' in contributing docs --- docs/source/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index d1723c7fc5..2d3d35b754 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -5,7 +5,7 @@ Contributing to Trio and related projects So you're interested in contributing to Trio or `one of our associated projects `__? That's awesome! Trio is -is an open-source project maintained by an informal group of +an open-source project maintained by an informal group of volunteers. Our goal is to make async I/O in Python more fun, easy, and reliable, and we can't do it without help from people like you. We welcome contributions from anyone willing to work in good faith with From 64f57ee79c5a89b764faba1050609b7d637859a8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 14 Jun 2020 21:09:45 -0400 Subject: [PATCH 0285/1498] Update Google docstring format example link --- docs/source/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index d1723c7fc5..57d1d80728 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -363,7 +363,7 @@ Documentation is hosted at `Read the Docs rebuilding it after every commit. For docstrings, we use `the Google docstring format -`__. +`__. If you add a new function or class, there's no mechanism for automatically adding that to the docs: you'll have to at least add a line like ``.. autofunction:: `` in the appropriate From ae070d715c010bec04031e98368fd6a40fdb76b3 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 14 Jun 2020 21:01:06 -0700 Subject: [PATCH 0286/1498] Add a fast path for the simplest cases of socket address resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Profiling a user-written DNS microbenchmark in gh-1595 showed that UDP sendto operations were spending a substantial amount of time in _resolve_address, which is responsible for resolving any hostname given in the sendto call. This is weird, since the benchmark wasn't using hostnames, just a raw IP address. Surprisingly, it turns out that calling getaddrinfo at all is also quite expensive, even you give it an already-resolved plain IP address so there's no work for it to do: In [1]: import socket In [2]: %timeit socket.getaddrinfo("127.0.0.1", 80) 5.84 µs ± 53.6 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each) In [3]: %timeit socket.inet_aton("127.0.0.1") 187 ns ± 1.5 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each) I thought getaddrinfo would be effectively free in this case, but apparently I was wrong. Also, this doesn't really matter for TCP connections, since they only pass through this path once per connection, but UDP goes through here on every packet, so the overhead adds up quickly. Solution: add a fast-path to _resolve_address when user's address is already resolved, so we skip all the work. On the benchmark in gh-1595 on my laptop, this PR takes us from ~7000 queries/second to ~9000 queries/second, so a ~30% speedup. The patch is a bit more complicated than I expected. There are three parts: - The fast path itself - Skipping unnecessary checkpoints in _resolve_address, including when we're on the fastpath: this is an internal function and it turns out that almost all our callers are already doing checkpoints, so there's no need to do another checkpoint inside _resolve_address. Even with the fast checkpoints from gh-1613, the fast-path+checkpoint is still only ~8000 queries/second on the DNS microbenchmark, versus ~9000 queries/second without the checkpoint. - _resolve_address used to always normalize IPv6 addresses into 4-tuples, as a side-effect of how getaddrinfo works. The fast path doesn't call getaddrinfo, so the tests needed adjusting so they don't expect this normalization, and to make sure that our tests for the getaddrinfo normalization code don't hit the fast path. --- newsfragments/1595.feature.rst | 3 ++ trio/_socket.py | 40 +++++++++++++++------- trio/tests/test_socket.py | 61 +++++++++++++++++++++------------- 3 files changed, 68 insertions(+), 36 deletions(-) create mode 100644 newsfragments/1595.feature.rst diff --git a/newsfragments/1595.feature.rst b/newsfragments/1595.feature.rst new file mode 100644 index 0000000000..b7a0df3938 --- /dev/null +++ b/newsfragments/1595.feature.rst @@ -0,0 +1,3 @@ +If you pass a raw IP address into ``sendto``, it no longer spends any +time trying to resolve the hostname. If you're using UDP, this should +substantially reduce your per-packet overhead. diff --git a/trio/_socket.py b/trio/_socket.py index ccd2a6ff20..a2ca21afde 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -447,7 +447,7 @@ def close(self): self._sock.close() async def bind(self, address): - address = await self._resolve_local_address(address) + address = await self._resolve_local_address_nocp(address) if ( hasattr(_stdlib_socket, "AF_UNIX") and self.family == _stdlib_socket.AF_UNIX @@ -461,6 +461,7 @@ async def bind(self, address): # complete asynchronously, like connect. But in practice AFAICT # there aren't yet any real systems that do this, so we'll worry # about it when it happens. + await trio.lowlevel.checkpoint() return self._sock.bind(address) def shutdown(self, flag): @@ -489,7 +490,9 @@ async def wait_writable(self): # Take an address in Python's representation, and returns a new address in # the same representation, but with names resolved to numbers, # etc. - async def _resolve_address(self, address, flags): + # + # NOTE: this function does not always checkpoint + async def _resolve_address_nocp(self, address, flags): # Do some pre-checking (or exit early for non-IP sockets) if self._sock.family == _stdlib_socket.AF_INET: if not isinstance(address, tuple) or not len(address) == 2: @@ -500,14 +503,23 @@ async def _resolve_address(self, address, flags): "address should be a (host, port, [flowinfo, [scopeid]]) tuple" ) elif self._sock.family == _stdlib_socket.AF_UNIX: - await trio.lowlevel.checkpoint() # unwrap path-likes return _os.fspath(address) - else: - await trio.lowlevel.checkpoint() return address + + # -- From here on we know we have IPv4 or IPV6 -- host, port, *_ = address + # Fast path for the simple case: already-resolved IP address, + # already-resolved port. This is particularly important for UDP, since + # every sendto call goes through here. + if isinstance(port, int): + try: + _stdlib_socket.inet_pton(self._sock.family, address[0]) + except (OSError, TypeError): + pass + else: + return address # Special cases to match the stdlib, see gh-277 if host == "": host = None @@ -544,12 +556,16 @@ async def _resolve_address(self, address, flags): return normed # Returns something appropriate to pass to bind() - async def _resolve_local_address(self, address): - return await self._resolve_address(address, _stdlib_socket.AI_PASSIVE) + # + # NOTE: this function does not always checkpoint + async def _resolve_local_address_nocp(self, address): + return await self._resolve_address_nocp(address, _stdlib_socket.AI_PASSIVE) # Returns something appropriate to pass to connect()/sendto()/sendmsg() - async def _resolve_remote_address(self, address): - return await self._resolve_address(address, 0) + # + # NOTE: this function does not always checkpoint + async def _resolve_remote_address_nocp(self, address): + return await self._resolve_address_nocp(address, 0) async def _nonblocking_helper(self, fn, args, kwargs, wait_fn): # We have to reconcile two conflicting goals: @@ -606,7 +622,7 @@ async def connect(self, address): # notification. This means it isn't really cancellable... we close the # socket if cancelled, to avoid confusion. try: - address = await self._resolve_remote_address(address) + address = await self._resolve_remote_address_nocp(address) async with _try_sync(): # An interesting puzzle: can a non-blocking connect() return EINTR # (= raise InterruptedError)? PEP 475 specifically left this as @@ -732,7 +748,7 @@ async def sendto(self, *args): # args is: data[, flags], address) # and kwargs are not accepted args = list(args) - args[-1] = await self._resolve_remote_address(args[-1]) + args[-1] = await self._resolve_remote_address_nocp(args[-1]) return await self._nonblocking_helper( _stdlib_socket.socket.sendto, args, {}, _core.wait_writable ) @@ -755,7 +771,7 @@ async def sendmsg(self, *args): # and kwargs are not accepted if len(args) == 4 and args[-1] is not None: args = list(args) - args[-1] = await self._resolve_remote_address(args[-1]) + args[-1] = await self._resolve_remote_address_nocp(args[-1]) return await self._nonblocking_helper( _stdlib_socket.socket.sendmsg, args, {}, _core.wait_writable ) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 7e4c77ddbe..6bd00930b8 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -452,7 +452,6 @@ class Addresses: localhost = attr.ib() arbitrary = attr.ib() broadcast = attr.ib() - extra = attr.ib() # Direct thorough tests of the implicit resolver helpers @@ -466,7 +465,6 @@ class Addresses: localhost="127.0.0.1", arbitrary="1.2.3.4", broadcast="255.255.255.255", - extra=(), ), ), pytest.param( @@ -476,7 +474,6 @@ class Addresses: localhost="::1", arbitrary="1::2", broadcast="::ffff:255.255.255.255", - extra=(0, 0), ), marks=creates_ipv6, ), @@ -485,42 +482,58 @@ class Addresses: async def test_SocketType_resolve(socket_type, addrs): v6 = socket_type == tsocket.AF_INET6 + def pad(addr): + if v6: + while len(addr) < 4: + addr += (0,) + return addr + + def assert_eq(actual, expected): + assert pad(expected) == pad(actual) + with tsocket.socket(family=socket_type) as sock: # For some reason the stdlib special-cases "" to pass NULL to # getaddrinfo They also error out on None, but whatever, None is much # more consistent, so we accept it too. for null in [None, ""]: - got = await sock._resolve_local_address((null, 80)) - assert got == (addrs.bind_all, 80, *addrs.extra) - got = await sock._resolve_remote_address((null, 80)) - assert got == (addrs.localhost, 80, *addrs.extra) + got = await sock._resolve_local_address_nocp((null, 80)) + assert_eq(got, (addrs.bind_all, 80)) + got = await sock._resolve_remote_address_nocp((null, 80)) + assert_eq(got, (addrs.localhost, 80)) # AI_PASSIVE only affects the wildcard address, so for everything else - # _resolve_local_address and _resolve_remote_address should work the same: - for resolver in ["_resolve_local_address", "_resolve_remote_address"]: + # _resolve_local_address_nocp and _resolve_remote_address_nocp should + # work the same: + for resolver in ["_resolve_local_address_nocp", "_resolve_remote_address_nocp"]: async def res(*args): return await getattr(sock, resolver)(*args) - assert await res((addrs.arbitrary, "http")) == ( - addrs.arbitrary, - 80, - *addrs.extra, - ) + assert_eq(await res((addrs.arbitrary, "http")), (addrs.arbitrary, 80,)) if v6: - assert await res(("1::2", 80, 1)) == ("1::2", 80, 1, 0) - assert await res(("1::2", 80, 1, 2)) == ("1::2", 80, 1, 2) + # Check handling of different length ipv6 address tuples + assert_eq(await res(("1::2", 80)), ("1::2", 80, 0, 0)) + assert_eq(await res(("1::2", 80, 0)), ("1::2", 80, 0, 0)) + assert_eq(await res(("1::2", 80, 0, 0)), ("1::2", 80, 0, 0)) + # Non-zero flowinfo/scopeid get passed through + assert_eq(await res(("1::2", 80, 1)), ("1::2", 80, 1, 0)) + assert_eq(await res(("1::2", 80, 1, 2)), ("1::2", 80, 1, 2)) + + # And again with a string port, as a trick to avoid the + # already-resolved address fastpath and make sure we call + # getaddrinfo + assert_eq(await res(("1::2", "80")), ("1::2", 80, 0, 0)) + assert_eq(await res(("1::2", "80", 0)), ("1::2", 80, 0, 0)) + assert_eq(await res(("1::2", "80", 0, 0)), ("1::2", 80, 0, 0)) + assert_eq(await res(("1::2", "80", 1)), ("1::2", 80, 1, 0)) + assert_eq(await res(("1::2", "80", 1, 2)), ("1::2", 80, 1, 2)) # V4 mapped addresses resolved if V6ONLY is False sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, False) - assert await res(("1.2.3.4", "http")) == ("::ffff:1.2.3.4", 80, 0, 0,) + assert_eq(await res(("1.2.3.4", "http")), ("::ffff:1.2.3.4", 80)) # Check the special case, because why not - assert await res(("", 123)) == ( - addrs.broadcast, - 123, - *addrs.extra, - ) + assert_eq(await res(("", 123)), (addrs.broadcast, 123,)) # But not if it's true (at least on systems where getaddrinfo works # correctly) @@ -711,11 +724,11 @@ async def test_resolve_remote_address_exception_closes_socket(): with _core.CancelScope() as cancel_scope: with tsocket.socket() as sock: - async def _resolve_remote_address(self, *args, **kwargs): + async def _resolve_remote_address_nocp(self, *args, **kwargs): cancel_scope.cancel() await _core.checkpoint() - sock._resolve_remote_address = _resolve_remote_address + sock._resolve_remote_address_nocp = _resolve_remote_address_nocp with assert_checkpoints(): with pytest.raises(_core.Cancelled): await sock.connect("") From 702e03c50cf79eebda89fc83c98576bf38b4d2b0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2020 06:48:08 +0000 Subject: [PATCH 0287/1498] Bump pytest-cov from 2.9.0 to 2.10.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.9.0 to 2.10.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.9.0...v2.10.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4f39cc066d..485551cef1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,7 +43,7 @@ pygments==2.6.1 # via ipython pylint==2.5.3 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.9.0 # via -r test-requirements.in +pytest-cov==2.10.0 # via -r test-requirements.in pytest==5.4.3 # via -r test-requirements.in, pytest-cov regex==2020.6.8 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets From 63e14f45e59984dd628019605710fe7c19aee0e0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2020 06:48:53 +0000 Subject: [PATCH 0288/1498] Bump sphinx from 3.0.4 to 3.1.1 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.0.4 to 3.1.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.0.4...v3.1.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 3c8509536a..9e132f07af 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -29,7 +29,7 @@ sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.2.2 # via -r docs-requirements.in sphinx-rtd-theme==0.4.3 # via -r docs-requirements.in -sphinx==3.0.4 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx==3.1.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 4a7c865daf83cf91afa1bc45390c021cb771389a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2020 07:38:17 +0000 Subject: [PATCH 0289/1498] Bump more-itertools from 8.3.0 to 8.4.0 Bumps [more-itertools](https://github.com/more-itertools/more-itertools) from 8.3.0 to 8.4.0. - [Release notes](https://github.com/more-itertools/more-itertools/releases) - [Commits](https://github.com/more-itertools/more-itertools/compare/v8.3.0...v8.4.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 485551cef1..3bcd393e63 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ isort==4.3.21 # via pylint jedi==0.17.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint -more-itertools==8.3.0 # via pytest +more-itertools==8.4.0 # via pytest outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest parso==0.7.0 # via jedi From 74f905381c7068d5ec64163273c3b91b203a72eb Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Tue, 16 Jun 2020 10:42:48 +0900 Subject: [PATCH 0290/1498] add pura to Trio libraries list --- docs/source/awesome-trio-libraries.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 5e1f828e7c..bf91a55219 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -33,6 +33,7 @@ Web and HTML * `hypercorn `__ - An HTTP server for hosting your ASGI apps. Supports HTTP/1.1, HTTP/2, HTTP/3, and Websockets. Can be run as a standalone server, or embedded in a larger Trio app. Use it with ``quart-trio``, or any other Trio-compatible ASGI framework. * `httpx `__ - HTTPX is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. * `DeFramed `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi `__ GUI library. +* `pura `__ - A simple web framework for embedding realtime graphical visualization into Trio apps, enabling inspection and manipulation of program state during development. Database From 1d9d0cafce94f8b92d98515028310551b7b78e58 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 16 Jun 2020 00:44:16 -0700 Subject: [PATCH 0291/1498] In KqueueIOManager, close the force-wakeup fd This small bug was introduced in gh-1551 Fixes gh-1621 --- newsfragments/1621.fix.rst | 2 ++ trio/_core/_io_kqueue.py | 1 + 2 files changed, 3 insertions(+) create mode 100644 newsfragments/1621.fix.rst diff --git a/newsfragments/1621.fix.rst b/newsfragments/1621.fix.rst new file mode 100644 index 0000000000..2eb869c969 --- /dev/null +++ b/newsfragments/1621.fix.rst @@ -0,0 +1,2 @@ +On macOS and BSDs, explicitly close our wakeup socketpair when we're +done with it. diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index d2d80a7341..be8e9f1819 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -44,6 +44,7 @@ def statistics(self): def close(self): self._kqueue.close() + self._force_wakeup.close() def force_wakeup(self): self._force_wakeup.wakeup_thread_and_signal_safe() From f05e74eb149e41246270f75c3cf95857e21e9d1b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2020 09:01:42 +0000 Subject: [PATCH 0292/1498] Bump py from 1.8.1 to 1.8.2 Bumps [py](https://github.com/pytest-dev/py) from 1.8.1 to 1.8.2. - [Release notes](https://github.com/pytest-dev/py/releases) - [Changelog](https://github.com/pytest-dev/py/blob/master/CHANGELOG) - [Commits](https://github.com/pytest-dev/py/compare/1.8.1...1.8.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3bcd393e63..a083a98eb6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -35,7 +35,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest prompt-toolkit==3.0.5 # via ipython ptyprocess==0.6.0 # via pexpect -py==1.8.1 # via pytest +py==1.8.2 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 From 14d64088f5e92cfdf86bf69a07a41f1e5b1c2ab0 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 16 Jun 2020 14:44:02 +0400 Subject: [PATCH 0293/1498] Remove now-unused .yapfignore file We switched to Black. --- .yapfignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .yapfignore diff --git a/.yapfignore b/.yapfignore deleted file mode 100644 index 677b760f4c..0000000000 --- a/.yapfignore +++ /dev/null @@ -1 +0,0 @@ -**/_generated* From 16177477557076ececf2f10f9e8797399b58d56f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 18 Jun 2020 06:37:30 +0000 Subject: [PATCH 0294/1498] Bump requests from 2.23.0 to 2.24.0 Bumps [requests](https://github.com/psf/requests) from 2.23.0 to 2.24.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.23.0...v2.24.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 9e132f07af..eb28f5d17f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -23,7 +23,7 @@ packaging==20.4 # via sphinx pygments==2.6.1 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.1 # via babel -requests==2.23.0 # via sphinx +requests==2.24.0 # via sphinx six==1.15.0 # via packaging sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx From beaea20ff24d8a8682e3a9b6510f7435e3ea4c03 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 18 Jun 2020 06:38:06 +0000 Subject: [PATCH 0295/1498] Bump sphinx-rtd-theme from 0.4.3 to 0.5.0 Bumps [sphinx-rtd-theme](https://github.com/rtfd/sphinx_rtd_theme) from 0.4.3 to 0.5.0. - [Release notes](https://github.com/rtfd/sphinx_rtd_theme/releases) - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/rtfd/sphinx_rtd_theme/compare/0.4.3...0.5.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 9e132f07af..e890b9a60d 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -28,8 +28,8 @@ six==1.15.0 # via packaging sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.2.2 # via -r docs-requirements.in -sphinx-rtd-theme==0.4.3 # via -r docs-requirements.in sphinx==3.1.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx_rtd_theme==0.5.0 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From de9371988bb8cef49b925cfd8c541efe8fe207fd Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 18 Jun 2020 21:22:26 +0400 Subject: [PATCH 0296/1498] Add newsfragment --- newsfragments/1596.deprecated.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1596.deprecated.rst diff --git a/newsfragments/1596.deprecated.rst b/newsfragments/1596.deprecated.rst new file mode 100644 index 0000000000..9736419485 --- /dev/null +++ b/newsfragments/1596.deprecated.rst @@ -0,0 +1 @@ +Remove ``wait_socket_*``, ``notify_socket_closing``, ``notify_fd_closing``, ``run_sync_in_worker_thread`` and ``current_default_worker_thread_limiter``. They were deprecated in 0.12.0. From a47fbc7ba3bfcb92f98354e76fd015892e83e458 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 20 Jun 2020 03:11:19 -0700 Subject: [PATCH 0297/1498] Switch to heap-based deadline tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Based on Tronic's work here: https://github.com/python-trio/trio/pull/1351 On my laptop, this gives a ~35% increase in requests-per-second in their microbenchmark: https://github.com/python-trio/trio/pull/1351#issuecomment-581390193 Co-authored-by: L. Kärkkäinen --- trio/_core/_run.py | 89 +++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 127657fc85..7e768b7e63 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -20,6 +20,7 @@ from sniffio import current_async_library_cvar import attr +from heapq import heapify, heappop, heappush from sortedcontainers import SortedDict from outcome import Error, Value, capture @@ -123,6 +124,55 @@ class IdlePrimedTypes(enum.Enum): ################################################################ +@attr.s(eq=False, slots=True) +class Deadlines: + """A container of deadlined cancel scopes. + + Only contains scopes with non-infinite deadlines that are currently + attached to at least one task. + + """ + + # Heap of (deadline, id(CancelScope), CancelScope) + _heap = attr.ib(factory=list) + # Count of active deadlines (those that haven't been changed) + _active = attr.ib(default=0) + + def add(self, deadline, cancel_scope): + heappush(self._heap, (deadline, id(cancel_scope), cancel_scope)) + self._active += 1 + + def remove(self, deadline, cancel_scope): + self._active -= 1 + + def next_deadline(self): + while self._heap: + deadline, _, cancel_scope = self._heap[0] + if deadline == cancel_scope._registered_deadline: + return deadline + else: + # This entry is stale; discard it and try again + heappop(self._heap) + return inf + + def expire(self, now): + did_something = False + while self._heap and self._heap[0][0] <= now: + deadline, _, cancel_scope = heappop(self._heap) + if deadline == cancel_scope._registered_deadline: + did_something = True + # This implicitly calls self.remove(), so we don't need to + # decrement _active here + cancel_scope.cancel() + # If we've accumulated too many stale entries, then prune the heap to + # keep it under control. (We only do this occasionally in a batch, to + # keep the amortized cost down) + if len(self._heap) > self._active * 2 + 1000: + self._heap = [t for t in self._heap if t[0] == t[2]._registered_deadline] + heapify(self._heap) + return did_something + + @attr.s(eq=False, slots=True) class CancelStatus: """Tracks the cancellation status for a contiguous extent @@ -518,13 +568,13 @@ def _might_change_registered_deadline(self): self._registered_deadline = new runner = GLOBAL_RUN_CONTEXT.runner if runner.is_guest: - old_next_deadline = runner.next_deadline() + old_next_deadline = runner.deadlines.next_deadline() if old != inf: - del runner.deadlines[old, id(self)] + runner.deadlines.remove(old, self) if new != inf: - runner.deadlines[new, id(self)] = self + runner.deadlines.add(new, self) if runner.is_guest: - new_next_deadline = runner.next_deadline() + new_next_deadline = runner.deadlines.next_deadline() if old_next_deadline != new_next_deadline: runner.force_guest_tick_asap() @@ -1201,10 +1251,7 @@ class Runner: runq = attr.ib(factory=deque) tasks = attr.ib(factory=set) - # {(deadline, id(CancelScope)): CancelScope} - # only contains scopes with non-infinite deadlines that are currently - # attached to at least one task - deadlines = attr.ib(factory=SortedDict) + deadlines = attr.ib(factory=Deadlines) init_task = attr.ib(default=None) system_nursery = attr.ib(default=None) @@ -1236,14 +1283,6 @@ def close(self): # This is where KI protection gets disabled, so we do it last self.ki_manager.close() - def next_deadline(self): - try: - (next_deadline, _), _ = self.deadlines.peekitem(0) - except IndexError: - return inf - else: - return next_deadline - @_public def current_statistics(self): """Returns an object containing run-loop-level debugging information. @@ -1268,7 +1307,7 @@ def current_statistics(self): other attributes vary between backends. """ - seconds_to_next_deadline = self.next_deadline() - self.current_time() + seconds_to_next_deadline = self.deadlines.next_deadline() - self.current_time() return _RunStatistics( tasks_living=len(self.tasks), tasks_runnable=len(self.runq), @@ -2009,11 +2048,9 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): while runner.tasks: if runner.runq: timeout = 0 - elif runner.deadlines: - deadline, _ = runner.deadlines.keys()[0] - timeout = runner.clock.deadline_to_sleep_time(deadline) else: - timeout = _MAX_TIMEOUT + deadline = runner.deadlines.next_deadline() + timeout = runner.clock.deadline_to_sleep_time(deadline) timeout = min(max(0, timeout), _MAX_TIMEOUT) idle_primed = None @@ -2042,14 +2079,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # Process cancellations due to deadline expiry now = runner.clock.current_time() - while runner.deadlines: - (deadline, _), cancel_scope = runner.deadlines.peekitem(0) - if deadline <= now: - # This removes the given scope from runner.deadlines: - cancel_scope.cancel() - idle_primed = None - else: - break + if runner.deadlines.expire(now): + idle_primed = None # idle_primed != None means: if the IO wait hit the timeout, and # still nothing is happening, then we should start waking up From addc08436147e093cbcb26ea0772b17e2f76eadd Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 20 Jun 2020 03:44:22 -0700 Subject: [PATCH 0298/1498] Make deadline pruning more reliable --- trio/_core/_run.py | 28 +++++++++++++++++++++++++--- trio/_core/tests/test_run.py | 12 ++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 7e768b7e63..2dcb18e083 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -45,6 +45,8 @@ from .._deprecate import warn_deprecated from .._util import Final, NoPublicConstructor, coroutine_or_error +DEADLINE_HEAP_MIN_PRUNE_THRESHOLD = 1000 + _NO_SEND = object() @@ -155,6 +157,27 @@ def next_deadline(self): heappop(self._heap) return inf + def _prune(self): + # In principle, it's possible for a cancel scope to toggle back and + # forth repeatedly between the same two deadlines, and end up with + # lots of stale entries that *look* like they're still active, because + # their deadline is correct, but in fact are redundant. So when + # pruning we have to eliminate entries with the wrong deadline, *and* + # eliminate duplicates. + seen = set() + pruned_heap = [] + for deadline, tiebreaker, cancel_scope in self._heap: + if deadline == cancel_scope._registered_deadline: + if cancel_scope in seen: + continue + seen.add(cancel_scope) + pruned_heap.append((deadline, tiebreaker, cancel_scope)) + # See test_cancel_scope_deadline_duplicates for a test that exercises + # this assert: + assert len(pruned_heap) == self._active + heapify(pruned_heap) + self._heap = pruned_heap + def expire(self, now): did_something = False while self._heap and self._heap[0][0] <= now: @@ -167,9 +190,8 @@ def expire(self, now): # If we've accumulated too many stale entries, then prune the heap to # keep it under control. (We only do this occasionally in a batch, to # keep the amortized cost down) - if len(self._heap) > self._active * 2 + 1000: - self._heap = [t for t in self._heap if t[0] == t[2]._registered_deadline] - heapify(self._heap) + if len(self._heap) > self._active * 2 + DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: + self._prune() return did_something diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 78b46b7adc..ce1dc2eeb4 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -23,6 +23,7 @@ ) from ... import _core +from .._run import DEADLINE_HEAP_MIN_PRUNE_THRESHOLD from ..._threads import to_thread_run_sync from ..._timeouts import sleep, fail_after from ...testing import ( @@ -2351,3 +2352,14 @@ async def test_very_deep_cancel_scope_nesting(): for _ in range(5000): exit_stack.enter_context(_core.CancelScope()) outermost_scope.cancel() + + +async def test_cancel_scope_deadline_duplicates(): + # This exercises an assert in Deadlines._prune, by intentionally creating + # duplicate entries in the deadline heap. + now = _core.current_time() + with _core.CancelScope() as cscope: + for _ in range(DEADLINE_HEAP_MIN_PRUNE_THRESHOLD * 2): + cscope.deadline = now + 9998 + cscope.deadline = now + 9999 + await sleep(0.01) From aab851d7f565e6f8fcd1a1f21ed62506d270132c Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 20 Jun 2020 04:17:24 -0700 Subject: [PATCH 0299/1498] Add newsfragment --- newsfragments/1629.feature.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/1629.feature.rst diff --git a/newsfragments/1629.feature.rst b/newsfragments/1629.feature.rst new file mode 100644 index 0000000000..6047ae4ccf --- /dev/null +++ b/newsfragments/1629.feature.rst @@ -0,0 +1,2 @@ +We switched to a new, lower-overhead data structure to track upcoming +timeouts, which should make your programs faster. From c999d87216ef6c95ae59e11cae94368a69c55b79 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 20 Jun 2020 21:10:12 -0400 Subject: [PATCH 0300/1498] Correct file I/O 'jump down to api' doc link --- docs/source/reference-io.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index 7c700b1328..ef971f6ea8 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -472,8 +472,7 @@ people switch to async I/O, and then they're surprised and confused when they find it doesn't speed up their program. The next section explains the theory behind async file I/O, to help you better understand your code's behavior. Or, if you just want to get started, -you can `jump down to the API overview -`__. +you can :ref:`jump down to the API overview `. Background: Why is async file I/O useful? The answer may surprise you From c1fa36f2c62e99a9c3d890939f1028de8d70cdec Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 21 Jun 2020 15:47:37 -0400 Subject: [PATCH 0301/1498] Start an RPC list Add the grpc and jsonrpc libs. Slight tweak to the `tractor` description. --- docs/source/awesome-trio-libraries.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index bf91a55219..479f74943d 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -62,10 +62,16 @@ Building Command Line Apps Multi-Core/Multiprocessing -------------------------- -* `tractor `__ - tractor is an attempt to bring trionic structured concurrency to distributed multi-core Python. +* `tractor `__ - An experimental, trionic (aka structured concurrent) "actor model" for distributed multi-core Python. * `Trio run_in_process `__ - Trio based API for running code in a separate process. +RPC +--- +* `purepc `__ - Asynchronous pure Python gRPC client and server implementation using anyio. +* `trio-jsonrpc `__ - JSON-RPC v2.0 for Trio. + + Testing ------- * `pytest-trio `__ - Pytest plugin for trio. From 5ce4a1c0b16446f2d2fc4916ee1b7e035e180b60 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 22 Jun 2020 10:02:53 +0400 Subject: [PATCH 0302/1498] Fix unrolled_run_next_send mypy error mypy thought the lambda was returning "object". --- trio/_core/_run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 57982a33cc..34ad13a299 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -16,7 +16,7 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Callable from sniffio import current_async_library_cvar @@ -1151,7 +1151,8 @@ class GuestState: run_sync_soon_not_threadsafe = attr.ib() done_callback = attr.ib() unrolled_run_gen = attr.ib() - unrolled_run_next_send = attr.ib(factory=lambda: Value(None), type=object) + _value_factory: Callable[[], Value] = lambda: Value(None) + unrolled_run_next_send = attr.ib(factory=_value_factory, type=Value) def guest_tick(self): try: From c570ccc6ec7df378f498729596ac78bad0e3647f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 22 Jun 2020 10:35:20 +0400 Subject: [PATCH 0303/1498] Stop using RawGit workaround The original issue was fixed by GitHub. --- README.rst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/README.rst b/README.rst index d85cd64173..cc200bbcca 100644 --- a/README.rst +++ b/README.rst @@ -25,17 +25,7 @@ Trio – a friendly Python library for async concurrency and I/O ============================================================== -.. Github carefully breaks rendering of SVG directly out of the repo, - so we have to redirect through cdn.rawgit.com - See: - https://github.com/isaacs/github/issues/316 - https://github.com/github/markup/issues/556#issuecomment-288581799 - I also tried rendering to PNG and linking to that locally, which - "works" in that it displays the image, but for some reason it - ignores the width and align directives, so it's actually pretty - useless... - -.. image:: https://cdn.rawgit.com/python-trio/trio/9b0bec646a31e0d0f67b8b6ecc6939726faf3e17/logo/logo-with-background.svg +.. image:: https://raw.githubusercontent.com/python-trio/trio/9b0bec646a31e0d0f67b8b6ecc6939726faf3e17/logo/logo-with-background.svg :width: 200px :align: right From 1a9fdee11448d1ea910bdacbfa120768c95a1643 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2020 07:10:55 +0000 Subject: [PATCH 0304/1498] Bump certifi from 2020.4.5.2 to 2020.6.20 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.4.5.2 to 2020.6.20. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.04.05.2...2020.06.20) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 747c623c5f..f4aa519b3f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in attrs==19.3.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx -certifi==2020.4.5.2 # via requests +certifi==2020.6.20 # via requests chardet==3.0.4 # via requests click==7.1.2 # via towncrier docutils==0.16 # via sphinx @@ -28,8 +28,8 @@ six==1.15.0 # via packaging sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.2.2 # via -r docs-requirements.in +sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in sphinx==3.1.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio -sphinx_rtd_theme==0.5.0 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From b1bcbda996e15a4eb3bf0211bd1a4b50fe9abd16 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2020 07:11:25 +0000 Subject: [PATCH 0305/1498] Bump jedi from 0.17.0 to 0.17.1 Bumps [jedi](https://github.com/davidhalter/jedi) from 0.17.0 to 0.17.1. - [Release notes](https://github.com/davidhalter/jedi/releases) - [Changelog](https://github.com/davidhalter/jedi/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/jedi/compare/v0.17.0...v0.17.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a083a98eb6..2ab7ff9650 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.15.0 # via -r test-requirements.in isort==4.3.21 # via pylint -jedi==0.17.0 # via -r test-requirements.in, ipython +jedi==0.17.1 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint more-itertools==8.4.0 # via pytest From 2cc70a16deff3d425370e5229b5363250517a2e1 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 22 Jun 2020 11:47:49 +0400 Subject: [PATCH 0306/1498] Fix type annotation in _core._run The value can be an Error too, so we should use Outcome, not Value. --- trio/_core/_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 080ce553f5..31f9a3089a 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -23,7 +23,7 @@ import attr from heapq import heapify, heappop, heappush from sortedcontainers import SortedDict -from outcome import Error, Value, capture +from outcome import Error, Outcome, Value, capture from ._entry_queue import EntryQueue, TrioToken from ._exceptions import TrioInternalError, RunFinishedError, Cancelled @@ -1225,7 +1225,7 @@ class GuestState: done_callback = attr.ib() unrolled_run_gen = attr.ib() _value_factory: Callable[[], Value] = lambda: Value(None) - unrolled_run_next_send = attr.ib(factory=_value_factory, type=Value) + unrolled_run_next_send = attr.ib(factory=_value_factory, type=Outcome) def guest_tick(self): try: From 0efb647c500b8030557e292990b8a351dc2454a2 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 22 Jun 2020 13:16:54 +0400 Subject: [PATCH 0307/1498] Fix typo (epoll -> kqueue) --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 31f9a3089a..1debde4a0c 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2385,7 +2385,7 @@ async def checkpoint_if_cancelled(): elif ( sys.platform == "darwin" or sys.platform.startswith("freebsd") - or (not TYPE_CHECKING and hasattr(select, "epoll")) + or (not TYPE_CHECKING and hasattr(select, "kqueue")) ): from ._io_kqueue import KqueueIOManager as TheIOManager from ._generated_io_kqueue import * From 70491469202b7ea03b2a610408098e3e9f96fc6b Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Mon, 22 Jun 2020 14:48:21 +0200 Subject: [PATCH 0308/1498] Don't hold a reference to the last job in the thread cache --- trio/_core/_thread_cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index 8faf554322..ae5e8450b9 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -69,6 +69,8 @@ def _work(self): # instead of spawning a new thread. self._thread_cache._idle_workers[self] = None deliver(result) + del fn + del deliver else: # Timeout acquiring lock, so we can probably exit. But, # there's a race condition: we might be assigned a job *just* From 699957a4668d96abdc470dae8e7424294f88db46 Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Tue, 23 Jun 2020 07:01:55 +0200 Subject: [PATCH 0309/1498] Add newsfragment --- newsfragments/1638.fix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1638.fix.rst diff --git a/newsfragments/1638.fix.rst b/newsfragments/1638.fix.rst new file mode 100644 index 0000000000..6c6a9a6a5d --- /dev/null +++ b/newsfragments/1638.fix.rst @@ -0,0 +1 @@ +The thread cache didn't release its reference to the previous job. From 63c4b780ff7cc43dd11fbfa882f7f1aaa8ee9774 Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Tue, 23 Jun 2020 07:11:53 +0200 Subject: [PATCH 0310/1498] Add test --- trio/_core/tests/test_thread_cache.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/test_thread_cache.py b/trio/_core/tests/test_thread_cache.py index f19ac1d4cc..0f6e0a0715 100644 --- a/trio/_core/tests/test_thread_cache.py +++ b/trio/_core/tests/test_thread_cache.py @@ -4,7 +4,7 @@ import time import sys -from .tutil import slow +from .tutil import slow, gc_collect_harder from .. import _thread_cache from .._thread_cache import start_thread_soon, ThreadCache @@ -25,6 +25,29 @@ def deliver(outcome): outcome.unwrap() +def test_thread_cache_deref(): + res = [False] + + class del_me: + def __call__(self): + return 42 + + def __del__(self): + res[0] = True + + q = Queue() + + def deliver(outcome): + q.put(outcome) + + start_thread_soon(del_me(), deliver) + outcome = q.get() + assert outcome.unwrap() == 42 + + gc_collect_harder() + assert res[0] + + @slow def test_spawning_new_thread_from_deliver_reuses_starting_thread(): # We know that no-one else is using the thread cache, so if we keep From 81f83f5b09b1e1e474b60f04a43afa2587248fd6 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 23 Jun 2020 10:25:05 +0400 Subject: [PATCH 0311/1498] Reuse mypy checks for kqueue/select detection An added benefit is that we currently don't need to look at TYPE_CHECKING or add ugly pragmas. This might change when we run mypy on more platforms. --- trio/_core/__init__.py | 11 +++-------- trio/_core/_run.py | 10 +++------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 9ba611dcce..7d57d9c4d1 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -4,7 +4,6 @@ are publicly available in either trio, trio.lowlevel, or trio.testing. """ -import typing as _t import sys from ._exceptions import ( @@ -76,13 +75,6 @@ from ._mock_clock import MockClock -# Kqueue imports -if not _t.TYPE_CHECKING: # pragma: no branch - try: - from ._run import current_kqueue, monitor_kevent, wait_kevent - except ImportError: - pass - # Windows imports if sys.platform == "win32": from ._run import ( @@ -93,3 +85,6 @@ write_overlapped, readinto_overlapped, ) +# Kqueue imports +elif sys.platform != "linux" and sys.platform != "win32": + from ._run import current_kqueue, monitor_kevent, wait_kevent diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 1debde4a0c..e2ccb194ad 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -16,7 +16,7 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import TYPE_CHECKING, Any, Callable +from typing import Callable from sniffio import current_async_library_cvar @@ -2379,14 +2379,10 @@ async def checkpoint_if_cancelled(): if sys.platform == "win32": from ._io_windows import WindowsIOManager as TheIOManager from ._generated_io_windows import * -elif sys.platform == "linux" or (not TYPE_CHECKING and hasattr(select, "epoll")): +elif sys.platform == "linux" or hasattr(select, "epoll"): from ._io_epoll import EpollIOManager as TheIOManager from ._generated_io_epoll import * -elif ( - sys.platform == "darwin" - or sys.platform.startswith("freebsd") - or (not TYPE_CHECKING and hasattr(select, "kqueue")) -): +elif hasattr(select, "kqueue"): from ._io_kqueue import KqueueIOManager as TheIOManager from ._generated_io_kqueue import * else: # pragma: no cover From 049e716e79cb7cf57a63bacb89c799e663c005c0 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 23 Jun 2020 10:37:01 +0400 Subject: [PATCH 0312/1498] Remove docs check We're now testing docs via the ReadTheDocs pull request integration. --- .github/workflows/ci.yml | 5 ----- ci.sh | 9 +-------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 963f9e1531..9592be955a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,13 +51,9 @@ jobs: fail-fast: false matrix: python: ['3.6', '3.7', '3.8'] - check_docs: ['0'] check_formatting: ['0'] extra_name: [''] include: - - python: '3.8' - check_docs: '1' - extra_name: ', check docs' - python: '3.8' check_formatting: '1' extra_name: ', check formatting' @@ -71,7 +67,6 @@ jobs: - name: Run tests run: ./ci.sh env: - CHECK_DOCS: '${{ matrix.check_docs }}' CHECK_FORMATTING: '${{ matrix.check_formatting }}' # Should match 'name:' up above JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' diff --git a/ci.sh b/ci.sh index 73a3d8d0a9..87fa3c4d77 100755 --- a/ci.sh +++ b/ci.sh @@ -393,14 +393,7 @@ python -m pip --version python setup.py sdist --formats=zip python -m pip install dist/*.zip -if [ "$CHECK_DOCS" = "1" ]; then - python -m pip install -r docs-requirements.txt - towncrier --yes # catch errors in newsfragments - cd docs - # -n (nit-picky): warn on missing references - # -W: turn warnings into errors - sphinx-build -nW -b html source build -elif [ "$CHECK_FORMATTING" = "1" ]; then +if [ "$CHECK_FORMATTING" = "1" ]; then python -m pip install -r test-requirements.txt source check.sh else From abc2b8dfa5fdaf47200f3c61b7b14a35f29524f7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2020 11:27:48 +0000 Subject: [PATCH 0313/1498] Bump wcwidth from 0.2.4 to 0.2.5 Bumps [wcwidth](https://github.com/jquast/wcwidth) from 0.2.4 to 0.2.5. - [Release notes](https://github.com/jquast/wcwidth/releases) - [Commits](https://github.com/jquast/wcwidth/compare/0.2.4...0.2.5) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2ab7ff9650..460608e09d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,5 +53,5 @@ toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black -wcwidth==0.2.4 # via prompt-toolkit, pytest +wcwidth==0.2.5 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid From 18d428863b9d4ee427d133cafa7259115dd0c2a8 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 24 Jun 2020 10:52:14 -0700 Subject: [PATCH 0314/1498] Add local_address= kwarg to open_tcp_stream Fixes gh-275 --- newsfragments/275.feature.rst | 3 + trio/_highlevel_open_tcp_stream.py | 70 ++++++++++++++++++-- trio/socket.py | 6 ++ trio/tests/test_highlevel_open_tcp_stream.py | 53 +++++++++++++++ 4 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 newsfragments/275.feature.rst diff --git a/newsfragments/275.feature.rst b/newsfragments/275.feature.rst new file mode 100644 index 0000000000..26b8ebb521 --- /dev/null +++ b/newsfragments/275.feature.rst @@ -0,0 +1,3 @@ +`trio.open_tcp_stream` has a new ``local_address=`` keyword argument +that can be used on machines with multiple IP addresses to control +which IP is used for the outgoing connection. diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 99cf8bb1c3..27b1ed0672 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -165,11 +165,7 @@ def format_host_port(host, port): # AF_INET6: "..."} # this might be simpler after async def open_tcp_stream( - host, - port, - *, - # No trailing comma b/c bpo-9232 (fixed in py36) - happy_eyeballs_delay=DEFAULT_DELAY, + host, port, *, happy_eyeballs_delay=DEFAULT_DELAY, local_address=None, ): """Connect to the given host and port over TCP. @@ -205,13 +201,30 @@ async def open_tcp_stream( Args: host (str or bytes): The host to connect to. Can be an IPv4 address, IPv6 address, or a hostname. + port (int): The port to connect to. + happy_eyeballs_delay (float): How many seconds to wait for each connection attempt to succeed or fail before getting impatient and starting another one in parallel. Set to `math.inf` if you want to limit to only one connection attempt at a time (like :func:`socket.create_connection`). Default: 0.25 (250 ms). + local_address (None or str): The local IP address or hostname to use as + the source for outgoing connections. If ``None``, we let the OS pick + the source IP. + + This is useful in some exotic networking configurations where your + host has multiple IP addresses, and you want to force the use of a + specific one. + + Note that if you pass an IPv4 ``local_address``, then you won't be + able to connect to IPv6 hosts, and vice-versa. If you want to take + advantage of this to force the use of IPv4 or IPv6 without + specifying an exact source address, you can use the IPv4 wildcard + address ``local_address="0.0.0.0"``, or the IPv6 wildcard address + ``local_address="::"``. + Returns: SocketStream: a :class:`~trio.abc.Stream` connected to the given server. @@ -269,6 +282,53 @@ async def attempt_connect(socket_args, sockaddr, attempt_failed): sock = socket(*socket_args) open_sockets.add(sock) + if local_address is not None: + # TCP connections are identified by a 4-tuple: + # + # (local IP, local port, remote IP, remote port) + # + # So if a single local IP wants to make multiple connections + # to the same (remote IP, remote port) pair, then those + # connections have to use different local ports, or else TCP + # won't be able to tell them apart. OTOH, if you have multiple + # connections to different remote IP/ports, then those + # connections can share a local port. + # + # Normally, when you call bind(), the kernel will immediately + # assign a specific local port to your socket. At this point + # the kernel doesn't know which (remote IP, remote port) + # you're going to use, so it has to pick a local port that + # *no* other connection is using. That's the only way to + # guarantee that this local port will be usable later when we + # call connect(). (Alternatively, you can set SO_REUSEADDR to + # allow multiple nascent connections to share the same port, + # but then connect() might fail with EADDRNOTAVAIL if we get + # unlucky and our TCP 4-tuple ends up colliding with another + # unrelated connection.) + # + # So calling bind() before connect() works, but it disables + # sharing of local ports. This is inefficient: it makes you + # more likely to run out of local ports. + # + # But on some versions of Linux, we can re-enable sharing of + # local ports by setting a special flag. This flag tells + # bind() to only bind the IP, and not the port. That way, + # connect() is allowed to pick the the port, and it can do a + # better job of it because it knows the remote IP/port. + try: + sock.setsockopt( + trio.socket.IPPROTO_IP, trio.socket.IP_BIND_ADDRESS_NO_PORT, 1 + ) + except (OSError, NameError): + pass + try: + await sock.bind((local_address, 0)) + except OSError: + raise OSError( + f"local_address={local_address!r} is incompatible " + f"with remote address {sockaddr}" + ) + await sock.connect(sockaddr) # Success! Save the winning socket and cancel all outstanding diff --git a/trio/socket.py b/trio/socket.py index ebbccd50ea..0f20d2f5d4 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -194,3 +194,9 @@ TCP_NOTSENT_LOWAT = 0x201 elif _sys.platform == "linux": TCP_NOTSENT_LOWAT = 25 + +try: + IP_BIND_ADDRESS_NO_PORT +except NameError: + if _sys.platform == "linux": + IP_BIND_ADDRESS_NO_PORT = 24 diff --git a/trio/tests/test_highlevel_open_tcp_stream.py b/trio/tests/test_highlevel_open_tcp_stream.py index 9fd0f3992a..c04a86d9c4 100644 --- a/trio/tests/test_highlevel_open_tcp_stream.py +++ b/trio/tests/test_highlevel_open_tcp_stream.py @@ -1,4 +1,6 @@ import pytest +import sys +import socket import attr @@ -112,6 +114,57 @@ async def test_open_tcp_stream_input_validation(): await open_tcp_stream("127.0.0.1", b"80") +def can_bind_127_0_0_2(): + with socket.socket() as s: + try: + s.bind(("127.0.0.2", 0)) + except OSError: + return False + return s.getsockname()[0] == "127.0.0.2" + + +async def test_local_address_real(): + with trio.socket.socket() as listener: + await listener.bind(("127.0.0.1", 0)) + listener.listen() + + # It's hard to test local_address properly, because you need multiple + # local addresses that you can bind to. Fortunately, on most Linux + # systems, you can bind to any 127.*.*.* address, and they all go + # through the loopback interface. So we can use a non-standard + # loopback address. On other systems, the only address we know for + # certain we have is 127.0.0.1, so we can't really test local_address= + # properly -- passing local_address=127.0.0.1 is indistinguishable + # from not passing local_address= at all. But, we can still do a smoke + # test to make sure the local_address= code doesn't crash. + if can_bind_127_0_0_2(): + local_address = "127.0.0.2" + else: + local_address = "127.0.0.1" + + async with await open_tcp_stream( + *listener.getsockname(), local_address=local_address + ) as client_stream: + assert client_stream.socket.getsockname()[0] == local_address + server_sock, remote_addr = await listener.accept() + await client_stream.aclose() + server_sock.close() + assert remote_addr[0] == local_address + + # Trying to connect to an ipv4 address with the ipv6 wildcard + # local_address should fail + with pytest.raises(OSError): + await open_tcp_stream(*listener.getsockname(), local_address="::") + + # But the ipv4 wildcard address should work + async with await open_tcp_stream( + *listener.getsockname(), local_address="0.0.0.0" + ) as client_stream: + server_sock, remote_addr = await listener.accept() + server_sock.close() + assert remote_addr == client_stream.socket.getsockname() + + # Now, thorough tests using fake sockets From 0d628cbdc615a495fa9629c30bf301dc5ed7c664 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 24 Jun 2020 11:40:31 -0700 Subject: [PATCH 0315/1498] Fix exception type --- trio/_highlevel_open_tcp_stream.py | 2 +- trio/tests/test_highlevel_open_tcp_stream.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 27b1ed0672..0ff2e7900f 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -319,7 +319,7 @@ async def attempt_connect(socket_args, sockaddr, attempt_failed): sock.setsockopt( trio.socket.IPPROTO_IP, trio.socket.IP_BIND_ADDRESS_NO_PORT, 1 ) - except (OSError, NameError): + except (OSError, AttributeError): pass try: await sock.bind((local_address, 0)) diff --git a/trio/tests/test_highlevel_open_tcp_stream.py b/trio/tests/test_highlevel_open_tcp_stream.py index c04a86d9c4..bcd3ef7f5a 100644 --- a/trio/tests/test_highlevel_open_tcp_stream.py +++ b/trio/tests/test_highlevel_open_tcp_stream.py @@ -146,6 +146,11 @@ async def test_local_address_real(): *listener.getsockname(), local_address=local_address ) as client_stream: assert client_stream.socket.getsockname()[0] == local_address + if hasattr(trio.socket, "IP_BIND_ADDRESS_NO_PORT"): + assert client_stream.socket.getsockopt( + trio.socket.IPPROTO_IP, trio.socket.IP_BIND_ADDRESS_NO_PORT + ) + server_sock, remote_addr = await listener.accept() await client_stream.aclose() server_sock.close() From 2947de7eb248e2f33790fb00bc7465490b0e0922 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 01:50:28 +0000 Subject: [PATCH 0316/1498] Respond to review comments + add more tests --- trio/_core/_entry_queue.py | 6 +- trio/_core/_run.py | 151 ++++++++++++++---------------- trio/_core/tests/test_asyncgen.py | 97 ++++++++++++++++++- 3 files changed, 170 insertions(+), 84 deletions(-) diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index 357ea2cca9..a1587a18cd 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -59,8 +59,10 @@ async def kill_everything(exc): try: _core.spawn_system_task(kill_everything, exc) except RuntimeError: - # We're quite late in the shutdown process and - # the system nursery is already closed. + # We're quite late in the shutdown process and the + # system nursery is already closed. + # TODO(2020-06): this is a gross hack and should + # be fixed soon when we address #1607. _core.current_task().parent_nursery.start_soon(kill_everything, exc) return True diff --git a/trio/_core/_run.py b/trio/_core/_run.py index f891fda7b3..9efc9644ac 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1227,10 +1227,18 @@ class Runner: # Async generators are added to this set when first iterated. Any # left after the main task exits will be closed before trio.run() - # returns. During the execution of the main task, this is a - # WeakSet so GC works. During shutdown, it's a regular set so we - # don't have to deal with GC firing at unexpected times. + # returns. During most of the run, this is a WeakSet so GC works. + # During shutdown, when we're finalizign all the remaining + # asyncgens after the system nursery has been closed, it's a + # regular set so we don't have to deal with GC firing at + # unexpected times. asyncgens = attr.ib(factory=weakref.WeakSet) + + # This collects async generators that get garbage collected during + # the one-tick window between the system nursery closing and the + # init task starting end-of-run asyncgen finalization. + trailing_finalizer_asyncgens = attr.ib(factory=set) + prev_asyncgen_hooks = attr.ib(default=None) def force_guest_tick_asap(self): @@ -1467,21 +1475,7 @@ def task_exited(self, task, outcome): # Async generator finalization support ################ - async def finalize_asyncgen(self, agen, name, *, check_running): - if check_running and agen.ag_running: - # Another async generator is iterating this one, which is - # suspended at an event loop trap. Add it back to the - # asyncgens set and we'll get it on the next round. Note - # that this is only possible during end-of-run - # finalization; in GC-directed finalization, no one has a - # reference to agen anymore, so no one can be iterating it. - # - # This field is only reliable on 3.8+ due to - # ttps://bugs.python.org/issue32526. Pythons below - # 3.8 use a workaround in finalize_remaining_asyncgens. - self.asyncgens.add(agen) - return - + async def finalize_asyncgen(self, agen, name): try: # This shield ensures that finalize_asyncgen never exits # with an exception, not even a Cancelled. The inside @@ -1503,17 +1497,32 @@ async def finalize_remaining_asyncgens(self): # shut down the system nursery, so no more can appear.) # Neither one uses async generators, so every async generator # must be suspended at a yield point -- there's no one to be - # doing the iteration. However, once we start aclose() of one - # async generator, it might start fetching the next value from - # another, thus preventing us from closing that other. + # doing the iteration. That's good, because aclose() only + # works on an asyncgen that's suspended at a yield point. + # (If it's suspended at an event loop trap, because someone + # is in the middle of iterating it, then you get a RuntimeError + # on 3.8+, and a nasty surprise on earlier versions due to + # https://bugs.python.org/issue32526.) # - # On 3.8+, we can detect this condition by looking at - # ag_running. On earlier versions, ag_running doesn't provide - # useful information. We could look at ag_await, but that - # would fail in case of shenanigans like - # https://github.com/python-trio/async_generator/pull/16. - # It's easier to just not parallelize the shutdowns. - finalize_in_parallel = sys.version_info >= (3, 8) + # However, once we start aclose() of one async generator, it + # might start fetching the next value from another, thus + # preventing us from closing that other (at least until + # aclose() of the first one is complete). This constraint + # effectively requires us to finalize the remaining asyncgens + # in arbitrary order, rather than doing all of them at the + # same time. On 3.8+ we could defer any generator with + # ag_running=True to a later batch, but that only catches + # the case where our aclose() starts after the user's + # asend()/etc. If our aclose() starts first, then the + # user's asend()/etc will raise RuntimeError, since they're + # probably not checking ag_running. + # + # It might be possible to allow some parallelized cleanup if + # we can determine that a certain set of asyncgens have no + # interdependencies, using gc.get_referents() and such. + # But just doing one at a time will typically work well enough + # (since each aclose() executes in a cancelled scope) and + # is much easier to reason about. # It's possible that that cleanup code will itself create # more async generators, so we iterate repeatedly until @@ -1521,38 +1530,26 @@ async def finalize_remaining_asyncgens(self): while self.asyncgens: batch = self.asyncgens self.asyncgens = set() - - if finalize_in_parallel: - async with open_nursery() as kill_them_all: - # This shield is needed to avoid the checkpoint - # in Nursery.__aexit__ raising Cancelled if we're - # in a cancelled scope. (Which can happen if - # a run_sync_soon callback raises an exception.) - kill_them_all.cancel_scope.shield = True - for agen in batch: - name = name_asyncgen(agen) - kill_them_all.start_soon( - partial( - self.finalize_asyncgen, agen, name, check_running=True - ), - name="close asyncgen {} (outlived run)".format(name), - ) - - if self.asyncgens == batch: # pragma: no cover - # Something about the running-detection seems - # to have failed; fall back to one-at-a-time mode - # instead of looping forever - finalize_in_parallel = False - else: - for agen in batch: - await self.finalize_asyncgen( - agen, name_asyncgen(agen), check_running=False - ) + for agen in batch: + await self.finalize_asyncgen(agen, name_asyncgen(agen)) def setup_asyncgen_hooks(self): def firstiter(agen): self.asyncgens.add(agen) + def finalize_in_trio_context(agen, agen_name): + try: + self.spawn_system_task( + self.finalize_asyncgen, agen, agen_name, + name=f"close asyncgen {agen_name} (abandoned)", + ) + except RuntimeError: + # There is a one-tick window where the system nursery + # is closed but the init task hasn't yet made + # self.asyncgens a strong set to disable GC. We seem to + # have hit it. + self.trailing_finalizer_asyncgens.add(agen) + def finalizer(agen): agen_name = name_asyncgen(agen) warnings.warn( @@ -1561,16 +1558,9 @@ def finalizer(agen): f"to ensure that it gets cleaned up as soon as you're done using it.", ResourceWarning, stacklevel=2, + source=agen, ) - self.entry_queue.run_sync_soon( - partial( - self.spawn_system_task, - partial( - self.finalize_asyncgen, agen, agen_name, check_running=False - ), - name=f"close asyncgen {agen_name} (abandoned)", - ), - ) + self.entry_queue.run_sync_soon(finalize_in_trio_context, agen, agen_name) self.prev_asyncgen_hooks = sys.get_asyncgen_hooks() sys.set_asyncgen_hooks(firstiter=firstiter, finalizer=finalizer) @@ -1645,22 +1635,23 @@ async def init(self, async_fn, args): system_task=True, ) - # Main task is done. We should be exiting soon, so - # we're going to shut down GC-mediated async generator - # finalization by turning the asyncgens WeakSet into a - # regular set. We must do that before closing the system - # nursery, since finalization spawns a new system tasks. - self.asyncgens = set(self.asyncgens) + # Main task is done; start shutting down system tasks + self.system_nursery.cancel_scope.cancel() - # Process all pending run_sync_soon callbacks, in case one of - # them was an asyncgen finalizer. - self.entry_queue.run_sync_soon(self.reschedule, self.init_task) - await wait_task_rescheduled(lambda _: Abort.FAILED) + # The only tasks running at this point are init (this one) + # and the run_sync_soon task. We still need to finalize + # remaining async generators. To make that easier to reason + # about, we'll shut down their garbage collection by turning + # the asyncgens WeakSet into a regular set. + self.asyncgens = set(self.asyncgens) - # Now it's safe to proceed with shutting down system tasks - self.system_nursery.cancel_scope.cancel() + # Process all pending run_sync_soon callbacks, in case one of + # them was an asyncgen finalizer that snuck in under the wire. + self.entry_queue.run_sync_soon(self.reschedule, self.init_task) + await wait_task_rescheduled(lambda _: Abort.FAILED) + self.asyncgens.update(self.trailing_finalizer_asyncgens) + self.trailing_finalizer_asyncgens.clear() - # System tasks are gone and no more will be appearing. # The only async-colored user code left to run is the # finalizers for the async generators that remain alive. await self.finalize_remaining_asyncgens() @@ -1948,6 +1939,8 @@ def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints) # particular before we start modifying global state like GLOBAL_RUN_CONTEXT ki_manager.install(runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints) + runner.setup_asyncgen_hooks() + GLOBAL_RUN_CONTEXT.runner = runner return runner @@ -2150,10 +2143,6 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): if not host_uses_signal_set_wakeup_fd: runner.entry_queue.wakeup.wakeup_on_signals() - # Do this before before_run in case before_run wants to override - # our hooks - runner.setup_asyncgen_hooks() - if runner.instruments: runner.instrument("before_run") runner.clock.start_clock() diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index ecc4c4ba54..5750b09489 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -1,4 +1,5 @@ import sys +import weakref import pytest from math import inf from functools import partial @@ -18,7 +19,6 @@ async def example(cause): pass await _core.checkpoint() except _core.Cancelled: - assert "example" in _core.current_task().name assert "exhausted" not in cause task_name = _core.current_task().name assert cause in task_name or task_name == "" @@ -113,3 +113,98 @@ async def async_main(): _core.run(async_main) assert record == ["cleanup 2", "cleanup 1"] + + +def test_interdependent_asyncgen_cleanup_order(): + saved = [] + record = [] + + async def innermost(): + try: + yield 1 + finally: + await _core.cancel_shielded_checkpoint() + record.append("innermost") + + async def agen(label, inner): + try: + yield await inner.asend(None) + finally: + # Either `inner` has already been cleaned up, or + # we're about to exhaust it. Either way, we wind + # up with `record` containing the labels in + # innermost-to-outermost order. + with pytest.raises(StopAsyncIteration): + await inner.asend(None) + record.append(label) + + async def async_main(): + # This makes a chain of 101 interdependent asyncgens: + # agen(99)'s cleanup will iterate agen(98)'s will iterate + # ... agen(0)'s will iterate innermost()'s + ag_chain = innermost() + for idx in range(100): + ag_chain = agen(idx, ag_chain) + saved.append(ag_chain) + async for val in ag_chain: + assert val == 1 + break + assert record == [] + + _core.run(async_main) + assert record == ["innermost"] + list(range(100)) + + +def test_last_minute_gc_edge_case(): + saved = [] + record = [] + needs_retry = True + + async def agen(): + try: + yield 1 + finally: + record.append("cleaned up") + + def collect_at_opportune_moment(token): + runner = _core._run.GLOBAL_RUN_CONTEXT.runner + if runner.system_nursery._closed and isinstance(runner.asyncgens, weakref.WeakSet): + saved.clear() + record.append("final collection") + gc_collect_harder() + record.append("done") + else: + try: + token.run_sync_soon(collect_at_opportune_moment, token) + except _core.RunFinishedError: + nonlocal needs_retry + needs_retry = True + + async def async_main(): + token = _core.current_trio_token() + token.run_sync_soon(collect_at_opportune_moment, token) + saved.append(agen()) + async for _ in saved[-1]: + break + + # Actually running into the edge case requires that the run_sync_soon task + # execute in between the system nursery's closure and the strong-ification + # of runner.asyncgens. There's about a 25% chance that it doesn't + # (if the run_sync_soon task runs before init on one tick and after init + # on the next tick); if we try enough times, we can make the chance of + # failure as small as we want. + for attempt in range(50): + needs_retry = False + del record[:] + del saved[:] + _core.run(async_main) + if needs_retry: + assert record == ["cleaned up"] + else: + assert record == ["final collection", "done", "cleaned up"] + break + else: + pytest.fail( + f"Didn't manage to hit the trailing_finalizer_asyncgens case " + f"despite trying {attempt} times" + ) From d198ed20844783f2ae9a843ca3238c1eb580367e Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 01:51:00 +0000 Subject: [PATCH 0317/1498] blacken --- trio/_core/_run.py | 4 +++- trio/_core/tests/test_asyncgen.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 9efc9644ac..0df293881f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1540,7 +1540,9 @@ def firstiter(agen): def finalize_in_trio_context(agen, agen_name): try: self.spawn_system_task( - self.finalize_asyncgen, agen, agen_name, + self.finalize_asyncgen, + agen, + agen_name, name=f"close asyncgen {agen_name} (abandoned)", ) except RuntimeError: diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index 5750b09489..4d939a9f72 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -168,7 +168,9 @@ async def agen(): def collect_at_opportune_moment(token): runner = _core._run.GLOBAL_RUN_CONTEXT.runner - if runner.system_nursery._closed and isinstance(runner.asyncgens, weakref.WeakSet): + if runner.system_nursery._closed and isinstance( + runner.asyncgens, weakref.WeakSet + ): saved.clear() record.append("final collection") gc_collect_harder() From 45c2eeea66207bb92831fcb941472c7c42f73dc4 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 01:55:52 +0000 Subject: [PATCH 0318/1498] flake8 --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 4a94ecdaad..f36f4616a2 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1544,7 +1544,7 @@ async def finalize_asyncgen(self, agen, name): with CancelScope(shield=True) as cancel_scope: cancel_scope.cancel() await agen.aclose() - except BaseException as exc: + except BaseException: ASYNCGEN_LOGGER.exception( "Exception ignored during finalization of async generator %r -- " "surround your use of the generator in 'async with aclosing(...):' " From 2d81bb00f5fd8fd5f5954be1997ee8a51907a77b Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 01:57:49 +0000 Subject: [PATCH 0319/1498] Fix mismerge with master --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index f36f4616a2..e74c73a884 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1448,7 +1448,7 @@ def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): if name is None: name = async_fn - if isinstance(name, partial): + if isinstance(name, functools.partial): name = name.func if not isinstance(name, str): try: From 5a37f78697c4194e8ac2c5fc65f611e5cf0770f0 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 02:06:08 +0000 Subject: [PATCH 0320/1498] Work correctly in -Werror mode too --- trio/_core/_run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index e74c73a884..b7e8763b8f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1615,6 +1615,11 @@ def finalize_in_trio_context(agen, agen_name): def finalizer(agen): agen_name = name_asyncgen(agen) + self.entry_queue.run_sync_soon(finalize_in_trio_context, agen, agen_name) + + # Do this last, because it might raise an exception depending on the + # user's warnings filter. (That exception will be printed to the + # terminal and ignored, since we're running in GC context.) warnings.warn( f"Async generator {agen_name!r} was garbage collected before it had " f"been exhausted. Surround its use in 'async with aclosing(...):' " @@ -1623,7 +1628,6 @@ def finalizer(agen): stacklevel=2, source=agen, ) - self.entry_queue.run_sync_soon(finalize_in_trio_context, agen, agen_name) self.prev_asyncgen_hooks = sys.get_asyncgen_hooks() sys.set_asyncgen_hooks(firstiter=firstiter, finalizer=finalizer) From 3acf34be7ecf2699d14741aa3fd95fabd1357650 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 04:27:26 +0000 Subject: [PATCH 0321/1498] Handle asyncgens correctly when Trio is the guest of an asyncio host --- trio/_core/_run.py | 73 +++++++++++++++++++----- trio/_core/tests/test_asyncgen.py | 87 +++++++++++++++++++++++++++++ trio/_core/tests/test_guest_mode.py | 39 +++++++++++++ 3 files changed, 185 insertions(+), 14 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index b7e8763b8f..dc9526fc2e 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1596,7 +1596,18 @@ async def finalize_remaining_asyncgens(self): def setup_asyncgen_hooks(self): def firstiter(agen): - self.asyncgens.add(agen) + if hasattr(GLOBAL_RUN_CONTEXT, "task"): + self.asyncgens.add(agen) + else: + # An async generator first iterated outside of a Trio + # task doesn't belong to Trio. Probably we're in guest + # mode and the async generator belongs to our host. + # The locals dictionary is the only good place to + # remember this fact, at least until + # https://bugs.python.org/issue40916 is implemented. + agen.ag_frame.f_locals["@trio_foreign_asyncgen"] = True + if self.prev_asyncgen_hooks.firstiter is not None: + self.prev_asyncgen_hooks.firstiter(agen) def finalize_in_trio_context(agen, agen_name): try: @@ -1615,19 +1626,53 @@ def finalize_in_trio_context(agen, agen_name): def finalizer(agen): agen_name = name_asyncgen(agen) - self.entry_queue.run_sync_soon(finalize_in_trio_context, agen, agen_name) - - # Do this last, because it might raise an exception depending on the - # user's warnings filter. (That exception will be printed to the - # terminal and ignored, since we're running in GC context.) - warnings.warn( - f"Async generator {agen_name!r} was garbage collected before it had " - f"been exhausted. Surround its use in 'async with aclosing(...):' " - f"to ensure that it gets cleaned up as soon as you're done using it.", - ResourceWarning, - stacklevel=2, - source=agen, - ) + try: + is_ours = not agen.ag_frame.f_locals.get("@trio_foreign_asyncgen") + except AttributeError: + is_ours = True + + if is_ours: + self.entry_queue.run_sync_soon( + finalize_in_trio_context, agen, agen_name + ) + + # Do this last, because it might raise an exception + # depending on the user's warnings filter. (That + # exception will be printed to the terminal and + # ignored, since we're running in GC context.) + warnings.warn( + f"Async generator {agen_name!r} was garbage collected before it " + f"had been exhausted. Surround its use in 'async with " + f"aclosing(...):' to ensure that it gets cleaned up as soon as " + f"you're done using it.", + ResourceWarning, + stacklevel=2, + source=agen, + ) + else: + # Not ours -> forward to the host loop's async generator finalizer + if self.prev_asyncgen_hooks.finalizer is not None: + self.prev_asyncgen_hooks.finalizer(agen) + else: + # Host has no finalizer. Reimplement the default + # Python behavior with no hooks installed: throw in + # GeneratorExit, step once, raise RuntimeError if + # it doesn't exit. + closer = agen.aclose() + try: + # If the next thing is a yield, this will raise RuntimeError + # which we allow to propagate + closer.send(None) + except StopIteration: + pass + else: + # If the next thing is an await, we get here. Give a nicer + # error than the default "async generator ignored GeneratorExit" + raise RuntimeError( + f"async generator {agen_name!r} awaited during " + f"finalization; install a finalization hook to support " + f"this, or wrap it in 'async with aclosing(...):'" + ) self.prev_asyncgen_hooks = sys.get_asyncgen_hooks() sys.set_asyncgen_hooks(firstiter=firstiter, finalizer=finalizer) diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index 4d939a9f72..ff3e42266d 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -210,3 +210,90 @@ async def async_main(): f"Didn't manage to hit the trailing_finalizer_asyncgens case " f"despite trying {attempt} times" ) + + +async def step_outside_async_context(aiter): + # abort_fns run outside of task context, at least if they're + # triggered by a deadline expiry rather than a direct + # cancellation. Thus, an asyncgen first iterated inside one + # will appear non-Trio, and since no other hooks were installed, + # will use the last-ditch fallback handling (that tries to mimic + # CPython's behavior with no hooks). + # + # NB: the strangeness with aiter being an attribute of abort_fn is + # to make it as easy as possible to ensure we don't hang onto a + # reference to aiter inside the guts of the run loop. + def abort_fn(_): + with pytest.raises(StopIteration, match="42"): + abort_fn.aiter.asend(None).send(None) + del abort_fn.aiter + return _core.Abort.SUCCEEDED + + abort_fn.aiter = aiter + + async with _core.open_nursery() as nursery: + nursery.start_soon(_core.wait_task_rescheduled, abort_fn) + await _core.wait_all_tasks_blocked() + nursery.cancel_scope.deadline = _core.current_time() + + +async def test_fallback_when_no_hook_claims_it(capsys): + async def well_behaved(): + yield 42 + + async def yields_after_yield(): + with pytest.raises(GeneratorExit): + yield 42 + yield 100 + + async def awaits_after_yield(): + with pytest.raises(GeneratorExit): + yield 42 + await _core.cancel_shielded_checkpoint() + + await step_outside_async_context(well_behaved()) + gc_collect_harder() + assert capsys.readouterr().err == "" + + await step_outside_async_context(yields_after_yield()) + gc_collect_harder() + assert "ignored GeneratorExit" in capsys.readouterr().err + + await step_outside_async_context(awaits_after_yield()) + gc_collect_harder() + assert "awaited during finalization" in capsys.readouterr().err + + +def test_delegation_to_existing_hooks(): + record = [] + + def my_firstiter(agen): + record.append("firstiter " + agen.ag_frame.f_locals["arg"]) + + def my_finalizer(agen): + record.append("finalizer " + agen.ag_frame.f_locals["arg"]) + + async def example(arg): + try: + yield 42 + finally: + with pytest.raises(_core.Cancelled): + await _core.checkpoint() + record.append("trio collected " + arg) + + async def async_main(): + await step_outside_async_context(example("theirs")) + assert 42 == await example("ours").asend(None) + gc_collect_harder() + assert record == ["firstiter theirs", "finalizer theirs"] + record[:] = [] + await _core.wait_all_tasks_blocked() + assert record == ["trio collected ours"] + + old_hooks = sys.get_asyncgen_hooks() + sys.set_asyncgen_hooks(my_firstiter, my_finalizer) + try: + _core.run(async_main) + finally: + assert sys.get_asyncgen_hooks() == (my_firstiter, my_finalizer) + sys.set_asyncgen_hooks(*old_hooks) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index e58ec3ebc2..3412cb0d6b 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -497,3 +497,42 @@ async def trio_main(in_host): # Should be basically instantaneous, but we'll leave a generous buffer to # account for any CI weirdness assert end - start < DURATION / 2 + + +def test_guest_mode_asyncgens(): + import sniffio, sys + + record = set() + + async def agen(label): + assert sniffio.current_async_library() == label + try: + yield 1 + finally: + library = sniffio.current_async_library() + try: + await sys.modules[library].sleep(0) + except trio.Cancelled: + pass + record.add((label, library)) + + async def iterate_in_aio(): + # "trio" gets inherited from our Trio caller if we don't set this + sniffio.current_async_library_cvar.set("asyncio") + async for _ in agen("asyncio"): + break + + async def trio_main(): + task = asyncio.ensure_future(iterate_in_aio()) + done_evt = trio.Event() + task.add_done_callback(lambda _: done_evt.set()) + with trio.fail_after(1): + await done_evt.wait() + + async for _ in agen("trio"): + break + + gc_collect_harder() + + aiotrio_run(trio_main, host_uses_signal_set_wakeup_fd=True) + assert record == {("asyncio", "asyncio"), ("trio", "trio")} From 80e8ab3db35451502d11294cffd8421a0985b84a Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 04:55:59 +0000 Subject: [PATCH 0322/1498] Fix 3.6 --- trio/_core/tests/test_guest_mode.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 3412cb0d6b..fdc304c3ae 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -1,5 +1,6 @@ import pytest import asyncio +import contextvars import traceback import queue from functools import partial @@ -534,5 +535,9 @@ async def trio_main(): gc_collect_harder() - aiotrio_run(trio_main, host_uses_signal_set_wakeup_fd=True) + # Ensure we don't pollute the thread-level context if run under + # an asyncio without contextvars support (3.6) + context = contextvars.copy_context() + context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) + assert record == {("asyncio", "asyncio"), ("trio", "trio")} From cfdb8501836eb10370bd69b04fbbf004eae638b7 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 05:22:58 +0000 Subject: [PATCH 0323/1498] Make tests pass on pypy 7.2 which doesn't run firstiter hooks --- trio/_core/tests/test_asyncgen.py | 28 ++++++++++++++++++++++------ trio/_core/tests/test_guest_mode.py | 5 +++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index ff3e42266d..efc3d562fe 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -8,6 +8,14 @@ from .tutil import gc_collect_harder +# PyPy 7.2 was released with a bug that just never called firstiter at +# all. This impacts tests of end-of-run finalization (nothing gets +# added to runner.asyncgens) and tests of "foreign" async generator +# behavior (since the firstiter hook is what marks the asyncgen as +# foreign), but most tests of GC-mediated finalization still work. +bad_pypy = sys.implementation.name == "pypy" and sys.pypy_version_info < (7, 3) + + def test_asyncgen_basics(): collected = [] @@ -76,11 +84,14 @@ async def async_main(): assert collected.pop() == "exhausted 4" # Leave one referenced-but-unexhausted and make sure it gets cleaned up - saved.append(example("outlived run")) - async for val in saved[-1]: - assert val == 42 - break - assert collected == [] + if bad_pypy: + collected.append("outlived run") + else: + saved.append(example("outlived run")) + async for val in saved[-1]: + assert val == 42 + break + assert collected == [] _core.run(async_main) assert collected.pop() == "outlived run" @@ -88,6 +99,7 @@ async def async_main(): assert agen.ag_frame is None # all should now be exhausted +@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") def test_firstiter_after_closing(): saved = [] record = [] @@ -115,6 +127,7 @@ async def async_main(): assert record == ["cleanup 2", "cleanup 1"] +@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") def test_interdependent_asyncgen_cleanup_order(): saved = [] record = [] @@ -201,7 +214,8 @@ async def async_main(): del saved[:] _core.run(async_main) if needs_retry: - assert record == ["cleaned up"] + if not bad_pypy: + assert record == ["cleaned up"] else: assert record == ["final collection", "done", "cleaned up"] break @@ -237,6 +251,7 @@ def abort_fn(_): nursery.cancel_scope.deadline = _core.current_time() +@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") async def test_fallback_when_no_hook_claims_it(capsys): async def well_behaved(): yield 42 @@ -264,6 +279,7 @@ async def awaits_after_yield(): assert "awaited during finalization" in capsys.readouterr().err +@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") def test_delegation_to_existing_hooks(): record = [] diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index fdc304c3ae..caef096ffa 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -1,6 +1,7 @@ import pytest import asyncio import contextvars +import sys import traceback import queue from functools import partial @@ -500,6 +501,10 @@ async def trio_main(in_host): assert end - start < DURATION / 2 +@pytest.mark.skipif( + sys.implementation.name == "pypy" and sys.pypy_version_info < (7, 3), + reason="PyPy 7.2 has a buggy implementation of async generator hooks", +) def test_guest_mode_asyncgens(): import sniffio, sys From affba27a798367066804df4c14d2d33b436ae50a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jun 2020 09:28:31 +0400 Subject: [PATCH 0324/1498] Remove _sys from trio._core exports --- trio/_core/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 7d57d9c4d1..c19b273c52 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -88,3 +88,5 @@ # Kqueue imports elif sys.platform != "linux" and sys.platform != "win32": from ._run import current_kqueue, monitor_kevent, wait_kevent + +del sys # It would be better to import sys as _sys, but mypy does not understand it From 205a710bfd99e1b15f77032d2f88541fb7c04680 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jun 2020 09:33:11 +0400 Subject: [PATCH 0325/1498] Tell mypy to ignore hasattr checks --- trio/_core/_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index e2ccb194ad..9d6d388ca7 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -16,7 +16,7 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable +from typing import Callable, TYPE_CHECKING from sniffio import current_async_library_cvar @@ -2379,10 +2379,10 @@ async def checkpoint_if_cancelled(): if sys.platform == "win32": from ._io_windows import WindowsIOManager as TheIOManager from ._generated_io_windows import * -elif sys.platform == "linux" or hasattr(select, "epoll"): +elif sys.platform == "linux" or (not TYPE_CHECKING and hasattr(select, "epoll")): from ._io_epoll import EpollIOManager as TheIOManager from ._generated_io_epoll import * -elif hasattr(select, "kqueue"): +elif TYPE_CHECKING or hasattr(select, "kqueue"): from ._io_kqueue import KqueueIOManager as TheIOManager from ._generated_io_kqueue import * else: # pragma: no cover From 5d2470fcea5e2b1fc45f82eec0be74e123befd01 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jun 2020 09:34:24 +0400 Subject: [PATCH 0326/1498] Explain future mypy plans in check.sh --- check.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/check.sh b/check.sh index 62ee6ab848..8f5757edbc 100755 --- a/check.sh +++ b/check.sh @@ -27,6 +27,8 @@ flake8 trio/ \ # We specify Linux so that we get the same results on all platforms. Without # that, mypy would complain on macOS that epoll does not exist, and we can't # ignore it as it would cause a mypy error on Linux +# In the future, we will run mypy on all platforms and use platform assert +# guards to avoid typechecking the wrong IOManagers mypy -p trio._core --platform linux || EXIT_STATUS=$? # Finally, leave a really clear warning of any issues and exit From 316b640a954bab3a9cbe7afcb20ee39706db528c Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jun 2020 09:35:20 +0400 Subject: [PATCH 0327/1498] Just use typing.Dict instead of DefaultDict --- trio/_core/_io_epoll.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index 4f6317df90..96aad1db48 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -1,7 +1,7 @@ import select import attr from collections import defaultdict -from typing import DefaultDict +from typing import Dict from .. import _core from ._run import _public @@ -186,7 +186,7 @@ class EpollIOManager: _epoll = attr.ib(factory=select.epoll) # {fd: EpollWaiters} _registered = attr.ib( - factory=lambda: defaultdict(EpollWaiters), type=DefaultDict[int, EpollWaiters] + factory=lambda: defaultdict(EpollWaiters), type=Dict[int, EpollWaiters] ) _force_wakeup = attr.ib(factory=WakeupSocketpair, type=WakeupSocketpair) _force_wakeup_fd = attr.ib(default=None) From 296490fcf419349cf6cc6491e3838e9ebb989e84 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jun 2020 09:35:40 +0400 Subject: [PATCH 0328/1498] Remove unneeded attr.ib type kwarg mypy can infer the type from the factory. --- trio/_core/_io_epoll.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index 96aad1db48..e70ad8e0e9 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -188,7 +188,7 @@ class EpollIOManager: _registered = attr.ib( factory=lambda: defaultdict(EpollWaiters), type=Dict[int, EpollWaiters] ) - _force_wakeup = attr.ib(factory=WakeupSocketpair, type=WakeupSocketpair) + _force_wakeup = attr.ib(factory=WakeupSocketpair) _force_wakeup_fd = attr.ib(default=None) def __attrs_post_init__(self): From 3a21fb3c0d6586f89e3b877fa5bc7b4bcadc1e60 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jun 2020 09:47:32 +0400 Subject: [PATCH 0329/1498] Fix formatting --- trio/_core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index c19b273c52..2bd0c74e67 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -89,4 +89,4 @@ elif sys.platform != "linux" and sys.platform != "win32": from ._run import current_kqueue, monitor_kevent, wait_kevent -del sys # It would be better to import sys as _sys, but mypy does not understand it +del sys # It would be better to import sys as _sys, but mypy does not understand it From 153ce13991e01fe32684b7c707f6b7b433e6b46a Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 06:23:54 +0000 Subject: [PATCH 0330/1498] Hopefully resolve coverage issues --- trio/_core/_run.py | 14 +++--- trio/_core/tests/test_asyncgen.py | 75 +++++++++++++++-------------- trio/_core/tests/test_guest_mode.py | 15 ++---- trio/_core/tests/test_run.py | 29 ++++++++++- trio/_core/tests/tutil.py | 11 +++++ trio/_util.py | 2 +- 6 files changed, 90 insertions(+), 56 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index dc9526fc2e..ea1424605a 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1308,7 +1308,10 @@ class Runner: # init task starting end-of-run asyncgen finalization. trailing_finalizer_asyncgens = attr.ib(factory=set) - prev_asyncgen_hooks = attr.ib(default=None) + prev_asyncgen_hooks = attr.ib(init=False) + + def __attrs_post_init__(self): + self.setup_asyncgen_hooks() def force_guest_tick_asap(self): if self.guest_tick_scheduled: @@ -1319,8 +1322,7 @@ def force_guest_tick_asap(self): def close(self): self.io_manager.close() self.entry_queue.close() - if self.prev_asyncgen_hooks is not None: - sys.set_asyncgen_hooks(*self.prev_asyncgen_hooks) + sys.set_asyncgen_hooks(*self.prev_asyncgen_hooks) if self.instruments: self.instrument("after_run") # This is where KI protection gets disabled, so we do it last @@ -1628,7 +1630,7 @@ def finalizer(agen): agen_name = name_asyncgen(agen) try: is_ours = not agen.ag_frame.f_locals.get("@trio_foreign_asyncgen") - except AttributeError: + except AttributeError: # pragma: no cover is_ours = True if is_ours: @@ -1760,7 +1762,7 @@ async def init(self, async_fn, args): # Process all pending run_sync_soon callbacks, in case one of # them was an asyncgen finalizer that snuck in under the wire. self.entry_queue.run_sync_soon(self.reschedule, self.init_task) - await wait_task_rescheduled(lambda _: Abort.FAILED) + await wait_task_rescheduled(lambda _: Abort.FAILED) # pragma: no cover self.asyncgens.update(self.trailing_finalizer_asyncgens) self.trailing_finalizer_asyncgens.clear() @@ -2059,8 +2061,6 @@ def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints) # particular before we start modifying global state like GLOBAL_RUN_CONTEXT ki_manager.install(runner.deliver_ki, restrict_keyboard_interrupt_to_checkpoints) - runner.setup_asyncgen_hooks() - GLOBAL_RUN_CONTEXT.runner = runner return runner diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index efc3d562fe..829eb23db9 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -5,15 +5,7 @@ from functools import partial from async_generator import aclosing from ... import _core -from .tutil import gc_collect_harder - - -# PyPy 7.2 was released with a bug that just never called firstiter at -# all. This impacts tests of end-of-run finalization (nothing gets -# added to runner.asyncgens) and tests of "foreign" async generator -# behavior (since the firstiter hook is what marks the asyncgen as -# foreign), but most tests of GC-mediated finalization still work. -bad_pypy = sys.implementation.name == "pypy" and sys.pypy_version_info < (7, 3) +from .tutil import gc_collect_harder, buggy_pypy_asyncgens def test_asyncgen_basics(): @@ -48,18 +40,14 @@ async def async_main(): with pytest.warns( ResourceWarning, match="Async generator.*collected before.*exhausted", ): - async for val in example("abandoned"): - assert val == 42 - break + assert 42 == await example("abandoned").asend(None) gc_collect_harder() await _core.wait_all_tasks_blocked() assert collected.pop() == "abandoned" # aclosing() ensures it's cleaned up at point of use async with aclosing(example("exhausted 1")) as aiter: - async for val in aiter: - assert val == 42 - break + assert 42 == await aiter.asend(None) assert collected.pop() == "exhausted 1" # Also fine if you exhaust it at point of use @@ -72,9 +60,7 @@ async def async_main(): # No problems saving the geniter when using either of these patterns async with aclosing(example("exhausted 3")) as aiter: saved.append(aiter) - async for val in aiter: - assert val == 42 - break + assert 42 == await aiter.asend(None) assert collected.pop() == "exhausted 3" # Also fine if you exhaust it at point of use @@ -84,13 +70,11 @@ async def async_main(): assert collected.pop() == "exhausted 4" # Leave one referenced-but-unexhausted and make sure it gets cleaned up - if bad_pypy: + if buggy_pypy_asyncgens: collected.append("outlived run") else: saved.append(example("outlived run")) - async for val in saved[-1]: - assert val == 42 - break + assert 42 == await saved[-1].asend(None) assert collected == [] _core.run(async_main) @@ -99,7 +83,28 @@ async def async_main(): assert agen.ag_frame is None # all should now be exhausted -@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") +async def test_asyncgen_throws_during_finalization(caplog): + record = [] + + async def agen(): + try: + yield 1 + finally: + await _core.cancel_shielded_checkpoint() + record.append("crashing") + raise ValueError("oops") + + await agen().asend(None) + gc_collect_harder() + await _core.wait_all_tasks_blocked() + assert record == ["crashing"] + exc_type, exc_value, exc_traceback = caplog.records[0].exc_info + assert exc_type is ValueError + assert str(exc_value) == "oops" + assert "during finalization of async generator" in caplog.records[0].message + + +@pytest.mark.skipif(buggy_pypy_asyncgens, reason="pypy 7.2.0 is buggy") def test_firstiter_after_closing(): saved = [] record = [] @@ -114,8 +119,7 @@ async def funky_agen(): yield 2 finally: record.append("cleanup 2") - async for _ in funky_agen(): - break + await funky_agen().asend(None) async def async_main(): aiter = funky_agen() @@ -127,7 +131,7 @@ async def async_main(): assert record == ["cleanup 2", "cleanup 1"] -@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") +@pytest.mark.skipif(buggy_pypy_asyncgens, reason="pypy 7.2.0 is buggy") def test_interdependent_asyncgen_cleanup_order(): saved = [] record = [] @@ -159,9 +163,7 @@ async def async_main(): for idx in range(100): ag_chain = agen(idx, ag_chain) saved.append(ag_chain) - async for val in ag_chain: - assert val == 1 - break + assert 1 == await ag_chain.asend(None) assert record == [] _core.run(async_main) @@ -191,7 +193,7 @@ def collect_at_opportune_moment(token): else: try: token.run_sync_soon(collect_at_opportune_moment, token) - except _core.RunFinishedError: + except _core.RunFinishedError: # pragma: no cover nonlocal needs_retry needs_retry = True @@ -199,8 +201,7 @@ async def async_main(): token = _core.current_trio_token() token.run_sync_soon(collect_at_opportune_moment, token) saved.append(agen()) - async for _ in saved[-1]: - break + await saved[-1].asend(None) # Actually running into the edge case requires that the run_sync_soon task # execute in between the system nursery's closure and the strong-ification @@ -213,13 +214,13 @@ async def async_main(): del record[:] del saved[:] _core.run(async_main) - if needs_retry: - if not bad_pypy: + if needs_retry: # pragma: no cover + if not buggy_pypy_asyncgens: assert record == ["cleaned up"] else: assert record == ["final collection", "done", "cleaned up"] break - else: + else: # pragma: no cover pytest.fail( f"Didn't manage to hit the trailing_finalizer_asyncgens case " f"despite trying {attempt} times" @@ -251,7 +252,7 @@ def abort_fn(_): nursery.cancel_scope.deadline = _core.current_time() -@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") +@pytest.mark.skipif(buggy_pypy_asyncgens, reason="pypy 7.2.0 is buggy") async def test_fallback_when_no_hook_claims_it(capsys): async def well_behaved(): yield 42 @@ -279,7 +280,7 @@ async def awaits_after_yield(): assert "awaited during finalization" in capsys.readouterr().err -@pytest.mark.skipif(bad_pypy, reason="pypy 7.2.0 is buggy") +@pytest.mark.skipif(buggy_pypy_asyncgens, reason="pypy 7.2.0 is buggy") def test_delegation_to_existing_hooks(): record = [] diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index caef096ffa..381d966f81 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -13,7 +13,7 @@ import trio import trio.testing -from .tutil import gc_collect_harder +from .tutil import gc_collect_harder, buggy_pypy_asyncgens from ..._util import signal_raise # The simplest possible "host" loop. @@ -501,12 +501,9 @@ async def trio_main(in_host): assert end - start < DURATION / 2 -@pytest.mark.skipif( - sys.implementation.name == "pypy" and sys.pypy_version_info < (7, 3), - reason="PyPy 7.2 has a buggy implementation of async generator hooks", -) +@pytest.mark.skipif(buggy_pypy_asyncgens, reason="PyPy 7.2 is buggy") def test_guest_mode_asyncgens(): - import sniffio, sys + import sniffio record = set() @@ -525,8 +522,7 @@ async def agen(label): async def iterate_in_aio(): # "trio" gets inherited from our Trio caller if we don't set this sniffio.current_async_library_cvar.set("asyncio") - async for _ in agen("asyncio"): - break + await agen("asyncio").asend(None) async def trio_main(): task = asyncio.ensure_future(iterate_in_aio()) @@ -535,8 +531,7 @@ async def trio_main(): with trio.fail_after(1): await done_evt.wait() - async for _ in agen("trio"): - break + await agen("trio").asend(None) gc_collect_harder() diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index e6f6280080..0bdd46ae44 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -20,6 +20,7 @@ check_sequence_matches, gc_collect_harder, ignore_coroutine_never_awaited_warnings, + buggy_pypy_asyncgens, ) from ... import _core @@ -529,7 +530,6 @@ async def main(): assert str(exc_value) == "oops" assert "Instrument has been disabled" in caplog.records[0].message - async def test_cancel_scope_repr(mock_clock): scope = _core.CancelScope() assert "unbound" in repr(scope) @@ -1526,6 +1526,33 @@ def cb(i): assert counter[0] == COUNT +@pytest.mark.skipif(buggy_pypy_asyncgens, reason="PyPy 7.2 is buggy") +def test_TrioToken_run_sync_soon_late_crash(): + # Crash after system nursery is closed -- easiest way to do that is + # from an async generator finalizer. + record = [] + saved = [] + + async def agen(): + token = _core.current_trio_token() + try: + yield 1 + finally: + token.run_sync_soon(lambda: {}["nope"]) + token.run_sync_soon(lambda: record.append("2nd ran")) + + async def main(): + saved.append(agen()) + await saved[-1].asend(None) + record.append("main exiting") + + with pytest.raises(_core.TrioInternalError) as excinfo: + _core.run(main) + + assert type(excinfo.value.__cause__) is KeyError + assert record == ["main exiting", "2nd ran"] + + async def test_slow_abort_basic(): with _core.CancelScope() as scope: scope.cancel() diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index b569371482..84feabf6d1 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -7,12 +7,23 @@ from contextlib import contextmanager import gc +import sys # See trio/tests/conftest.py for the other half of this from trio.tests.conftest import RUN_SLOW slow = pytest.mark.skipif(not RUN_SLOW, reason="use --run-slow to run slow tests",) +# PyPy 7.2 was released with a bug that just never called the async +# generator 'firstiter' hook at all. This impacts tests of end-of-run +# finalization (nothing gets added to runner.asyncgens) and tests of +# "foreign" async generator behavior (since the firstiter hook is what +# marks the asyncgen as foreign), but most tests of GC-mediated +# finalization still work. +buggy_pypy_asyncgens = ( + sys.implementation.name == "pypy" and sys.pypy_version_info < (7, 3) +) + try: s = stdlib_socket.socket(stdlib_socket.AF_INET6, stdlib_socket.SOCK_STREAM, 0) except OSError: # pragma: no cover diff --git a/trio/_util.py b/trio/_util.py index 939eaad332..1ebd0081f9 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -357,7 +357,7 @@ def name_asyncgen(agen): """Return the fully-qualified name of the async generator function that produced the async generator iterator *agen*. """ - if not hasattr(agen, "ag_code"): + if not hasattr(agen, "ag_code"): # pragma: no cover return repr(agen) try: module = agen.ag_frame.f_globals["__name__"] From ebaf69cb1a7e1196439ed78003c94f18c0bf4017 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 06:29:49 +0000 Subject: [PATCH 0331/1498] blacken --- trio/_core/tests/test_run.py | 1 + trio/_core/tests/tutil.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 0bdd46ae44..82e251b652 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -530,6 +530,7 @@ async def main(): assert str(exc_value) == "oops" assert "Instrument has been disabled" in caplog.records[0].message + async def test_cancel_scope_repr(mock_clock): scope = _core.CancelScope() assert "unbound" in repr(scope) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 84feabf6d1..92c5ef2c66 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -20,8 +20,9 @@ # "foreign" async generator behavior (since the firstiter hook is what # marks the asyncgen as foreign), but most tests of GC-mediated # finalization still work. -buggy_pypy_asyncgens = ( - sys.implementation.name == "pypy" and sys.pypy_version_info < (7, 3) +buggy_pypy_asyncgens = sys.implementation.name == "pypy" and sys.pypy_version_info < ( + 7, + 3, ) try: From 6f3825a536bb78c3bb93a60d1f75dbb2685f2298 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 25 Jun 2020 11:04:27 +0400 Subject: [PATCH 0332/1498] Run mypy on all platforms --- check.sh | 9 +++------ trio/_core/_io_epoll.py | 3 +++ trio/_core/_io_kqueue.py | 5 ++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/check.sh b/check.sh index 8f5757edbc..9ac8ebb935 100755 --- a/check.sh +++ b/check.sh @@ -23,13 +23,10 @@ flake8 trio/ \ --ignore=D,E,W,F401,F403,F405,F821,F822\ || EXIT_STATUS=$? -# Run mypy -# We specify Linux so that we get the same results on all platforms. Without -# that, mypy would complain on macOS that epoll does not exist, and we can't -# ignore it as it would cause a mypy error on Linux -# In the future, we will run mypy on all platforms and use platform assert -# guards to avoid typechecking the wrong IOManagers +# Run mypy on all supported platforms mypy -p trio._core --platform linux || EXIT_STATUS=$? +mypy -p trio._core --platform macos || EXIT_STATUS=$? # tests FreeBSD too +mypy -p trio._core --platform windows || EXIT_STATUS=$? # Finally, leave a really clear warning of any issues and exit if [ $EXIT_STATUS -ne 0 ]; then diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index e70ad8e0e9..f473595627 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -1,4 +1,5 @@ import select +import sys import attr from collections import defaultdict from typing import Dict @@ -8,6 +9,8 @@ from ._io_common import wake_all from ._wakeup_socketpair import WakeupSocketpair +assert sys.platform == "linux" + @attr.s(slots=True, eq=False, frozen=True) class _EpollStatistics: diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index f8ca732213..f6210f232b 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -1,4 +1,5 @@ import select +import sys import outcome from contextlib import contextmanager @@ -9,6 +10,8 @@ from ._run import _public from ._wakeup_socketpair import WakeupSocketpair +assert sys.platform != "linux" and sys.platform != "win32" + @attr.s(slots=True, eq=False, frozen=True) class _KqueueStatistics: @@ -19,7 +22,7 @@ class _KqueueStatistics: @attr.s(slots=True, eq=False) class KqueueIOManager: - _kqueue = attr.ib(factory=select.kqueue) # type: ignore + _kqueue = attr.ib(factory=select.kqueue) # {(ident, filter): Task or UnboundedQueue} _registered = attr.ib(factory=dict) _force_wakeup = attr.ib(factory=WakeupSocketpair) From 936ccdb549b6381f46d8ba96fa5f314d27c6ddbb Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 08:52:16 +0000 Subject: [PATCH 0333/1498] Add docs and newsfragment --- docs/source/conf.py | 0 docs/source/reference-core.rst | 174 +++++++++++++++++++++++++++++++++ newsfragments/265.headline.rst | 6 ++ trio/_core/tests/tutil.py | 8 +- 4 files changed, 185 insertions(+), 3 deletions(-) mode change 100644 => 100755 docs/source/conf.py create mode 100644 newsfragments/265.headline.rst diff --git a/docs/source/conf.py b/docs/source/conf.py old mode 100644 new mode 100755 diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index b5f6d7c4d8..477a393f75 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1460,6 +1460,180 @@ don't have any special access to Trio's internals.) :members: +.. _async-generators: + +Notes on async generators +------------------------- + +Python 3.6 added support for *async generators*, which can use +``await``, ``async for``, and ``async with`` in between their ``yield`` +statements. As you might expect, you use ``async for`` to iterate +over them. :pep:`525` has many more details if you want them. + +For example, the following is a roundabout way to print +the numbers 0 through 9 with a 1-second delay before each one:: + + async def range_slowly(*args): + """Like range(), but adds a 1-second sleep before each value.""" + for value in range(*args): + await trio.sleep(1) + yield value + + async def use_it(): + async for value in range_slowly(10): + print(value) + + trio.run(use_it) + +Trio supports async generators, with some caveats described in this section. + +Finalization +~~~~~~~~~~~~ + +If you iterate over an async generator in its entirety, like the +example above does, then the execution of the async generator will +occur completely in the context of the code that's iterating over it, +and there aren't too many surprises. + +If you abandon a partially-completed async generator, though, such as +by ``break``\ing out of the iteration, things aren't so simple. The +async generator iterator object is still alive, waiting for you to +resume iterating it so it can produce more values. At some point, +Python will realize that you've dropped all references to the +iterator, and will call on Trio to help with executing any remaining +cleanup code inside the generator: ``finally`` blocks, ``__aexit__`` +handlers, and so on. + +So far, so good. Unfortunately, Python provides no guarantees about +*when* this happens. It could be as soon as you break out of the +``async for`` loop, or an arbitrary amount of time later. It could +even be after the entire Trio run has finished! Just about the only +guarantee is that it *won't* happen in the task that was using the +generator. That task will continue on with whatever else it's doing, +and the async generator cleanup will happen "sometime later, +somewhere else". + +If you don't like that ambiguity, and you want to ensure that a +generator's ``finally`` blocks and ``__aexit__`` handlers execute as +soon as you're done using it, then you'll need to wrap your use of the +generator in something like `async_generator.aclosing() +`__:: + + # Instead of this: + async for value in my_generator(): + if value == 42: + break + + # Do this: + async with aclosing(my_generator()) as aiter: + async for value in aiter: + if value == 42: + break + +This is cumbersome, but Python unfortunately doesn't provide any other +reliable options. If you use ``aclosing()``, then +your generator's cleanup code executes in the same context as the +rest of its iterations, so timeouts, exceptions, and context +variables work like you'd expect. + +If you don't use ``aclosing()``, then Trio will do +its best anyway, but you'll have to contend with the following semantics: + +* The cleanup of the generator occurs in a cancelled context, i.e., + all blocking calls executed during cleanup will raise `Cancelled`. + This is to compensate for the fact that any timeouts surrounding + the original use of the generator have been long since forgotten. + +* The cleanup runs without access to any :ref:`context variables + ` that may have been present when the generator + was originally being used. + +* If the generator raises an exception during cleanup, then it's + printed to the ``trio.async_generator_errors`` logger and otherwise + ignored. + +* If an async generator is still alive at the end of the whole + call to :func:`trio.run`, then it will be cleaned up after all + tasks have exited and before :func:`trio.run` returns. + Since the "system nursery" has already been closed at this point, + Trio isn't able to support any new calls to + :func:`trio.lowlevel.spawn_system_task`. + +If you plan to run your code on PyPy to take advantage of its better +performance, you should be aware that PyPy is *far more likely* than +CPython to perform async generator cleanup at a time well after the +last use of the generator. (This is a consequence of the fact that +PyPy does not use reference counting to manage memory.) To help catch +issues like this, Trio will issue a `ResourceWarning` (ignored by +default, but enabled when running under ``python -X dev`` for example) +for each async generator that needs to be handled through the fallback +finalization path. + +Cancel scopes and nurseries +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**You may not write a ``yield`` statement that suspends an async generator +inside a `CancelScope` or `Nursery` that was entered within the generator.** + +That is, this is OK:: + + async def some_agen(): + with trio.move_on_after(1): + await long_operation() + yield "first" + async with trio.open_nursery() as nursery: + nursery.start_soon(task1) + nursery.start_soon(task2) + yield "second" + ... + +But this is not:: + + async def some_agen(): + with trio.move_on_after(1): + yield "first" + async with trio.open_nursery() as nursery: + yield "second" + ... + +Async generators decorated with ``@asynccontextmanager`` to serve as +the template for an async context manager are *not* subject to this +constraint, because ``@asynccontextmanager`` uses them in a limited +way that doesn't create problems. + +Violating the rule described in this section will sometimes get you a +useful error message, but Trio is not able to detect all such cases, +so sometimes you'll get an unhelpful `TrioInternalError`. (And +sometimes it will seem to work, which is probably the worst outcome of +all, since then you might not notice the issue until you perform some +minor refactoring of the generator or the code that's iterating it, or +just get unlucky. There is a `proposed Python enhancement +`__ +that would at least make it fail consistently.) + +The reason for the restriction on cancel scopes has to do with the +difficulty of noticing when a generator gets suspended and +resumed. The cancel scopes inside the generator shouldn't affect code +running outside the generator, but Trio isn't involved in the process +of exiting and reentering the generator, so it would be hard pressed +to keep its cancellation plumbing in the correct state. Nurseries +use a cancel scope internally, so they have all the problems of cancel +scopes plus a number of problems of their own: for example, when +the generator is suspended, what should the background tasks do? +There's no good way to suspend them, but if they keep running and throw +an exception, where can that exception be reraised? + +If you have an async generator that wants to ``yield`` from within a nursery +or cancel scope, your best bet is to refactor it to be a separate task +that communicates over memory channels. + +For more discussion and some experimental partial workarounds, see +Trio issues `264 `__ +(especially `this comment +`__) +and `638 `__. + + .. _threads: Threads (if you must) diff --git a/newsfragments/265.headline.rst b/newsfragments/265.headline.rst new file mode 100644 index 0000000000..57a1ae30b4 --- /dev/null +++ b/newsfragments/265.headline.rst @@ -0,0 +1,6 @@ +Trio now supports automatic :ref:`async generator finalization +`, so more async generators will work even if you +don't wrap them in ``async with async_generator.aclosing():`` +blocks. Please see the documentation for important caveats; in +particular, yielding within a nursery or cancel scope remains +unsupported. diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 92c5ef2c66..ee870958db 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -1,6 +1,7 @@ # Utilities for testing import socket as stdlib_socket import os +from typing import TYPE_CHECKING import pytest import warnings @@ -20,9 +21,10 @@ # "foreign" async generator behavior (since the firstiter hook is what # marks the asyncgen as foreign), but most tests of GC-mediated # finalization still work. -buggy_pypy_asyncgens = sys.implementation.name == "pypy" and sys.pypy_version_info < ( - 7, - 3, +buggy_pypy_asyncgens = ( + not TYPE_CHECKING + and sys.implementation.name == "pypy" + and sys.pypy_version_info < (7, 3) ) try: From 47d17e1f858d9d13a92a27ef12e2806fe03cae92 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2020 11:21:56 +0000 Subject: [PATCH 0334/1498] Bump mypy from 0.780 to 0.782 Bumps [mypy](https://github.com/python/mypy) from 0.780 to 0.782. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.780...v0.782) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index adb9fc105f..e8a1a4ab6f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -26,8 +26,8 @@ jedi==0.17.1 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint more-itertools==8.4.0 # via pytest -mypy-extensions==0.4.3 ; implementation_name == "cpython" # via mypy -mypy==0.780 ; implementation_name == "cpython" # via -r test-requirements.in +mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, mypy +mypy==0.782 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest parso==0.7.0 # via jedi @@ -54,7 +54,7 @@ sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in -typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black -typing-extensions==3.7.4.2; implementation_name == "cpython" # via mypy +typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy +typing-extensions==3.7.4.2 ; implementation_name == "cpython" # via -r test-requirements.in, mypy wcwidth==0.2.5 # via prompt-toolkit, pytest wrapt==1.12.1 # via astroid From 7d0fcd9a53c44983c7dd353f726e708782bcf605 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Thu, 25 Jun 2020 22:19:55 +0000 Subject: [PATCH 0335/1498] Fix formatting --- docs/source/reference-core.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 477a393f75..ffd546eaaf 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1572,8 +1572,8 @@ finalization path. Cancel scopes and nurseries ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -**You may not write a ``yield`` statement that suspends an async generator -inside a `CancelScope` or `Nursery` that was entered within the generator.** +.. warning:: You may not write a ``yield`` statement that suspends an async generator + inside a `CancelScope` or `Nursery` that was entered within the generator. That is, this is OK:: From 22ae21d2042042d084521e78f833e6a2f3440f84 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 26 Jun 2020 04:18:15 +0000 Subject: [PATCH 0336/1498] Organize instruments for reduced overhead --- newsfragments/1340.misc.rst | 6 + trio/_abc.py | 2 + trio/_core/_generated_instrumentation.py | 47 ++++ trio/_core/_generated_io_epoll.py | 1 + trio/_core/_generated_io_kqueue.py | 1 + trio/_core/_generated_io_windows.py | 1 + trio/_core/_generated_run.py | 37 +--- trio/_core/_instrumentation.py | 174 +++++++++++++++ trio/_core/_run.py | 109 +++------- trio/_core/tests/test_instrumentation.py | 266 +++++++++++++++++++++++ trio/_core/tests/test_run.py | 196 ----------------- trio/_tools/gen_exports.py | 2 + 12 files changed, 528 insertions(+), 314 deletions(-) create mode 100644 newsfragments/1340.misc.rst create mode 100644 trio/_core/_generated_instrumentation.py create mode 100644 trio/_core/_instrumentation.py create mode 100644 trio/_core/tests/test_instrumentation.py diff --git a/newsfragments/1340.misc.rst b/newsfragments/1340.misc.rst new file mode 100644 index 0000000000..9899505e2f --- /dev/null +++ b/newsfragments/1340.misc.rst @@ -0,0 +1,6 @@ +When using :ref:`instruments `, you now only "pay for what you use": +if there are no instruments installed that override a particular hook such as +:meth:`~trio.abc.Instrument.before_task_step`, then Trio doesn't waste any effort +on checking its instruments when the event corresponding to that hook occurs. +Previously, installing any instrument would incur all the instrumentation overhead, +even for hooks no one was interested in. diff --git a/trio/_abc.py b/trio/_abc.py index e3ccb930ee..e44e1affe7 100644 --- a/trio/_abc.py +++ b/trio/_abc.py @@ -1,3 +1,5 @@ +# coding: utf-8 + from abc import ABCMeta, abstractmethod from typing import Generic, TypeVar import trio diff --git a/trio/_core/_generated_instrumentation.py b/trio/_core/_generated_instrumentation.py new file mode 100644 index 0000000000..986ab2c7f5 --- /dev/null +++ b/trio/_core/_generated_instrumentation.py @@ -0,0 +1,47 @@ +# *********************************************************** +# ******* WARNING: AUTOGENERATED! ALL EDITS WILL BE LOST ****** +# ************************************************************* +from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND +from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED +from ._instrumentation import Instrument + +# fmt: off + + +def add_instrument(instrument: Instrument) ->None: + """Start instrumenting the current run loop with the given instrument. + + Args: + instrument (trio.abc.Instrument): The instrument to activate. + + If ``instrument`` is already active, does nothing. + + """ + locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True + try: + return GLOBAL_RUN_CONTEXT.runner.instruments.add_instrument(instrument) + except AttributeError: + raise RuntimeError("must be called from async context") + + +def remove_instrument(instrument: Instrument) ->None: + """Stop instrumenting the current run loop with the given instrument. + + Args: + instrument (trio.abc.Instrument): The instrument to de-activate. + + Raises: + KeyError: if the instrument is not currently active. This could + occur either because you never added it, or because you added it + and then it raised an unhandled exception and was automatically + deactivated. + + """ + locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True + try: + return GLOBAL_RUN_CONTEXT.runner.instruments.remove_instrument(instrument) + except AttributeError: + raise RuntimeError("must be called from async context") + + +# fmt: on diff --git a/trio/_core/_generated_io_epoll.py b/trio/_core/_generated_io_epoll.py index 9583c7ff4f..9ae54e4f68 100644 --- a/trio/_core/_generated_io_epoll.py +++ b/trio/_core/_generated_io_epoll.py @@ -3,6 +3,7 @@ # ************************************************************* from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED +from ._instrumentation import Instrument # fmt: off diff --git a/trio/_core/_generated_io_kqueue.py b/trio/_core/_generated_io_kqueue.py index ab95d1e30c..7549899dbe 100644 --- a/trio/_core/_generated_io_kqueue.py +++ b/trio/_core/_generated_io_kqueue.py @@ -3,6 +3,7 @@ # ************************************************************* from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED +from ._instrumentation import Instrument # fmt: off diff --git a/trio/_core/_generated_io_windows.py b/trio/_core/_generated_io_windows.py index d6a5760374..e6337e94b0 100644 --- a/trio/_core/_generated_io_windows.py +++ b/trio/_core/_generated_io_windows.py @@ -3,6 +3,7 @@ # ************************************************************* from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED +from ._instrumentation import Instrument # fmt: off diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index 49e2d6acd9..6b41f3bfb4 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -3,6 +3,7 @@ # ************************************************************* from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED +from ._instrumentation import Instrument # fmt: off @@ -226,40 +227,4 @@ async def test_lock_fairness(): raise RuntimeError("must be called from async context") -def add_instrument(instrument): - """Start instrumenting the current run loop with the given instrument. - - Args: - instrument (trio.abc.Instrument): The instrument to activate. - - If ``instrument`` is already active, does nothing. - - """ - locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True - try: - return GLOBAL_RUN_CONTEXT.runner.add_instrument(instrument) - except AttributeError: - raise RuntimeError("must be called from async context") - - -def remove_instrument(instrument): - """Stop instrumenting the current run loop with the given instrument. - - Args: - instrument (trio.abc.Instrument): The instrument to de-activate. - - Raises: - KeyError: if the instrument is not currently active. This could - occur either because you never added it, or because you added it - and then it raised an unhandled exception and was automatically - deactivated. - - """ - locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True - try: - return GLOBAL_RUN_CONTEXT.runner.remove_instrument(instrument) - except AttributeError: - raise RuntimeError("must be called from async context") - - # fmt: on diff --git a/trio/_core/_instrumentation.py b/trio/_core/_instrumentation.py new file mode 100644 index 0000000000..f7ca9ab027 --- /dev/null +++ b/trio/_core/_instrumentation.py @@ -0,0 +1,174 @@ +import logging +import types +import attr +import copy +from typing import Any, Callable, Dict, List, Sequence, Iterator, TypeVar + +from .._abc import Instrument + +# Used to log exceptions in instruments +INSTRUMENT_LOGGER = logging.getLogger("trio.abc.Instrument") + + +F = TypeVar("F", bound=Callable[..., Any]) + +# Decorator to mark methods public. This does nothing by itself, but +# trio/_tools/gen_exports.py looks for it. +def _public(fn: F) -> F: + return fn + + +HookImpl = Callable[..., Any] + + +class Hook(Dict[Instrument, HookImpl]): + """Manages installed instruments for a single hook such as before_run(). + + The base dictionary maps each instrument to the method of that + instrument that will be called when the hook is invoked. We use + inheritance so that 'if hook:' is fast (no Python-level function + calls needed). + + """ + + __slots__ = ("_name", "_parent", "_in_call") + + def __init__(self, name: str, parent: "Instruments"): + self._name = name # "before_run" or similar + self._parent = parent + self._in_call = 0 + + def __call__(self, *args: Any): + """Invoke the instrumentation hook with the given arguments.""" + self._in_call += 1 + try: + for instrument, method in self.items(): + try: + method(*args) + except: + self._parent.remove_instrument(instrument) + INSTRUMENT_LOGGER.exception( + "Exception raised when calling %r on instrument %r. " + "Instrument has been disabled.", + self._name, + instrument, + ) + finally: + self._in_call -= 1 + + def as_mutable(self) -> "Hook": + """Return a Hook object to which any desired modifications should be made. + + If this Hook is not in the middle of a call, it can be safely + mutated, and as_mutable() just returns self. If this Hook is + in the middle of a call, though, any mutation will cause the + call to raise a concurrent modification error. To handle the + latter case, we replace this Hook with a copy in our parent + Instruments collection, and return that copy. + """ + + if self._in_call: + # We're in the middle of a call on this hook, so + # we must replace it with a copy in order to avoid + # a "dict changed size during iteration" error. + replacement = copy.copy(self) + replacement.in_call = 0 + setattr(self._parent, self._name, replacement) + return replacement + return self + + +class AnnotatedFieldsAreSlots(type): + def __new__(mcls, clsname, bases, dct): + dct["__slots__"] = tuple(dct["__annotations__"].keys()) + return super().__new__(mcls, clsname, bases, dct) + + +class Instruments(metaclass=AnnotatedFieldsAreSlots): + """A collection of `trio.abc.Instrument` with some optimizations. + + Instrumentation calls are rather expensive, and we don't want a + rarely-used instrument (like before_run()) to slow down hot + operations (like before_task_step()). Thus, we cache the set of + handlers to be called for each hook, and skip the instrumentation + call if there's nothing currently installed for that hook. + """ + + before_run: Hook + after_run: Hook + task_spawned: Hook + task_scheduled: Hook + before_task_step: Hook + after_task_step: Hook + task_exited: Hook + before_io_wait: Hook + after_io_wait: Hook + + # Maps each installed instrument to the list of hook names that it implements. + _instruments: Dict[Instrument, List[str]] + + def __init__(self, incoming: Sequence[Instrument]): + self._instruments = {} + for name in Instruments.__slots__: # filled in by the metaclass + if not hasattr(self, name): + setattr(self, name, Hook(name, self)) + for instrument in incoming: + self.add_instrument(instrument) + + def __bool__(self) -> bool: + return bool(self._instruments) + + def __iter__(self) -> Iterator[Instrument]: + return iter(self._instruments) + + @_public + def add_instrument(self, instrument: Instrument) -> None: + """Start instrumenting the current run loop with the given instrument. + + Args: + instrument (trio.abc.Instrument): The instrument to activate. + + If ``instrument`` is already active, does nothing. + + """ + if instrument in self._instruments: + return + hooknames = self._instruments[instrument] = [] + try: + for name in dir(instrument): + if name.startswith("_"): + continue + try: + prototype = getattr(Instrument, name) + except AttributeError: + continue + impl: HookImpl = getattr(instrument, name) + if isinstance(impl, types.MethodType) and impl.__func__ is prototype: + # Inherited unchanged from _abc.Instrument + continue + hook: Hook = getattr(self, name).as_mutable() + hook[instrument] = impl + hooknames.append(name) + except: + self.remove_instrument(instrument) + raise + + @_public + def remove_instrument(self, instrument: Instrument) -> None: + """Stop instrumenting the current run loop with the given instrument. + + Args: + instrument (trio.abc.Instrument): The instrument to de-activate. + + Raises: + KeyError: if the instrument is not currently active. This could + occur either because you never added it, or because you added it + and then it raised an unhandled exception and was automatically + deactivated. + + """ + # If instrument isn't present, the KeyError propagates out + hooknames = self._instruments.pop(instrument) + for name in hooknames: + hook: Hook = getattr(self, name).as_mutable() + del hook[instrument] diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 9d6d388ca7..9d9725365e 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -42,6 +42,7 @@ WaitTaskRescheduled, ) from ._thread_cache import start_thread_soon +from ._instrumentation import Instruments from .. import _core from .._deprecate import warn_deprecated from .._util import Final, NoPublicConstructor, coroutine_or_error @@ -65,9 +66,6 @@ def _public(fn): _ALLOW_DETERMINISTIC_SCHEDULING = False _r = random.Random() -# Used to log exceptions in instruments -INSTRUMENT_LOGGER = logging.getLogger("trio.abc.Instrument") - # On 3.7+, Context.run() is implemented in C and doesn't show up in # tracebacks. On 3.6, we use the contextvars backport, which is @@ -1186,7 +1184,13 @@ def raise_cancel(): # The central Runner object ################################################################ -GLOBAL_RUN_CONTEXT = threading.local() + +class RunContext(threading.local): + runner: "Runner" + task: Task + + +GLOBAL_RUN_CONTEXT = RunContext() @attr.s(frozen=True) @@ -1265,7 +1269,7 @@ def in_main_thread(): @attr.s(eq=False, hash=False, slots=True) class Runner: clock = attr.ib() - instruments = attr.ib() + instruments: Instruments = attr.ib() io_manager = attr.ib() ki_manager = attr.ib() @@ -1302,8 +1306,7 @@ def force_guest_tick_asap(self): def close(self): self.io_manager.close() self.entry_queue.close() - if self.instruments: - self.instrument("after_run") + self.instruments.after_run() # This is where KI protection gets disabled, so we do it last self.ki_manager.close() @@ -1404,8 +1407,8 @@ def reschedule(self, task, next_send=_NO_SEND): if not self.runq and self.is_guest: self.force_guest_tick_asap() self.runq.append(task) - if self.instruments: - self.instrument("task_scheduled", task) + if self.instruments.task_scheduled: + self.instruments.task_scheduled(task) def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): @@ -1462,8 +1465,8 @@ async def python_wrapper(orig_coro): nursery._children.add(task) task._activate_cancel_status(nursery._cancel_status) - if self.instruments: - self.instrument("task_spawned", task) + if self.instruments.task_spawned: + self.instruments.task_spawned(task) # Special case: normally next_send should be an Outcome, but for the # very first send we have to send a literal unboxed None. self.reschedule(task, None) @@ -1511,8 +1514,8 @@ def task_exited(self, task, outcome): else: task._parent_nursery._child_finished(task, outcome) - if self.instruments: - self.instrument("task_exited", task) + if self.instruments.task_exited: + self.instruments.task_exited(task) ################ # System tasks and init @@ -1704,64 +1707,6 @@ def abort(_): await wait_task_rescheduled(abort) - ################ - # Instrumentation - ################ - - def instrument(self, method_name, *args): - if not self.instruments: - return - - for instrument in list(self.instruments): - try: - method = getattr(instrument, method_name) - except AttributeError: - continue - try: - method(*args) - except: - self.instruments.remove(instrument) - INSTRUMENT_LOGGER.exception( - "Exception raised when calling %r on instrument %r. " - "Instrument has been disabled.", - method_name, - instrument, - ) - - @_public - def add_instrument(self, instrument): - """Start instrumenting the current run loop with the given instrument. - - Args: - instrument (trio.abc.Instrument): The instrument to activate. - - If ``instrument`` is already active, does nothing. - - """ - if instrument not in self.instruments: - self.instruments.append(instrument) - - @_public - def remove_instrument(self, instrument): - """Stop instrumenting the current run loop with the given instrument. - - Args: - instrument (trio.abc.Instrument): The instrument to de-activate. - - Raises: - KeyError: if the instrument is not currently active. This could - occur either because you never added it, or because you added it - and then it raised an unhandled exception and was automatically - deactivated. - - """ - # We're moving 'instruments' to being a set, so raise KeyError like - # set.remove does. - try: - self.instruments.remove(instrument) - except ValueError as exc: - raise KeyError(*exc.args) - ################################################################ # run @@ -1840,7 +1785,7 @@ def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints) if clock is None: clock = SystemClock() - instruments = list(instruments) + instruments = Instruments(instruments) io_manager = TheIOManager() system_context = copy_context() system_context.run(current_async_library_cvar.set, "trio") @@ -2060,8 +2005,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): if not host_uses_signal_set_wakeup_fd: runner.entry_queue.wakeup.wakeup_on_signals() - if runner.instruments: - runner.instrument("before_run") + runner.instruments.before_run() runner.clock.start_clock() runner.init_task = runner.spawn_impl( runner.init, (async_fn, args), None, "", system_task=True, @@ -2090,16 +2034,16 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): timeout = runner.clock_autojump_threshold idle_primed = IdlePrimedTypes.AUTOJUMP_CLOCK - if runner.instruments: - runner.instrument("before_io_wait", timeout) + if runner.instruments.before_io_wait: + runner.instruments.before_io_wait(timeout) # Driver will call io_manager.get_events(timeout) and pass it back # in throuh the yield events = yield timeout runner.io_manager.process_events(events) - if runner.instruments: - runner.instrument("after_io_wait", timeout) + if runner.instruments.after_io_wait: + runner.instruments.after_io_wait(timeout) # Process cancellations due to deadline expiry now = runner.clock.current_time() @@ -2176,8 +2120,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): task = batch.pop() GLOBAL_RUN_CONTEXT.task = task - if runner.instruments: - runner.instrument("before_task_step", task) + if runner.instruments.before_task_step: + runner.instruments.before_task_step(task) next_send_fn = task._next_send_fn next_send = task._next_send @@ -2239,8 +2183,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): runner.reschedule(task, exc) task._next_send_fn = task.coro.throw - if runner.instruments: - runner.instrument("after_task_step", task) + if runner.instruments.after_task_step: + runner.instruments.after_task_step(task) del GLOBAL_RUN_CONTEXT.task except GeneratorExit: @@ -2389,3 +2333,4 @@ async def checkpoint_if_cancelled(): raise NotImplementedError("unsupported platform") from ._generated_run import * +from ._generated_instrumentation import * diff --git a/trio/_core/tests/test_instrumentation.py b/trio/_core/tests/test_instrumentation.py new file mode 100644 index 0000000000..ab3821a646 --- /dev/null +++ b/trio/_core/tests/test_instrumentation.py @@ -0,0 +1,266 @@ +import attr +import pytest +from ... import _core, _abc +from .tutil import check_sequence_matches + + +@attr.s(eq=False, hash=False) +class TaskRecorder: + record = attr.ib(factory=list) + + def before_run(self): + self.record.append(("before_run",)) + + def task_scheduled(self, task): + self.record.append(("schedule", task)) + + def before_task_step(self, task): + assert task is _core.current_task() + self.record.append(("before", task)) + + def after_task_step(self, task): + assert task is _core.current_task() + self.record.append(("after", task)) + + def after_run(self): + self.record.append(("after_run",)) + + def filter_tasks(self, tasks): + for item in self.record: + if item[0] in ("schedule", "before", "after") and item[1] in tasks: + yield item + if item[0] in ("before_run", "after_run"): + yield item + + +def test_instruments(recwarn): + r1 = TaskRecorder() + r2 = TaskRecorder() + r3 = TaskRecorder() + + task = None + + # We use a child task for this, because the main task does some extra + # bookkeeping stuff that can leak into the instrument results, and we + # don't want to deal with it. + async def task_fn(): + nonlocal task + task = _core.current_task() + + for _ in range(4): + await _core.checkpoint() + # replace r2 with r3, to test that we can manipulate them as we go + _core.remove_instrument(r2) + with pytest.raises(KeyError): + _core.remove_instrument(r2) + # add is idempotent + _core.add_instrument(r3) + _core.add_instrument(r3) + for _ in range(1): + await _core.checkpoint() + + async def main(): + async with _core.open_nursery() as nursery: + nursery.start_soon(task_fn) + + _core.run(main, instruments=[r1, r2]) + + # It sleeps 5 times, so it runs 6 times. Note that checkpoint() + # reschedules the task immediately upon yielding, before the + # after_task_step event fires. + expected = ( + [("before_run",), ("schedule", task)] + + [("before", task), ("schedule", task), ("after", task)] * 5 + + [("before", task), ("after", task), ("after_run",)] + ) + assert len(r1.record) > len(r2.record) > len(r3.record) + assert r1.record == r2.record + r3.record + assert list(r1.filter_tasks([task])) == expected + + +def test_instruments_interleave(): + tasks = {} + + async def two_step1(): + tasks["t1"] = _core.current_task() + await _core.checkpoint() + + async def two_step2(): + tasks["t2"] = _core.current_task() + await _core.checkpoint() + + async def main(): + async with _core.open_nursery() as nursery: + nursery.start_soon(two_step1) + nursery.start_soon(two_step2) + + r = TaskRecorder() + _core.run(main, instruments=[r]) + + expected = [ + ("before_run",), + ("schedule", tasks["t1"]), + ("schedule", tasks["t2"]), + { + ("before", tasks["t1"]), + ("schedule", tasks["t1"]), + ("after", tasks["t1"]), + ("before", tasks["t2"]), + ("schedule", tasks["t2"]), + ("after", tasks["t2"]), + }, + { + ("before", tasks["t1"]), + ("after", tasks["t1"]), + ("before", tasks["t2"]), + ("after", tasks["t2"]), + }, + ("after_run",), + ] + print(list(r.filter_tasks(tasks.values()))) + check_sequence_matches(list(r.filter_tasks(tasks.values())), expected) + + +def test_null_instrument(): + # undefined instrument methods are skipped + class NullInstrument: + def something_unrelated(self): + pass # pragma: no cover + + async def main(): + await _core.checkpoint() + + _core.run(main, instruments=[NullInstrument()]) + + +def test_instrument_before_after_run(): + record = [] + + class BeforeAfterRun: + def before_run(self): + record.append("before_run") + + def after_run(self): + record.append("after_run") + + async def main(): + pass + + _core.run(main, instruments=[BeforeAfterRun()]) + assert record == ["before_run", "after_run"] + + +def test_instrument_task_spawn_exit(): + record = [] + + class SpawnExitRecorder: + def task_spawned(self, task): + record.append(("spawned", task)) + + def task_exited(self, task): + record.append(("exited", task)) + + async def main(): + return _core.current_task() + + main_task = _core.run(main, instruments=[SpawnExitRecorder()]) + assert ("spawned", main_task) in record + assert ("exited", main_task) in record + + +# This test also tests having a crash before the initial task is even spawned, +# which is very difficult to handle. +def test_instruments_crash(caplog): + record = [] + + class BrokenInstrument: + def task_scheduled(self, task): + record.append("scheduled") + raise ValueError("oops") + + def close(self): + # Shouldn't be called -- tests that the instrument disabling logic + # works right. + record.append("closed") # pragma: no cover + + async def main(): + record.append("main ran") + return _core.current_task() + + r = TaskRecorder() + main_task = _core.run(main, instruments=[r, BrokenInstrument()]) + assert record == ["scheduled", "main ran"] + # the TaskRecorder kept going throughout, even though the BrokenInstrument + # was disabled + assert ("after", main_task) in r.record + assert ("after_run",) in r.record + # And we got a log message + exc_type, exc_value, exc_traceback = caplog.records[0].exc_info + assert exc_type is ValueError + assert str(exc_value) == "oops" + assert "Instrument has been disabled" in caplog.records[0].message + + +def test_instruments_monkeypatch(): + class NullInstrument(_abc.Instrument): + pass + + instrument = NullInstrument() + + async def main(): + record = [] + + # Changing the set of hooks implemented by an instrument after + # it's installed doesn't make them start being called right away + instrument.before_task_step = record.append + await _core.checkpoint() + await _core.checkpoint() + assert len(record) == 0 + + # But if we remove and re-add the instrument, the new hooks are + # picked up + _core.remove_instrument(instrument) + _core.add_instrument(instrument) + await _core.checkpoint() + await _core.checkpoint() + assert record.count(_core.current_task()) == 2 + + _core.remove_instrument(instrument) + await _core.checkpoint() + await _core.checkpoint() + assert record.count(_core.current_task()) == 2 + + _core.run(main, instruments=[instrument]) + + +def test_instrument_that_raises_on_getattr(): + class EvilInstrument: + @property + def after_run(self): + raise ValueError("oops") + + def task_exited(self, task): + assert False # pragma: no cover + + async def main(): + with pytest.raises(ValueError): + _core.add_instrument(EvilInstrument()) + + # Make sure the instrument is fully removed from the counts and list + runner = _core.current_task()._runner + assert not runner.instruments.after_run + assert not runner.instruments.task_exited + assert not runner.instruments + assert list(runner.instruments) == [] + + _core.run(main) + + +def test_instrumentation_hooks_match(): + slot_names = { + name + for name in _core._instrumentation.Instruments.__slots__ + if not name.startswith("_") + } + hook_names = {name for name in _abc.Instrument.__dict__ if not name.startswith("_")} + assert slot_names == hook_names diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index ce1dc2eeb4..04512f7b33 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -335,202 +335,6 @@ async def child(): assert stats.seconds_to_next_deadline == inf -@attr.s(eq=False, hash=False) -class TaskRecorder: - record = attr.ib(factory=list) - - def before_run(self): - self.record.append(("before_run",)) - - def task_scheduled(self, task): - self.record.append(("schedule", task)) - - def before_task_step(self, task): - assert task is _core.current_task() - self.record.append(("before", task)) - - def after_task_step(self, task): - assert task is _core.current_task() - self.record.append(("after", task)) - - def after_run(self): - self.record.append(("after_run",)) - - def filter_tasks(self, tasks): - for item in self.record: - if item[0] in ("schedule", "before", "after") and item[1] in tasks: - yield item - if item[0] in ("before_run", "after_run"): - yield item - - -def test_instruments(recwarn): - r1 = TaskRecorder() - r2 = TaskRecorder() - r3 = TaskRecorder() - - task = None - - # We use a child task for this, because the main task does some extra - # bookkeeping stuff that can leak into the instrument results, and we - # don't want to deal with it. - async def task_fn(): - nonlocal task - task = _core.current_task() - - for _ in range(4): - await _core.checkpoint() - # replace r2 with r3, to test that we can manipulate them as we go - _core.remove_instrument(r2) - with pytest.raises(KeyError): - _core.remove_instrument(r2) - # add is idempotent - _core.add_instrument(r3) - _core.add_instrument(r3) - for _ in range(1): - await _core.checkpoint() - - async def main(): - async with _core.open_nursery() as nursery: - nursery.start_soon(task_fn) - - _core.run(main, instruments=[r1, r2]) - - # It sleeps 5 times, so it runs 6 times. Note that checkpoint() - # reschedules the task immediately upon yielding, before the - # after_task_step event fires. - expected = ( - [("before_run",), ("schedule", task)] - + [("before", task), ("schedule", task), ("after", task)] * 5 - + [("before", task), ("after", task), ("after_run",)] - ) - assert len(r1.record) > len(r2.record) > len(r3.record) - assert r1.record == r2.record + r3.record - assert list(r1.filter_tasks([task])) == expected - - -def test_instruments_interleave(): - tasks = {} - - async def two_step1(): - tasks["t1"] = _core.current_task() - await _core.checkpoint() - - async def two_step2(): - tasks["t2"] = _core.current_task() - await _core.checkpoint() - - async def main(): - async with _core.open_nursery() as nursery: - nursery.start_soon(two_step1) - nursery.start_soon(two_step2) - - r = TaskRecorder() - _core.run(main, instruments=[r]) - - expected = [ - ("before_run",), - ("schedule", tasks["t1"]), - ("schedule", tasks["t2"]), - { - ("before", tasks["t1"]), - ("schedule", tasks["t1"]), - ("after", tasks["t1"]), - ("before", tasks["t2"]), - ("schedule", tasks["t2"]), - ("after", tasks["t2"]), - }, - { - ("before", tasks["t1"]), - ("after", tasks["t1"]), - ("before", tasks["t2"]), - ("after", tasks["t2"]), - }, - ("after_run",), - ] - print(list(r.filter_tasks(tasks.values()))) - check_sequence_matches(list(r.filter_tasks(tasks.values())), expected) - - -def test_null_instrument(): - # undefined instrument methods are skipped - class NullInstrument: - pass - - async def main(): - await _core.checkpoint() - - _core.run(main, instruments=[NullInstrument()]) - - -def test_instrument_before_after_run(): - record = [] - - class BeforeAfterRun: - def before_run(self): - record.append("before_run") - - def after_run(self): - record.append("after_run") - - async def main(): - pass - - _core.run(main, instruments=[BeforeAfterRun()]) - assert record == ["before_run", "after_run"] - - -def test_instrument_task_spawn_exit(): - record = [] - - class SpawnExitRecorder: - def task_spawned(self, task): - record.append(("spawned", task)) - - def task_exited(self, task): - record.append(("exited", task)) - - async def main(): - return _core.current_task() - - main_task = _core.run(main, instruments=[SpawnExitRecorder()]) - assert ("spawned", main_task) in record - assert ("exited", main_task) in record - - -# This test also tests having a crash before the initial task is even spawned, -# which is very difficult to handle. -def test_instruments_crash(caplog): - record = [] - - class BrokenInstrument: - def task_scheduled(self, task): - record.append("scheduled") - raise ValueError("oops") - - def close(self): - # Shouldn't be called -- tests that the instrument disabling logic - # works right. - record.append("closed") # pragma: no cover - - async def main(): - record.append("main ran") - return _core.current_task() - - r = TaskRecorder() - main_task = _core.run(main, instruments=[r, BrokenInstrument()]) - assert record == ["scheduled", "main ran"] - # the TaskRecorder kept going throughout, even though the BrokenInstrument - # was disabled - assert ("after", main_task) in r.record - assert ("after_run",) in r.record - # And we got a log message - exc_type, exc_value, exc_traceback = caplog.records[0].exc_info - assert exc_type is ValueError - assert str(exc_value) == "oops" - assert "Instrument has been disabled" in caplog.records[0].message - - async def test_cancel_scope_repr(mock_clock): scope = _core.CancelScope() assert "unbound" in repr(scope) diff --git a/trio/_tools/gen_exports.py b/trio/_tools/gen_exports.py index 33cc736ae1..e48114af6d 100755 --- a/trio/_tools/gen_exports.py +++ b/trio/_tools/gen_exports.py @@ -20,6 +20,7 @@ # ************************************************************* from ._run import GLOBAL_RUN_CONTEXT, _NO_SEND from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED +from ._instrumentation import Instrument # fmt: off """ @@ -179,6 +180,7 @@ def main(): # pragma: no cover core = source_root / "trio/_core" to_wrap = [ (core / "_run.py", "runner"), + (core / "_instrumentation.py", "runner.instruments"), (core / "_io_windows.py", "runner.io_manager"), (core / "_io_epoll.py", "runner.io_manager"), (core / "_io_kqueue.py", "runner.io_manager"), From 77cff0879622607ec8afdfe6d95f1b8d933a3f2d Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 26 Jun 2020 04:36:26 +0000 Subject: [PATCH 0337/1498] Improve static checkability of instrument calls, remove metaclass --- trio/_core/_instrumentation.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/trio/_core/_instrumentation.py b/trio/_core/_instrumentation.py index f7ca9ab027..18c09c994b 100644 --- a/trio/_core/_instrumentation.py +++ b/trio/_core/_instrumentation.py @@ -72,19 +72,13 @@ def as_mutable(self) -> "Hook": # we must replace it with a copy in order to avoid # a "dict changed size during iteration" error. replacement = copy.copy(self) - replacement.in_call = 0 + replacement._in_call = 0 setattr(self._parent, self._name, replacement) return replacement return self -class AnnotatedFieldsAreSlots(type): - def __new__(mcls, clsname, bases, dct): - dct["__slots__"] = tuple(dct["__annotations__"].keys()) - return super().__new__(mcls, clsname, bases, dct) - - -class Instruments(metaclass=AnnotatedFieldsAreSlots): +class Instruments(Instrument): """A collection of `trio.abc.Instrument` with some optimizations. Instrumentation calls are rather expensive, and we don't want a @@ -92,24 +86,23 @@ class Instruments(metaclass=AnnotatedFieldsAreSlots): operations (like before_task_step()). Thus, we cache the set of handlers to be called for each hook, and skip the instrumentation call if there's nothing currently installed for that hook. + + This inherits from `trio.abc.Instrument` for the benefit of + static type checking (to make sure you pass the right arguments + when calling an instrument). All of the class-level function + definitions are shadowed by instance-level Hooks. """ - before_run: Hook - after_run: Hook - task_spawned: Hook - task_scheduled: Hook - before_task_step: Hook - after_task_step: Hook - task_exited: Hook - before_io_wait: Hook - after_io_wait: Hook + # One Hook per instrument, with its same name + __slots__ = [name for name in Instrument.__dict__ if not name.startswith("_")] # Maps each installed instrument to the list of hook names that it implements. _instruments: Dict[Instrument, List[str]] + __slots__.append("_instruments") def __init__(self, incoming: Sequence[Instrument]): self._instruments = {} - for name in Instruments.__slots__: # filled in by the metaclass + for name in Instruments.__slots__: if not hasattr(self, name): setattr(self, name, Hook(name, self)) for instrument in incoming: From cbcb624dda028ae56d34b89a36db3e374ca80469 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 26 Jun 2020 04:55:25 +0000 Subject: [PATCH 0338/1498] copy.copy() of dict subclasses with slots didn't work on early 3.6.x --- trio/_core/_instrumentation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/trio/_core/_instrumentation.py b/trio/_core/_instrumentation.py index 18c09c994b..57cde37c0d 100644 --- a/trio/_core/_instrumentation.py +++ b/trio/_core/_instrumentation.py @@ -1,7 +1,6 @@ import logging import types import attr -import copy from typing import Any, Callable, Dict, List, Sequence, Iterator, TypeVar from .._abc import Instrument @@ -71,8 +70,8 @@ def as_mutable(self) -> "Hook": # We're in the middle of a call on this hook, so # we must replace it with a copy in order to avoid # a "dict changed size during iteration" error. - replacement = copy.copy(self) - replacement._in_call = 0 + replacement = Hook(self._name, self._parent) + replacement.update(self) setattr(self._parent, self._name, replacement) return replacement return self From 2ae51d682ff20270a9508e0021a5e2a17c9db910 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2020 06:21:42 +0000 Subject: [PATCH 0339/1498] Bump py from 1.8.2 to 1.9.0 Bumps [py](https://github.com/pytest-dev/py) from 1.8.2 to 1.9.0. - [Release notes](https://github.com/pytest-dev/py/releases) - [Changelog](https://github.com/pytest-dev/py/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/py/compare/1.8.2...1.9.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e8a1a4ab6f..6c5b9054f1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -37,7 +37,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest prompt-toolkit==3.0.5 # via ipython ptyprocess==0.6.0 # via pexpect -py==1.8.2 # via pytest +py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 From 96765d9bbd52b5d3a51e486850e0cb95efbd62cc Mon Sep 17 00:00:00 2001 From: RmStorm Date: Fri, 26 Jun 2020 10:06:33 +0200 Subject: [PATCH 0340/1498] Improved docs related to child cancellation closes #1640 --- docs/source/reference-core.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index b5f6d7c4d8..3601153485 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -670,14 +670,17 @@ expires:: Note that what matters here is the scopes that were active when :func:`open_nursery` was called, *not* the scopes active when -``start_soon`` is called. So for example, the timeout block below does -nothing at all:: +``start_soon`` is called. The reason for this is that the ``start_soon`` call immediately exits and the scheduled task is awaited by the nursery. So for example, the timeout block below does +nothing at all because the ``with move_on_after(TIMEOUT):`` block is immediately exited when ``start_soon`` returns and ``child`` is awaited by the nursery:: async with trio.open_nursery() as nursery: with move_on_after(TIMEOUT): # don't do this! nursery.start_soon(child) +This behaviour is also discussed here: `issue 1640 +`_. + Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 42b015cc0913dd7ff46539efcd4965c7366e67c0 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 27 Jun 2020 02:51:12 +0000 Subject: [PATCH 0341/1498] Use copy-on-iterate instead of copy-on-write for simplicity's sake, since COW isn't worth much performance --- trio/_core/_instrumentation.py | 60 ++++++------------------ trio/_core/tests/test_instrumentation.py | 20 ++------ 2 files changed, 18 insertions(+), 62 deletions(-) diff --git a/trio/_core/_instrumentation.py b/trio/_core/_instrumentation.py index 57cde37c0d..80f8645a2b 100644 --- a/trio/_core/_instrumentation.py +++ b/trio/_core/_instrumentation.py @@ -30,51 +30,25 @@ class Hook(Dict[Instrument, HookImpl]): """ - __slots__ = ("_name", "_parent", "_in_call") + __slots__ = ("_name", "_parent") def __init__(self, name: str, parent: "Instruments"): self._name = name # "before_run" or similar self._parent = parent - self._in_call = 0 def __call__(self, *args: Any): """Invoke the instrumentation hook with the given arguments.""" - self._in_call += 1 - try: - for instrument, method in self.items(): - try: - method(*args) - except: - self._parent.remove_instrument(instrument) - INSTRUMENT_LOGGER.exception( - "Exception raised when calling %r on instrument %r. " - "Instrument has been disabled.", - self._name, - instrument, - ) - finally: - self._in_call -= 1 - - def as_mutable(self) -> "Hook": - """Return a Hook object to which any desired modifications should be made. - - If this Hook is not in the middle of a call, it can be safely - mutated, and as_mutable() just returns self. If this Hook is - in the middle of a call, though, any mutation will cause the - call to raise a concurrent modification error. To handle the - latter case, we replace this Hook with a copy in our parent - Instruments collection, and return that copy. - """ - - if self._in_call: - # We're in the middle of a call on this hook, so - # we must replace it with a copy in order to avoid - # a "dict changed size during iteration" error. - replacement = Hook(self._name, self._parent) - replacement.update(self) - setattr(self._parent, self._name, replacement) - return replacement - return self + for instrument, method in list(self.items()): + try: + method(*args) + except: + self._parent.remove_instrument(instrument) + INSTRUMENT_LOGGER.exception( + "Exception raised when calling %r on instrument %r. " + "Instrument has been disabled.", + self._name, + instrument, + ) class Instruments(Instrument): @@ -107,12 +81,6 @@ def __init__(self, incoming: Sequence[Instrument]): for instrument in incoming: self.add_instrument(instrument) - def __bool__(self) -> bool: - return bool(self._instruments) - - def __iter__(self) -> Iterator[Instrument]: - return iter(self._instruments) - @_public def add_instrument(self, instrument: Instrument) -> None: """Start instrumenting the current run loop with the given instrument. @@ -138,7 +106,7 @@ def add_instrument(self, instrument: Instrument) -> None: if isinstance(impl, types.MethodType) and impl.__func__ is prototype: # Inherited unchanged from _abc.Instrument continue - hook: Hook = getattr(self, name).as_mutable() + hook: Hook = getattr(self, name) hook[instrument] = impl hooknames.append(name) except: @@ -162,5 +130,5 @@ def remove_instrument(self, instrument: Instrument) -> None: # If instrument isn't present, the KeyError propagates out hooknames = self._instruments.pop(instrument) for name in hooknames: - hook: Hook = getattr(self, name).as_mutable() + hook: Hook = getattr(self, name) del hook[instrument] diff --git a/trio/_core/tests/test_instrumentation.py b/trio/_core/tests/test_instrumentation.py index ab3821a646..580937bab6 100644 --- a/trio/_core/tests/test_instrumentation.py +++ b/trio/_core/tests/test_instrumentation.py @@ -235,32 +235,20 @@ async def main(): def test_instrument_that_raises_on_getattr(): class EvilInstrument: + def task_exited(self, task): + assert False # pragma: no cover + @property def after_run(self): raise ValueError("oops") - def task_exited(self, task): - assert False # pragma: no cover - async def main(): with pytest.raises(ValueError): _core.add_instrument(EvilInstrument()) - # Make sure the instrument is fully removed from the counts and list + # Make sure the instrument is fully removed from the per-method lists runner = _core.current_task()._runner assert not runner.instruments.after_run assert not runner.instruments.task_exited - assert not runner.instruments - assert list(runner.instruments) == [] _core.run(main) - - -def test_instrumentation_hooks_match(): - slot_names = { - name - for name in _core._instrumentation.Instruments.__slots__ - if not name.startswith("_") - } - hook_names = {name for name in _abc.Instrument.__dict__ if not name.startswith("_")} - assert slot_names == hook_names From 4076f8554685a80792588e5b1c87830096eb7cd9 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 27 Jun 2020 23:12:36 +0000 Subject: [PATCH 0342/1498] Rename newsfragments so they actually show up --- newsfragments/{1621.fix.rst => 1621.bugfix.rst} | 0 newsfragments/{1638.fix.rst => 1638.bugfix.rst} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{1621.fix.rst => 1621.bugfix.rst} (100%) rename newsfragments/{1638.fix.rst => 1638.bugfix.rst} (100%) diff --git a/newsfragments/1621.fix.rst b/newsfragments/1621.bugfix.rst similarity index 100% rename from newsfragments/1621.fix.rst rename to newsfragments/1621.bugfix.rst diff --git a/newsfragments/1638.fix.rst b/newsfragments/1638.bugfix.rst similarity index 100% rename from newsfragments/1638.fix.rst rename to newsfragments/1638.bugfix.rst From 5d00465e7d85ec911ec66a13ffb2216472c1f657 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 29 Jun 2020 10:12:11 +0400 Subject: [PATCH 0343/1498] Fix win32 typing issue --- trio/_core/tests/tutil.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index b569371482..9fcf3e2ea7 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -1,6 +1,7 @@ # Utilities for testing import socket as stdlib_socket import os +import sys import pytest import warnings @@ -80,7 +81,8 @@ def check_sequence_matches(seq, template): # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=246350 skip_if_fbsd_pipes_broken = pytest.mark.skipif( - hasattr(os, "uname") + sys.platform != "win32" # prevent mypy from complaining about missing uname + and hasattr(os, "uname") and os.uname().sysname == "FreeBSD" and os.uname().release[:4] < "12.2", reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350", From 32a06a811d44ee78238b5b8ed5a40c2d4d315f1d Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 29 Jun 2020 10:20:40 +0400 Subject: [PATCH 0344/1498] Add win32 sys.platform assert for consistency --- trio/_core/_io_windows.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 35fc15e02e..b4585ff69b 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -2,6 +2,7 @@ from contextlib import contextmanager import enum import socket +import sys import attr @@ -25,6 +26,8 @@ IoControlCodes, ) +assert sys.platform == "win32" + # There's a lot to be said about the overall design of a Windows event # loop. See # From 43716eba661e620f581741cb00e3c15b26b2652b Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 29 Jun 2020 10:20:57 +0400 Subject: [PATCH 0345/1498] Fix mypy platform names --- check.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check.sh b/check.sh index 9ac8ebb935..5cf0c04f1a 100755 --- a/check.sh +++ b/check.sh @@ -25,8 +25,8 @@ flake8 trio/ \ # Run mypy on all supported platforms mypy -p trio._core --platform linux || EXIT_STATUS=$? -mypy -p trio._core --platform macos || EXIT_STATUS=$? # tests FreeBSD too -mypy -p trio._core --platform windows || EXIT_STATUS=$? +mypy -p trio._core --platform darwin || EXIT_STATUS=$? # tests FreeBSD too +mypy -p trio._core --platform win32 || EXIT_STATUS=$? # Finally, leave a really clear warning of any issues and exit if [ $EXIT_STATUS -ne 0 ]; then From ed9174831d258d9feaf15cf77faa4c89983d3fb2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2020 07:00:29 +0000 Subject: [PATCH 0346/1498] Bump idna from 2.9 to 2.10 Bumps [idna](https://github.com/kjd/idna) from 2.9 to 2.10. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v2.9...v2.10) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f4aa519b3f..c5f72bd193 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -12,7 +12,7 @@ certifi==2020.6.20 # via requests chardet==3.0.4 # via requests click==7.1.2 # via towncrier docutils==0.16 # via sphinx -idna==2.9 # via -r docs-requirements.in, requests +idna==2.10 # via -r docs-requirements.in, requests imagesize==1.2.0 # via sphinx immutables==0.14 # via -r docs-requirements.in incremental==17.5.0 # via towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 6c5b9054f1..57a7158b7a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ coverage==5.1 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.3 # via -r test-requirements.in -idna==2.9 # via -r test-requirements.in, trustme +idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.15.0 # via -r test-requirements.in From 8a047c3db022ce87f4a738e86b9d9d138b60d8ab Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2020 07:12:01 +0000 Subject: [PATCH 0347/1498] Bump ipython from 7.15.0 to 7.16.1 Bumps [ipython](https://github.com/ipython/ipython) from 7.15.0 to 7.16.1. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.15.0...7.16.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57a7158b7a..ac8999097a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -20,7 +20,7 @@ flake8==3.8.3 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets -ipython==7.15.0 # via -r test-requirements.in +ipython==7.16.1 # via -r test-requirements.in isort==4.3.21 # via pylint jedi==0.17.1 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From 8e64dbf650b17caef62d5b8890ebaebc4af0ae05 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 29 Jun 2020 13:22:24 +0400 Subject: [PATCH 0348/1498] Only assert platform when type checking We would like to write: ``` if TYPE_CHECKING: assert sys.platform == "linux" ``` But that hides the assert to mypy. This commits finds a way to avoid that issue. --- trio/_core/_io_epoll.py | 4 ++-- trio/_core/_io_kqueue.py | 3 ++- trio/_core/_io_windows.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/trio/_core/_io_epoll.py b/trio/_core/_io_epoll.py index f473595627..c1537cf53e 100644 --- a/trio/_core/_io_epoll.py +++ b/trio/_core/_io_epoll.py @@ -2,14 +2,14 @@ import sys import attr from collections import defaultdict -from typing import Dict +from typing import Dict, TYPE_CHECKING from .. import _core from ._run import _public from ._io_common import wake_all from ._wakeup_socketpair import WakeupSocketpair -assert sys.platform == "linux" +assert not TYPE_CHECKING or sys.platform == "linux" @attr.s(slots=True, eq=False, frozen=True) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index f6210f232b..a52ff9b32b 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -1,5 +1,6 @@ import select import sys +from typing import TYPE_CHECKING import outcome from contextlib import contextmanager @@ -10,7 +11,7 @@ from ._run import _public from ._wakeup_socketpair import WakeupSocketpair -assert sys.platform != "linux" and sys.platform != "win32" +assert not TYPE_CHECKING or sys.platform != "linux" and sys.platform != "win32" @attr.s(slots=True, eq=False, frozen=True) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index b4585ff69b..40cd8ba5b0 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -3,6 +3,7 @@ import enum import socket import sys +from typing import TYPE_CHECKING import attr @@ -26,7 +27,7 @@ IoControlCodes, ) -assert sys.platform == "win32" +assert not TYPE_CHECKING or sys.platform == "win32" # There's a lot to be said about the overall design of a Windows event # loop. See From ae1cbfc158fed4f86cc86c3304ee508f1df06a5f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 29 Jun 2020 23:10:21 +0400 Subject: [PATCH 0349/1498] Clarify operator precedence in boolean expression --- trio/_core/_io_kqueue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index a52ff9b32b..7a723c5f98 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -11,7 +11,7 @@ from ._run import _public from ._wakeup_socketpair import WakeupSocketpair -assert not TYPE_CHECKING or sys.platform != "linux" and sys.platform != "win32" +assert not TYPE_CHECKING or (sys.platform != "linux" and sys.platform != "win32") @attr.s(slots=True, eq=False, frozen=True) From 31bf4e6fdc70827406bf73665be5147aabd59beb Mon Sep 17 00:00:00 2001 From: RmStorm Date: Wed, 1 Jul 2020 09:52:00 +0200 Subject: [PATCH 0350/1498] Clarified new paragraph about child tasks and cancellation --- docs/source/reference-core.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 3601153485..428a75e7b3 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -670,16 +670,14 @@ expires:: Note that what matters here is the scopes that were active when :func:`open_nursery` was called, *not* the scopes active when -``start_soon`` is called. The reason for this is that the ``start_soon`` call immediately exits and the scheduled task is awaited by the nursery. So for example, the timeout block below does -nothing at all because the ``with move_on_after(TIMEOUT):`` block is immediately exited when ``start_soon`` returns and ``child`` is awaited by the nursery:: +``start_soon`` is called. So for example, the timeout block below does +nothing at all:: async with trio.open_nursery() as nursery: with move_on_after(TIMEOUT): # don't do this! nursery.start_soon(child) - -This behaviour is also discussed here: `issue 1640 -`_. +The reason for this is that the ``start_soon`` call immediately returns and the scheduled task actually runs inside the nursery. So in the above snippet the contextmanager ``with move_on_after(TIMEOUT):`` block is immediately exited after ``start_soon`` returns. Since ``move_on_after`` only imposes a time limit on the body of its ``with`` block and that block is exited it cannot cancel the child task. Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 9004d3c8d5f3b40f9ecda682b4b78e2ddad4b0c7 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 29 Jun 2020 09:52:34 +0000 Subject: [PATCH 0351/1498] Make Trio pass 'mypy -m trio -m trio.testing' --- .coveragerc | 9 ++++ check.sh | 6 +-- mypy.ini | 3 +- trio/__init__.py | 2 + trio/_path.py | 2 + trio/_path.pyi | 1 + trio/_socket.py | 23 ++++---- trio/_subprocess.py | 77 +++++++++++++++------------ trio/_subprocess_platform/__init__.py | 32 ++++++----- trio/_subprocess_platform/kqueue.py | 4 ++ trio/_subprocess_platform/waitid.py | 3 +- trio/_util.py | 15 ++++-- trio/_windows_pipes.py | 4 ++ trio/lowlevel.py | 41 ++++++-------- trio/socket.py | 63 ++++++++++++---------- trio/tests/test_windows_pipes.py | 10 ++-- 16 files changed, 173 insertions(+), 122 deletions(-) create mode 100644 trio/_path.pyi diff --git a/.coveragerc b/.coveragerc index 4bdbf69131..4bbac7b27d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -17,3 +17,12 @@ precision = 1 exclude_lines = pragma: no cover abc.abstractmethod + if TYPE_CHECKING: + if _t.TYPE_CHECKING: + +partial_branches = + pragma: no branch + if not TYPE_CHECKING: + if not _t.TYPE_CHECKING: + if .* or not TYPE_CHECKING: + if .* or not _t.TYPE_CHECKING: diff --git a/check.sh b/check.sh index 5cf0c04f1a..57f1e2db40 100755 --- a/check.sh +++ b/check.sh @@ -24,9 +24,9 @@ flake8 trio/ \ || EXIT_STATUS=$? # Run mypy on all supported platforms -mypy -p trio._core --platform linux || EXIT_STATUS=$? -mypy -p trio._core --platform darwin || EXIT_STATUS=$? # tests FreeBSD too -mypy -p trio._core --platform win32 || EXIT_STATUS=$? +mypy -m trio -m trio.testing --platform linux || EXIT_STATUS=$? +mypy -m trio -m trio.testing --platform darwin || EXIT_STATUS=$? # tests FreeBSD too +mypy -m trio -m trio.testing --platform win32 || EXIT_STATUS=$? # Finally, leave a really clear warning of any issues and exit if [ $EXIT_STATUS -ne 0 ]; then diff --git a/mypy.ini b/mypy.ini index a93e159b42..31eeef1cd0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,8 +2,7 @@ # TODO: run mypy against several OS/version combos in CI # https://mypy.readthedocs.io/en/latest/command_line.html#platform-configuration -# Be flexible about unannotated imports -follow_imports = silent +# Be flexible about dependencies that don't have stubs yet (like pytest) ignore_missing_imports = True # Be strict about use of Mypy diff --git a/trio/__init__.py b/trio/__init__.py index 1a17c5ee67..63e74e9da8 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -102,6 +102,8 @@ if False: from . import testing +from . import _deprecate + _deprecate.enable_attribute_deprecations(__name__) __deprecated_attributes__ = { # NOTE: when you remove this, you should also remove the file diff --git a/trio/_path.py b/trio/_path.py index 095bdc779f..d2442c89f6 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -1,3 +1,5 @@ +# type: ignore + from functools import wraps, partial import os import types diff --git a/trio/_path.pyi b/trio/_path.pyi new file mode 100644 index 0000000000..85a8e1f960 --- /dev/null +++ b/trio/_path.pyi @@ -0,0 +1 @@ +class Path: ... diff --git a/trio/_socket.py b/trio/_socket.py index a2ca21afde..be4e611fb3 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -1,8 +1,9 @@ -import os as _os -import sys as _sys +import os +import sys import select import socket as _stdlib_socket from functools import wraps as _wraps +from typing import TYPE_CHECKING import idna as _idna @@ -51,7 +52,7 @@ async def __aexit__(self, etype, value, tb): except ImportError: # As of at least 3.6, python on Windows is missing IPPROTO_IPV6 # https://bugs.python.org/issue29515 - if _sys.platform == "win32": + if sys.platform == "win32": # pragma: no branch IPPROTO_IPV6 = 41 ################################################################ @@ -246,7 +247,9 @@ def fromfd(fd, family, type, proto=0): return from_stdlib_socket(_stdlib_socket.fromfd(fd, family, type, proto)) -if hasattr(_stdlib_socket, "fromshare"): +if sys.platform == "win32" or ( + not TYPE_CHECKING and hasattr(_stdlib_socket, "fromshare") +): @_wraps(_stdlib_socket.fromshare, assigned=(), updated=()) def fromshare(*args, **kwargs): @@ -293,7 +296,7 @@ def _sniff_sockopts_for_fileno(family, type, proto, fileno): # Wrap the raw fileno into a Python socket object # This object might have the wrong metadata, but it lets us easily call getsockopt # and then we'll throw it away and construct a new one with the correct metadata. - if not _sys.platform == "linux": + if sys.platform != "linux": return family, type, proto from socket import SO_DOMAIN, SO_PROTOCOL, SOL_SOCKET, SO_TYPE @@ -473,7 +476,7 @@ def shutdown(self, flag): def is_readable(self): # use select.select on Windows, and select.poll everywhere else - if _sys.platform == "win32": + if sys.platform == "win32": rready, _, _ = select.select([self._sock], [], [], 0) return bool(rready) p = select.poll() @@ -504,7 +507,7 @@ async def _resolve_address_nocp(self, address, flags): ) elif self._sock.family == _stdlib_socket.AF_UNIX: # unwrap path-likes - return _os.fspath(address) + return os.fspath(address) else: return address @@ -684,7 +687,7 @@ async def connect(self, address): # Okay, the connect finished, but it might have failed: err = self._sock.getsockopt(_stdlib_socket.SOL_SOCKET, _stdlib_socket.SO_ERROR) if err != 0: - raise OSError(err, "Error in connect: " + _os.strerror(err)) + raise OSError(err, "Error in connect: " + os.strerror(err)) ################################################################ # recv @@ -757,7 +760,9 @@ async def sendto(self, *args): # sendmsg ################################################################ - if hasattr(_stdlib_socket.socket, "sendmsg"): + if sys.platform != "win32" or ( + not TYPE_CHECKING and hasattr(_stdlib_socket.socket, "sendmsg") + ): @_wraps(_stdlib_socket.socket.sendmsg, assigned=(), updated=()) async def sendmsg(self, *args): diff --git a/trio/_subprocess.py b/trio/_subprocess.py index ac51915eb0..876cc0d7c9 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -1,9 +1,12 @@ +# coding: utf-8 + import os import subprocess import sys from typing import Optional from functools import partial import warnings +from typing import TYPE_CHECKING from ._abc import AsyncResource, SendStream, ReceiveStream from ._highlevel_generic import StapledStream @@ -18,38 +21,46 @@ # Linux-specific, but has complex lifetime management stuff so we hard-code it # here instead of hiding it behind the _subprocess_platform abstraction -can_try_pidfd_open = True -try: - from os import pidfd_open -except ImportError: - if sys.platform == "linux": - import ctypes - - _cdll_for_pidfd_open = ctypes.CDLL(None, use_errno=True) - _cdll_for_pidfd_open.syscall.restype = ctypes.c_long - # pid and flags are actually int-sized, but the syscall() function - # always takes longs. (Except on x32 where long is 32-bits and syscall - # takes 64-bit arguments. But in the unlikely case that anyone is - # using x32, this will still work, b/c we only need to pass in 32 bits - # of data, and the C ABI doesn't distinguish between passing 32-bit vs - # 64-bit integers; our 32-bit values will get loaded into 64-bit - # registers where syscall() will find them.) - _cdll_for_pidfd_open.syscall.argtypes = [ - ctypes.c_long, # syscall number - ctypes.c_long, # pid - ctypes.c_long, # flags - ] - __NR_pidfd_open = 434 - - def pidfd_open(fd, flags): - result = _cdll_for_pidfd_open.syscall(__NR_pidfd_open, fd, flags) - if result < 0: - err = ctypes.get_errno() - raise OSError(err, os.strerror(err)) - return result +can_try_pidfd_open: bool +if TYPE_CHECKING: - else: - can_try_pidfd_open = False + def pidfd_open(fd: int, flags: int) -> int: + ... + + +else: + can_try_pidfd_open = True + try: + from os import pidfd_open + except ImportError: + if sys.platform == "linux": + import ctypes + + _cdll_for_pidfd_open = ctypes.CDLL(None, use_errno=True) + _cdll_for_pidfd_open.syscall.restype = ctypes.c_long + # pid and flags are actually int-sized, but the syscall() function + # always takes longs. (Except on x32 where long is 32-bits and syscall + # takes 64-bit arguments. But in the unlikely case that anyone is + # using x32, this will still work, b/c we only need to pass in 32 bits + # of data, and the C ABI doesn't distinguish between passing 32-bit vs + # 64-bit integers; our 32-bit values will get loaded into 64-bit + # registers where syscall() will find them.) + _cdll_for_pidfd_open.syscall.argtypes = [ + ctypes.c_long, # syscall number + ctypes.c_long, # pid + ctypes.c_long, # flags + ] + __NR_pidfd_open = 434 + + def pidfd_open(fd: int, flags: int) -> int: + result = _cdll_for_pidfd_open.syscall(__NR_pidfd_open, fd, flags) + if result < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + return result + + else: + can_try_pidfd_open = False class Process(AsyncResource, metaclass=NoPublicConstructor): @@ -442,10 +453,10 @@ async def run_process( :attr:`~subprocess.CompletedProcess.stderr` attributes of the returned :class:`~subprocess.CompletedProcess` object. The value for any stream that was not captured will be ``None``. - + If you want to capture both stdout and stderr while keeping them separate, pass ``capture_stdout=True, capture_stderr=True``. - + If you want to capture both stdout and stderr but mixed together in the order they were printed, use: ``capture_stdout=True, stderr=subprocess.STDOUT``. This directs the child's stderr into its stdout, so the combined diff --git a/trio/_subprocess_platform/__init__.py b/trio/_subprocess_platform/__init__.py index 68132cbf67..37797424ab 100644 --- a/trio/_subprocess_platform/__init__.py +++ b/trio/_subprocess_platform/__init__.py @@ -1,12 +1,17 @@ # Platform-specific subprocess bits'n'pieces. import os -from typing import Tuple +import sys +from typing import Optional, Tuple, TYPE_CHECKING from .. import _core, _subprocess from .._abc import SendStream, ReceiveStream +_wait_child_exiting_error: Optional[ImportError] = None +_create_child_pipe_error: Optional[ImportError] = None + + # Fallback versions of the functions provided -- implementations # per OS are imported atop these at the bottom of the module. async def wait_child_exiting(process: "_subprocess.Process") -> None: @@ -21,7 +26,7 @@ async def wait_child_exiting(process: "_subprocess.Process") -> None: consumed by this call, since :class:`~subprocess.Popen` wants to be able to do that itself. """ - raise NotImplementedError from wait_child_exiting._error # pragma: no cover + raise NotImplementedError from _wait_child_exiting_error # pragma: no cover def create_pipe_to_child_stdin() -> Tuple[SendStream, int]: @@ -34,9 +39,7 @@ def create_pipe_to_child_stdin() -> Tuple[SendStream, int]: something suitable for passing as the ``stdin`` argument of :class:`subprocess.Popen`. """ - raise NotImplementedError from ( # pragma: no cover - create_pipe_to_child_stdin._error - ) + raise NotImplementedError from _create_child_pipe_error # pragma: no cover def create_pipe_from_child_output() -> Tuple[ReceiveStream, int]: @@ -50,23 +53,25 @@ def create_pipe_from_child_output() -> Tuple[ReceiveStream, int]: something suitable for passing as the ``stdin`` argument of :class:`subprocess.Popen`. """ - raise NotImplementedError from ( # pragma: no cover - create_pipe_to_child_stdin._error - ) + raise NotImplementedError from _create_child_pipe_error # pragma: no cover try: - if os.name == "nt": + if sys.platform == "win32": from .windows import wait_child_exiting # noqa: F811 - elif hasattr(_core, "wait_kevent"): + elif sys.platform != "linux" and (TYPE_CHECKING or hasattr(_core, "wait_kevent")): from .kqueue import wait_child_exiting # noqa: F811 else: from .waitid import wait_child_exiting # noqa: F811 except ImportError as ex: # pragma: no cover - wait_child_exiting._error = ex + _wait_child_exiting_error = ex try: - if os.name == "posix": + if TYPE_CHECKING: + # Not worth type checking these definitions + pass + + elif os.name == "posix": from ..lowlevel import FdStream def create_pipe_to_child_stdin(): # noqa: F811 @@ -103,5 +108,4 @@ def create_pipe_from_child_output(): # noqa: F811 raise ImportError("pipes not implemented on this platform") except ImportError as ex: # pragma: no cover - create_pipe_to_child_stdin._error = ex - create_pipe_from_child_output._error = ex + _create_child_pipe_error = ex diff --git a/trio/_subprocess_platform/kqueue.py b/trio/_subprocess_platform/kqueue.py index 17e2df5c6f..5907f8e963 100644 --- a/trio/_subprocess_platform/kqueue.py +++ b/trio/_subprocess_platform/kqueue.py @@ -1,6 +1,10 @@ +import sys import select +from typing import TYPE_CHECKING from .. import _core, _subprocess +assert (sys.platform != "win32" and sys.platform != "linux") or not TYPE_CHECKING + async def wait_child_exiting(process: "_subprocess.Process") -> None: kqueue = _core.current_kqueue() diff --git a/trio/_subprocess_platform/waitid.py b/trio/_subprocess_platform/waitid.py index 81fdf88884..cba5a15163 100644 --- a/trio/_subprocess_platform/waitid.py +++ b/trio/_subprocess_platform/waitid.py @@ -102,6 +102,7 @@ async def wait_child_exiting(process: "_subprocess.Process") -> None: # process. if process._wait_for_exit_data is None: - process._wait_for_exit_data = event = Event() + process._wait_for_exit_data = event = Event() # type: ignore _core.spawn_system_task(_waitid_system_task, process.pid, event) + assert isinstance(process._wait_for_exit_data, Event) await process._wait_for_exit_data.wait() diff --git a/trio/_util.py b/trio/_util.py index 03b79065e2..4f5475eff9 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -1,3 +1,5 @@ +# coding: utf-8 + # Little utilities we use internally from abc import ABCMeta @@ -279,7 +281,7 @@ def __getitem__(self, _): # inherit from these metaclasses. Fortunately, GenericMeta inherits from # ABCMeta, so inheriting from GenericMeta alone is sufficient (when it # exists at all). -if hasattr(t, "GenericMeta"): +if not t.TYPE_CHECKING and hasattr(t, "GenericMeta"): BaseMeta = t.GenericMeta else: BaseMeta = ABCMeta @@ -323,6 +325,9 @@ def __new__(cls, name, bases, cls_namespace): return super().__new__(cls, name, bases, cls_namespace) +T = t.TypeVar("T") + + class NoPublicConstructor(Final): """Metaclass that enforces a class to be final (i.e., subclass not allowed) and ensures a private constructor. @@ -342,10 +347,10 @@ class SomeClass(metaclass=NoPublicConstructor): - TypeError if a sub class or an instance is created. """ - def __call__(self, *args, **kwargs): + def __call__(cls, *args, **kwargs): raise TypeError( - f"{self.__module__}.{self.__qualname__} has no public constructor" + f"{cls.__module__}.{cls.__qualname__} has no public constructor" ) - def _create(self, *args, **kwargs): - return super().__call__(*args, **kwargs) + def _create(cls: t.Type[T], *args: t.Any, **kwargs: t.Any) -> T: + return super().__call__(*args, **kwargs) # type: ignore diff --git a/trio/_windows_pipes.py b/trio/_windows_pipes.py index 81b9834b19..fb420535f4 100644 --- a/trio/_windows_pipes.py +++ b/trio/_windows_pipes.py @@ -1,8 +1,12 @@ +import sys +from typing import TYPE_CHECKING from . import _core from ._abc import SendStream, ReceiveStream from ._util import ConflictDetector, Final from ._core._windows_cffi import _handle, raise_winerror, kernel32, ffi +assert sys.platform == "win32" or not TYPE_CHECKING + # XX TODO: don't just make this up based on nothing. DEFAULT_RECEIVE_SIZE = 65536 diff --git a/trio/lowlevel.py b/trio/lowlevel.py index b54b3dba52..8e6dfc5ee4 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -3,8 +3,9 @@ but useful for extending Trio's functionality. """ -import os +import select as _select import sys +import typing as _t # This is the union of a subset of trio/_core/ and some things from trio/*.py. # See comments in trio/__init__.py for details. To make static analysis easier, @@ -45,24 +46,8 @@ start_guest_run, ) -# Unix-specific symbols -try: - from ._unix_pipes import FdStream -except ImportError: - pass - -# Kqueue-specific symbols -try: - from ._core import ( - current_kqueue, - monitor_kevent, - wait_kevent, - ) -except ImportError: - pass - -# Windows symbols -try: +if sys.platform == "win32": + # Windows symbols from ._core import ( current_iocp, register_with_iocp, @@ -71,11 +56,17 @@ readinto_overlapped, write_overlapped, ) -except ImportError: - pass + from ._wait_for_object import WaitForSingleObject +else: + # Unix symbols + from ._unix_pipes import FdStream -from . import _core + # Kqueue-specific symbols + if sys.platform != "linux" and (_t.TYPE_CHECKING or not hasattr(_select, "epoll")): + from ._core import ( + current_kqueue, + monitor_kevent, + wait_kevent, + ) -# Import bits from trio/*.py -if sys.platform.startswith("win"): - from ._wait_for_object import WaitForSingleObject +del sys diff --git a/trio/socket.py b/trio/socket.py index 0f20d2f5d4..5402f5bc73 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -7,7 +7,8 @@ # We still have some underscore names though but only a few. from . import _socket -import sys as _sys +import sys +import typing as _t # The socket module exports a bunch of platform-specific constants. We want to # re-export them. Since the exact set of constants varies depending on Python @@ -21,7 +22,7 @@ # kept up to date. try: # fmt: off - from socket import ( + from socket import ( # type: ignore CMSG_LEN, CMSG_SPACE, CAPI, AF_UNSPEC, AF_INET, AF_UNIX, AF_IPX, AF_APPLETALK, AF_INET6, AF_ROUTE, AF_LINK, AF_SNA, PF_SYSTEM, AF_SYSTEM, SOCK_STREAM, SOCK_DGRAM, SOCK_RAW, SOCK_SEQPACKET, SOCK_RDM, @@ -126,8 +127,8 @@ # have: import socket as _stdlib_socket -_bad_symbols = set() -if _sys.platform == "win32": +_bad_symbols: _t.Set[str] = set() +if sys.platform == "win32": # See https://github.com/python-trio/trio/issues/39 # Do not import for windows platform # (you can still get it from stdlib socket, of course, if you want it) @@ -136,7 +137,7 @@ globals().update( { _name: getattr(_stdlib_socket, _name) - for _name in _stdlib_socket.__all__ + for _name in _stdlib_socket.__all__ # type: ignore if _name.isupper() and _name not in _bad_symbols } ) @@ -156,10 +157,11 @@ ) # not always available so expose only if -try: - from ._socket import fromshare -except ImportError: - pass +if sys.platform == "win32" or not _t.TYPE_CHECKING: + try: + from ._socket import fromshare + except ImportError: + pass # expose these functions to trio.socket from socket import ( @@ -176,27 +178,34 @@ ) # not always available so expose only if -try: - from socket import sethostname, if_nameindex, if_nametoindex, if_indextoname -except ImportError: - pass +if sys.platform != "win32" or not _t.TYPE_CHECKING: + try: + from socket import sethostname, if_nameindex, if_nametoindex, if_indextoname + except ImportError: + pass # get names used by Trio that we define on our own from ._socket import IPPROTO_IPV6 # Not defined in all python versions and platforms but sometimes needed -try: - TCP_NOTSENT_LOWAT -except NameError: - # Hopefully will show up in 3.7: - # https://github.com/python/cpython/pull/477 - if _sys.platform == "darwin": - TCP_NOTSENT_LOWAT = 0x201 - elif _sys.platform == "linux": - TCP_NOTSENT_LOWAT = 25 +if not _t.TYPE_CHECKING: + try: + TCP_NOTSENT_LOWAT + except NameError: + # Hopefully will show up in 3.7: + # https://github.com/python/cpython/pull/477 + if sys.platform == "darwin": + TCP_NOTSENT_LOWAT = 0x201 + elif sys.platform == "linux": + TCP_NOTSENT_LOWAT = 25 -try: - IP_BIND_ADDRESS_NO_PORT -except NameError: - if _sys.platform == "linux": - IP_BIND_ADDRESS_NO_PORT = 24 +if _t.TYPE_CHECKING: + IP_BIND_ADDRESS_NO_PORT: int +else: + try: + IP_BIND_ADDRESS_NO_PORT + except NameError: + if sys.platform == "linux": + IP_BIND_ADDRESS_NO_PORT = 24 + +del sys diff --git a/trio/tests/test_windows_pipes.py b/trio/tests/test_windows_pipes.py index 8fb29b632f..361cd64ce2 100644 --- a/trio/tests/test_windows_pipes.py +++ b/trio/tests/test_windows_pipes.py @@ -2,18 +2,22 @@ import select import os +import sys import pytest from .._core.tests.tutil import gc_collect_harder from .. import _core, move_on_after from ..testing import wait_all_tasks_blocked, check_one_way_stream -windows = os.name == "nt" -pytestmark = pytest.mark.skipif(not windows, reason="windows only") -if windows: +if sys.platform == "win32": from .._windows_pipes import PipeSendStream, PipeReceiveStream from .._core._windows_cffi import _handle, kernel32 from asyncio.windows_utils import pipe +else: + pytestmark = pytest.mark.skip(reason="windows only") + pipe = None # type: Any + PipeSendStream = None # type: Any + PipeReceiveStream = None # type: Any async def make_pipe() -> "Tuple[PipeSendStream, PipeReceiveStream]": From 02066df418baab1656dec640ff0a17e0f1085a22 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 1 Jul 2020 21:40:57 +0000 Subject: [PATCH 0352/1498] Copy .coveragerc into the empty dir so that codecov.sh sees our ignore patterns --- ci.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 87fa3c4d77..a505cc239b 100755 --- a/ci.sh +++ b/ci.sh @@ -441,7 +441,12 @@ else INSTALLDIR=$(python -c "import os, trio; print(os.path.dirname(trio.__file__))") cp ../setup.cfg $INSTALLDIR - if pytest -W error -r a --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --cov="$INSTALLDIR" --cov-config=../.coveragerc --verbose; then + # We have to copy .coveragerc into this directory, rather than passing + # --cov-config=../.coveragerc to pytest, because codecov.sh will run + # 'coverage xml' to generate the report that it uses, and that will only + # apply the ignore patterns in the current directory's .coveragerc. + cp ../.coveragerc . + if pytest -W error -r a --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --cov="$INSTALLDIR" --verbose; then PASSED=true else PASSED=false From 73baa4655b7951e8112acb8d2dc0f6db4899236b Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 1 Jul 2020 22:35:14 +0000 Subject: [PATCH 0353/1498] Final coverage tweaks --- trio/tests/test_exports.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 510a0987b2..40c1586578 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -33,12 +33,11 @@ def test_core_is_properly_reexported(): def public_modules(module): yield module for name, class_ in module.__dict__.items(): - # Deprecated classes are exported with a leading underscore - if name.startswith("_"): # pragma: nocover + if name.startswith("_"): # pragma: no cover continue if not isinstance(class_, types.ModuleType): continue - if not class_.__name__.startswith(module.__name__): + if not class_.__name__.startswith(module.__name__): # pragma: no cover continue if class_ is module: continue @@ -120,7 +119,7 @@ def test_classes_are_final(): if not isinstance(class_, type): continue # Deprecated classes are exported with a leading underscore - if name.startswith("_"): # pragma: nocover + if name.startswith("_"): # pragma: no cover continue # Abstract classes can be subclassed, because that's the whole From 6e409961e26775c0731ad2489161a3187113afb8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 2 Jul 2020 22:53:50 -0400 Subject: [PATCH 0354/1498] Save an s --- newsfragments/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/README.rst b/newsfragments/README.rst index 51dc92290b..349e67eec0 100644 --- a/newsfragments/README.rst +++ b/newsfragments/README.rst @@ -6,7 +6,7 @@ message and PR description, which are a description of the change as relevant to people working on the code itself.) Each file should be named like ``..rst``, where -```` is an issue numbers, and ```` is one of: +```` is an issue number, and ```` is one of: * ``headline``: a major new feature we want to highlight for users * ``breaking``: any breaking changes that happen without a proper From bfdb3ff7ed873ce7661f9e0f484e795aac15735b Mon Sep 17 00:00:00 2001 From: Roald Storm Date: Fri, 3 Jul 2020 09:48:35 +0200 Subject: [PATCH 0355/1498] Update docs/source/reference-core.rst Co-authored-by: Joshua Oreman --- docs/source/reference-core.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 428a75e7b3..24f9ab52e0 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -677,7 +677,11 @@ nothing at all:: with move_on_after(TIMEOUT): # don't do this! nursery.start_soon(child) -The reason for this is that the ``start_soon`` call immediately returns and the scheduled task actually runs inside the nursery. So in the above snippet the contextmanager ``with move_on_after(TIMEOUT):`` block is immediately exited after ``start_soon`` returns. Since ``move_on_after`` only imposes a time limit on the body of its ``with`` block and that block is exited it cannot cancel the child task. +Why is this so? Well, ``start_soon()`` returns as soon as it has scheduled the new task to start running. The flow of execution in the parent then continues on to exit the ``with move_on_after(TIMEOUT):`` block, at which point Trio forgets about the timeout entirely. In order for the timeout to apply to the child task, Trio must be able to tell that its associated cancel scope will stay open for at least as long as the child task is executing. And Trio can only know that for sure if the cancel scope block is outside the nursery block. + +You might wonder why Trio can't just remember "this task should be cancelled in ``TIMEOUT`` seconds", even after the ``with move_on_after(TIMEOUT):`` block is gone. The reason has to do with :ref:`how cancellation is implemented `. Recall that cancellation is represented by a `Cancelled` exception, which eventually needs to be caught by the cancel scope that caused it. (Otherwise, the exception would take down your whole program!) In order to be able to cancel the child tasks, the cancel scope has to be able to "see" the `Cancelled` exceptions that they raise -- and those exceptions come out of the ``async with open_nursery()`` block, not out of the call to ``start_soon()``. + +If you want a timeout to apply to one task but not another, then you need to put the cancel scope in that individual task's function -- ``child()``, in this example. Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 97d45e6c77da70ebd727810031258e08ea346242 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Sat, 4 Jul 2020 10:38:06 +0900 Subject: [PATCH 0356/1498] add per-timer to awesome libraries --- docs/source/awesome-trio-libraries.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 479f74943d..efd4c0e0ef 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -85,6 +85,7 @@ Tools and Utilities * `trio-util `__ - An assortment of utilities for the Trio async/await framework. * `tricycle `__ - This is a library of interesting-but-maybe-not-yet-fully-proven extensions to Trio. * `tenacity `__ - Retrying library for Python with async/await support. +* `perf-timer `__ - A code timer with Trio async support (see `TrioPerfTimer`). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers `trio_perf_counter()` for low-level timing. Trio/Asyncio Interoperability From 8083fce64d1a04858398f1791996a82b2016226d Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 4 Jul 2020 08:38:42 +0000 Subject: [PATCH 0357/1498] Windows: Deal with Komodia-based LSPs that block SIO_BASE_HANDLE --- newsfragments/1659.bugfix.rst | 7 +++ trio/_core/_io_windows.py | 95 ++++++++++++++++++++++++---- trio/_core/_windows_cffi.py | 2 + trio/_core/tests/test_windows.py | 103 +++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+), 13 deletions(-) create mode 100644 newsfragments/1659.bugfix.rst diff --git a/newsfragments/1659.bugfix.rst b/newsfragments/1659.bugfix.rst new file mode 100644 index 0000000000..395a369002 --- /dev/null +++ b/newsfragments/1659.bugfix.rst @@ -0,0 +1,7 @@ +On Windows, Trio now works around the buggy behavior of certain +Layered Service Providers (system components that can intercept +network activity) that are built on top of a commercially available +library called Komodia Redirector. This benefits users of products +such as Astrill VPN and Qustodio parental controls. Previously, Trio +would crash on startup when run on a system where such a product was +installed. diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 40cd8ba5b0..618d0283b6 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -185,7 +185,7 @@ def _check(success): return success -def _get_base_socket(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): +def _get_underlying_socket(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): if hasattr(sock, "fileno"): sock = sock.fileno() base_ptr = ffi.new("HANDLE *") @@ -207,6 +207,53 @@ def _get_base_socket(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): return base_ptr[0] +def _get_base_socket(sock): + # There is a development kit for LSPs called Komodia Redirector. + # It does some unusual (some might say evil) things like intercepting + # SIO_BASE_HANDLE (fails) and SIO_BSP_HANDLE_SELECT (returns the same + # socket) in a misguided attempt to prevent bypassing it. It's been used + # in malware including the infamous Lenovo Superfish incident from 2015, + # but unfortunately is also used in some legitimate products such as + # parental control tools and Astrill VPN. Komodia happens to not + # block SIO_BSP_HANDLE_POLL, so we'll try SIO_BASE_HANDLE and fall back + # to SIO_BSP_HANDLE_POLL if it doesn't work. + # References: + # - https://github.com/piscisaureus/wepoll/blob/0598a791bf9cbbf480793d778930fc635b044980/wepoll.c#L2223 + # - https://github.com/tokio-rs/mio/issues/1314 + + while True: + try: + # If this is not a Komodia-intercepted socket, we can just use + # SIO_BASE_HANDLE. + return _get_underlying_socket(sock) + except OSError as ex: + if ex.winerror == ErrorCodes.ERROR_NOT_SOCKET: + # SIO_BASE_HANDLE might fail even without LSP intervention, + # if we get something that's not a socket. + raise + if hasattr(sock, "fileno"): + sock = sock.fileno() + sock = _handle(sock) + next_sock = _get_underlying_socket( + sock, which=WSAIoctls.SIO_BSP_HANDLE_POLL + ) + if next_sock == sock: + # If BSP_HANDLE_POLL returns the same socket we already had, + # then there's no layering going on and we need to fail + # to prevent an infinite loop. + raise RuntimeError( + "Unexpected network configuration detected: " + "SIO_BASE_HANDLE failed and SIO_BSP_HANDLE_POLL didn't " + "return a different socket. Please file a bug at " + "https://github.com/python-trio/trio/issues/new, " + "and include the output of running: " + "netsh winsock show catalog" + ) + # Otherwise we've gotten at least one layer deeper, so + # loop back around to keep digging. + sock = next_sock + + def _afd_helper_handle(): # The "AFD" driver is exposed at the NT path "\Device\Afd". We're using # the Win32 CreateFile, though, so we have to pass a Win32 path. \\.\ is @@ -349,19 +396,41 @@ def __init__(self): self._completion_key_counter = itertools.count(CKeys.USER_DEFINED) with socket.socket() as s: - # LSPs can't override this. - base_handle = _get_base_socket(s, which=WSAIoctls.SIO_BASE_HANDLE) + # We assume we're not working with any LSP that changes + # how select() is supposed to work. Validate this by + # ensuring that the result of SIO_BSP_HANDLE_SELECT (the + # LSP-hookable mechanism for "what should I use for + # select()?") matches that of SIO_BASE_HANDLE ("what is + # the real non-hooked underlying socket here?"). + # + # This doesn't work for Komodia-based LSPs; see the comments + # in _get_base_socket() for details. But we have special + # logic for those, so we just skip this check if + # SIO_BASE_HANDLE fails. + # LSPs can in theory override this, but we believe that it never - # actually happens in the wild. - select_handle = _get_base_socket(s, which=WSAIoctls.SIO_BSP_HANDLE_SELECT) - if base_handle != select_handle: # pragma: no cover - raise RuntimeError( - "Unexpected network configuration detected. " - "Please file a bug at " - "https://github.com/python-trio/trio/issues/new, " - "and include the output of running: " - "netsh winsock show catalog" - ) + # actually happens in the wild (except Komodia) + select_handle = _get_underlying_socket( + s, which=WSAIoctls.SIO_BSP_HANDLE_SELECT + ) + try: + # LSPs shouldn't override this... + base_handle = _get_underlying_socket(s, which=WSAIoctls.SIO_BASE_HANDLE) + except OSError: + # But Komodia-based LSPs do anyway, in a way that causes + # a failure with WSAEFAULT. We have special handling for + # them in _get_base_socket(). Make sure it works. + _get_base_socket(s) + else: + if base_handle != select_handle: + raise RuntimeError( + "Unexpected network configuration detected: " + "SIO_BASE_HANDLE and SIO_BSP_HANDLE_SELECT differ. " + "Please file a bug at " + "https://github.com/python-trio/trio/issues/new, " + "and include the output of running: " + "netsh winsock show catalog" + ) def close(self): try: diff --git a/trio/_core/_windows_cffi.py b/trio/_core/_windows_cffi.py index 58845b86f4..a1071519e9 100644 --- a/trio/_core/_windows_cffi.py +++ b/trio/_core/_windows_cffi.py @@ -244,6 +244,7 @@ class ErrorCodes(enum.IntEnum): ERROR_INVALID_HANDLE = 6 ERROR_INVALID_PARMETER = 87 ERROR_NOT_FOUND = 1168 + ERROR_NOT_SOCKET = 10038 class FileFlags(enum.IntEnum): @@ -283,6 +284,7 @@ class AFDPollFlags(enum.IntFlag): class WSAIoctls(enum.IntEnum): SIO_BASE_HANDLE = 0x48000022 SIO_BSP_HANDLE_SELECT = 0x4800001C + SIO_BSP_HANDLE_POLL = 0x4800001D class CompletionModes(enum.IntFlag): diff --git a/trio/_core/tests/test_windows.py b/trio/_core/tests/test_windows.py index 0a8179f88f..ef67b48497 100644 --- a/trio/_core/tests/test_windows.py +++ b/trio/_core/tests/test_windows.py @@ -171,3 +171,106 @@ async def test_too_late_to_cancel(): # fallback completion that was posted when CancelIoEx failed. assert await _core.readinto_overlapped(read_handle, target) == 6 assert target[:6] == b"test2\n" + + +def test_lsp_that_hooks_select_gives_good_error(monkeypatch): + from .._windows_cffi import WSAIoctls, _handle + from .. import _io_windows + + def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): + if hasattr(sock, "fileno"): + sock = sock.fileno() + if which == WSAIoctls.SIO_BSP_HANDLE_SELECT: + return _handle(sock + 1) + else: + return _handle(sock) + + monkeypatch.setattr(_io_windows, "_get_underlying_socket", patched_get_underlying) + with pytest.raises( + RuntimeError, match="SIO_BASE_HANDLE and SIO_BSP_HANDLE_SELECT differ" + ): + _core.run(sleep, 0) + + +def test_komodia_behavior(monkeypatch): + # We can't install an actual Komodia LSP (they're all commercial + # products) but we can at least monkeypatch _get_underlying_socket + # to behave like it's been observed to do with a Komodia LSP + # installed, and make sure _get_base_socket DTRT in response. + from .._windows_cffi import WSAIoctls, ffi, _handle + from .. import _io_windows + import socket as stdlib_socket + from ... import socket as trio_socket + import signal + + orig_get_underlying = _io_windows._get_underlying_socket + + def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): + if hasattr(sock, "fileno"): + sock = sock.fileno() + sock = int(ffi.cast("int", sock)) + + # Provide fake behavior based on the low 2 bits of the handle. + # Note that all real socket handles are word-aligned so the low 2 bits + # will be zero (low 3 bits zere on 64-bit systems). + # + # Low bits 00: treat as base socket: always return self + # Low bits 01: BASE_HANDLE fails but HANDLE_POLL returns self --> error + # Low bits 10: treat as layered socket: BASE_HANDLE fails, + # BSP_HANDLE_SELECT returns self, BSP_HANDLE_POLL returns ...00 + # Low bits 11: treat as doubly layered socket: same as ...10 + # except that BSP_HANDLE_POLL returns ...10 + + if sock & 3 == 0: + # This call is needed to make the tests pass if they're run on + # a system with an actual Komodia LSP + return orig_get_underlying(sock, which=which) + + if which == WSAIoctls.SIO_BASE_HANDLE: + if sock & 3: + raise OSError("nope") + if which == WSAIoctls.SIO_BSP_HANDLE_POLL: + if sock & 3 == 3: + return _handle(sock - 1) + if sock & 3 == 2: + return _handle(sock - 2) + return _handle(sock) + + # We exercise the patched _get_underlying_socket by changing socket.fileno() + # to return a value adjusted upwards by 1, 2, or 3, depending on which + # path we want to exercise (error, single-layered, or double-layered). + orig_fileno = stdlib_socket.socket.fileno + delta: int + + def patched_fileno(sock): + if orig_fileno(sock) == -1: + return -1 + return orig_fileno(sock) + delta + + # Finally, we need to make signal.set_wakeup_fd() undo the fileno + # munging -- it's the one other place in Trio where we explicitly + # call fileno() on Windows. + orig_set_wakeup_fd = signal.set_wakeup_fd + + def patched_set_wakeup_fd(fd, **kw): + if fd != -1: + fd = fd & ~3 + return orig_set_wakeup_fd(fd, **kw) + + monkeypatch.setattr(_io_windows, "_get_underlying_socket", patched_get_underlying) + monkeypatch.setattr(stdlib_socket.socket, "fileno", patched_fileno) + monkeypatch.setattr(signal, "set_wakeup_fd", patched_set_wakeup_fd) + + async def main(): + s1, s2 = trio_socket.socketpair() + await s1.send(b"hi") + await _core.wait_readable(s2) + await _core.wait_readable(s2.fileno()) + + for delta in (0, 2, 3): + _core.run(main) + with pytest.raises( + RuntimeError, match="SIO_BASE_HANDLE failed and SIO_BSP_HANDLE_POLL didn't" + ): + delta = 1 + _core.run(main) From af2b808c7ff60464124c06a4ef8881a4a310ab93 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 4 Jul 2020 09:04:42 +0000 Subject: [PATCH 0358/1498] Fix coverage --- trio/_core/tests/test_windows.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/trio/_core/tests/test_windows.py b/trio/_core/tests/test_windows.py index ef67b48497..62559e24a6 100644 --- a/trio/_core/tests/test_windows.py +++ b/trio/_core/tests/test_windows.py @@ -178,7 +178,7 @@ def test_lsp_that_hooks_select_gives_good_error(monkeypatch): from .. import _io_windows def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): - if hasattr(sock, "fileno"): + if hasattr(sock, "fileno"): # pragma: no branch sock = sock.fileno() if which == WSAIoctls.SIO_BSP_HANDLE_SELECT: return _handle(sock + 1) @@ -227,8 +227,7 @@ def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): return orig_get_underlying(sock, which=which) if which == WSAIoctls.SIO_BASE_HANDLE: - if sock & 3: - raise OSError("nope") + raise OSError("nope") if which == WSAIoctls.SIO_BSP_HANDLE_POLL: if sock & 3 == 3: return _handle(sock - 1) @@ -243,7 +242,7 @@ def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): delta: int def patched_fileno(sock): - if orig_fileno(sock) == -1: + if orig_fileno(sock) == -1: # pragma: no cover return -1 return orig_fileno(sock) + delta From 07c54de19df5bd5ba5d859bc5542ef878d4af01f Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Sat, 4 Jul 2020 20:13:56 +0900 Subject: [PATCH 0359/1498] fix doc build --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index efd4c0e0ef..3122c52145 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -85,7 +85,7 @@ Tools and Utilities * `trio-util `__ - An assortment of utilities for the Trio async/await framework. * `tricycle `__ - This is a library of interesting-but-maybe-not-yet-fully-proven extensions to Trio. * `tenacity `__ - Retrying library for Python with async/await support. -* `perf-timer `__ - A code timer with Trio async support (see `TrioPerfTimer`). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers `trio_perf_counter()` for low-level timing. +* `perf-timer `__ - A code timer with Trio async support (see `TrioPerfTimer`). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers ``trio_perf_counter()`` for low-level timing. Trio/Asyncio Interoperability From 8ccd43ef6380e91c5e8cd5f36f3f887ed99fda10 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Jul 2020 04:02:34 -0700 Subject: [PATCH 0360/1498] Run tests with the "Astrill VPN" installed This is especially difficult for us to support, because it uses the "Komodia" LSP code, which contains a hack to intentionally break attempts to work around LSPs. See: https://github.com/tokio-rs/mio/issues/1314#issuecomment-626654811 https://github.com/python-trio/trio/issues/1659 --- .github/workflows/astrill-codesigning-cert.cer | Bin 0 -> 1214 bytes .github/workflows/ci.yml | 8 ++++++++ ci.sh | 7 +++++++ 3 files changed, 15 insertions(+) create mode 100644 .github/workflows/astrill-codesigning-cert.cer diff --git a/.github/workflows/astrill-codesigning-cert.cer b/.github/workflows/astrill-codesigning-cert.cer new file mode 100644 index 0000000000000000000000000000000000000000..58cc0c05fa74fa22596d31c40e25803560781e8c GIT binary patch literal 1214 zcmXqLV%cTT#Jp$$GZP~dlaQcd?X8aem*vhf|8iMnnO>w}Y{1LLsnzDu_MMlJk(HIf zAkdK8fRl|ml!Z;0$;s7F(m))<;Sv^b&&f|p%n8m+&r`@N(=ARkR5wrsNihq{AxSys zr=)_^WagzSI6ErnD!3aN$cghBni!ZG85vrbniyF|0l8)d7I3aX0FsrCp@uvLTp)8f zgqi(@IZ_A zmG~>aZB3rHh?kt&US;59scn-a{^i2rvQ?k8y%yVVU|#3E`Px2%6FCvmGsL7Hx9(UF zK4<44wGE<=y-X9U&e;lI}UtiR9x!v*b~lvNJJ?sb!Ru6ck(O>z9|8>!pJeM=>zs z^^)^*^@~yq@{2P|@{20`_L44PU3F^J zDaWsJGBYI~mH#s`kOxUCvq%_-HHdKJ-Pks#YyVrj>xWBX&os!d&=06WPO!j?2u!ey z4E)Ai$@eWAR&#%8T>pMT^p>~)9eL|uai#e-b#bS!Ch6AH95>=hTQhST>kXzHDHCb$ z*#>7r*^^bCo|`byitDTMT#k8@YHtZw8gJB|^V-$ReW&%ohwM9bUY3<@@!1^uDVMM8 zs)_^4eV5OE{&OUM9@y<(Q>~^{r50k%`@J;z(%Uz>&$Rg^^snEUd(`@OhPPO^mDS=D zmCy68J@Vdi_6N(FTknIb4+s{er19iO-2JlSQ+>4fKAlfHswQvVskL?2&M&vSd;R_P zZ!Gti{a0D(4BLb=w$t;CrN2!Tm3#H@#?{JRp3>VIVOF2Cmp_@N|JR#m)3uW&q3)T} PANJf){ru}-Kj$|9E4Z+n literal 0 HcmV?d00001 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9592be955a..a6d49fcc83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,14 @@ jobs: lsp: [''] extra_name: [''] include: + - python: '3.8' + arch: 'x64' + # This is a SUPER sketchy source for this -- I assume it + # has some kind of embedded malware -- but the real + # download site has the downloads locked behind a + # registration wall. + lsp: 'http://fs3.softfamous.com/downloads/tname-110575cb0f191/software/astrill-setup-win.exe' + extra_name: ', with Komodia LSP' - python: '3.8' arch: 'x64' lsp: 'http://www.proxifier.com/download/ProxifierSetup.exe' diff --git a/ci.sh b/ci.sh index a505cc239b..c4e046b986 100755 --- a/ci.sh +++ b/ci.sh @@ -421,6 +421,13 @@ else # machine for testing. So MITM attacks are really the least of our # worries. curl-harder --insecure -o lsp-installer.exe "$LSP" + # This is only needed for the Astrill LSP, but there's no harm in + # doing it all the time. The cert was manually extracted by installing + # the package in a VPN, clicking "Always trust from this publisher" + # when installing, and then running 'certmgr.msc' and exporting the + # certificate. See: + # http://www.migee.com/2010/09/24/solution-for-unattendedsilent-installs-and-would-you-like-to-install-this-device-software/ + certutil -addstore "TrustedPublisher" .github/workflows/astrill-codesigning-cert.cer # Double-slashes are how you tell windows-bash that you want a single # slash, and don't treat this as a unix-style filename that needs to # be replaced by a windows-style filename. From 23c3fcb076bb1fc10a3b034c7fb7faee34c3bfc3 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 4 Jul 2020 21:37:56 +0000 Subject: [PATCH 0361/1498] Simplify extra test to only test the error path --- trio/_core/tests/test_windows.py | 83 +++++--------------------------- 1 file changed, 12 insertions(+), 71 deletions(-) diff --git a/trio/_core/tests/test_windows.py b/trio/_core/tests/test_windows.py index 62559e24a6..230662cbaf 100644 --- a/trio/_core/tests/test_windows.py +++ b/trio/_core/tests/test_windows.py @@ -192,84 +192,25 @@ def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): _core.run(sleep, 0) -def test_komodia_behavior(monkeypatch): - # We can't install an actual Komodia LSP (they're all commercial - # products) but we can at least monkeypatch _get_underlying_socket - # to behave like it's been observed to do with a Komodia LSP - # installed, and make sure _get_base_socket DTRT in response. - from .._windows_cffi import WSAIoctls, ffi, _handle - from .. import _io_windows - import socket as stdlib_socket - from ... import socket as trio_socket - import signal +def test_lsp_that_completely_hides_base_socket_gives_good_error(monkeypatch): + # This tests behavior with an LSP that fails SIO_BASE_HANDLE and returns + # self for SIO_BSP_HANDLE_SELECT (like Komodia), but also returns + # self for SIO_BSP_HANDLE_POLL. No known LSP does this, but we want to + # make sure we get an error rather than an infinite loop. - orig_get_underlying = _io_windows._get_underlying_socket + from .._windows_cffi import WSAIoctls, _handle + from .. import _io_windows def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): - if hasattr(sock, "fileno"): + if hasattr(sock, "fileno"): # pragma: no branch sock = sock.fileno() - sock = int(ffi.cast("int", sock)) - - # Provide fake behavior based on the low 2 bits of the handle. - # Note that all real socket handles are word-aligned so the low 2 bits - # will be zero (low 3 bits zere on 64-bit systems). - # - # Low bits 00: treat as base socket: always return self - # Low bits 01: BASE_HANDLE fails but HANDLE_POLL returns self --> error - # Low bits 10: treat as layered socket: BASE_HANDLE fails, - # BSP_HANDLE_SELECT returns self, BSP_HANDLE_POLL returns ...00 - # Low bits 11: treat as doubly layered socket: same as ...10 - # except that BSP_HANDLE_POLL returns ...10 - - if sock & 3 == 0: - # This call is needed to make the tests pass if they're run on - # a system with an actual Komodia LSP - return orig_get_underlying(sock, which=which) - if which == WSAIoctls.SIO_BASE_HANDLE: raise OSError("nope") - if which == WSAIoctls.SIO_BSP_HANDLE_POLL: - if sock & 3 == 3: - return _handle(sock - 1) - if sock & 3 == 2: - return _handle(sock - 2) - return _handle(sock) - - # We exercise the patched _get_underlying_socket by changing socket.fileno() - # to return a value adjusted upwards by 1, 2, or 3, depending on which - # path we want to exercise (error, single-layered, or double-layered). - orig_fileno = stdlib_socket.socket.fileno - delta: int - - def patched_fileno(sock): - if orig_fileno(sock) == -1: # pragma: no cover - return -1 - return orig_fileno(sock) + delta - - # Finally, we need to make signal.set_wakeup_fd() undo the fileno - # munging -- it's the one other place in Trio where we explicitly - # call fileno() on Windows. - orig_set_wakeup_fd = signal.set_wakeup_fd - - def patched_set_wakeup_fd(fd, **kw): - if fd != -1: - fd = fd & ~3 - return orig_set_wakeup_fd(fd, **kw) + else: + return _handle(sock) monkeypatch.setattr(_io_windows, "_get_underlying_socket", patched_get_underlying) - monkeypatch.setattr(stdlib_socket.socket, "fileno", patched_fileno) - monkeypatch.setattr(signal, "set_wakeup_fd", patched_set_wakeup_fd) - - async def main(): - s1, s2 = trio_socket.socketpair() - await s1.send(b"hi") - await _core.wait_readable(s2) - await _core.wait_readable(s2.fileno()) - - for delta in (0, 2, 3): - _core.run(main) with pytest.raises( - RuntimeError, match="SIO_BASE_HANDLE failed and SIO_BSP_HANDLE_POLL didn't" + RuntimeError, match="SIO_BASE_HANDLE failed SIO_BSP_HANDLE_POLL didn't return" ): - delta = 1 - _core.run(main) + _core.run(sleep, 0) From 03b132d156d7032aa2d269a4367f1bb622168318 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 4 Jul 2020 21:45:28 +0000 Subject: [PATCH 0362/1498] Search for the right error message --- trio/_core/tests/test_windows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/test_windows.py b/trio/_core/tests/test_windows.py index 230662cbaf..563e157e78 100644 --- a/trio/_core/tests/test_windows.py +++ b/trio/_core/tests/test_windows.py @@ -211,6 +211,7 @@ def patched_get_underlying(sock, *, which=WSAIoctls.SIO_BASE_HANDLE): monkeypatch.setattr(_io_windows, "_get_underlying_socket", patched_get_underlying) with pytest.raises( - RuntimeError, match="SIO_BASE_HANDLE failed SIO_BSP_HANDLE_POLL didn't return" + RuntimeError, + match="SIO_BASE_HANDLE failed and SIO_BSP_HANDLE_POLL didn't return a diff", ): _core.run(sleep, 0) From 29ae2b12302c351dcaf6500e99111a793aacc107 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Sun, 5 Jul 2020 11:05:07 +0900 Subject: [PATCH 0363/1498] fix formatting --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 3122c52145..7ab0379447 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -85,7 +85,7 @@ Tools and Utilities * `trio-util `__ - An assortment of utilities for the Trio async/await framework. * `tricycle `__ - This is a library of interesting-but-maybe-not-yet-fully-proven extensions to Trio. * `tenacity `__ - Retrying library for Python with async/await support. -* `perf-timer `__ - A code timer with Trio async support (see `TrioPerfTimer`). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers ``trio_perf_counter()`` for low-level timing. +* `perf-timer `__ - A code timer with Trio async support (see ``TrioPerfTimer``). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers ``trio_perf_counter()`` for low-level timing. Trio/Asyncio Interoperability From 291ec4bb0ede8429cd644d59ca6f20ec437feb4f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 5 Jul 2020 00:06:03 -0400 Subject: [PATCH 0364/1498] Fix importing trio while sys.excepthook is a partial (#1662) * Fix importing trio while sys.excepthook is a partial Fixes #1630 * black * Update newsfragments/1630.bugfix.rst Co-authored-by: Joshua Oreman * Remove invalid @need_ipython decoration of new test Co-authored-by: Joshua Oreman --- newsfragments/1630.bugfix.rst | 2 ++ trio/_core/_multierror.py | 2 +- trio/_core/tests/test_multierror.py | 9 +++++++++ .../simple_excepthook_partial.py | 13 +++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 newsfragments/1630.bugfix.rst create mode 100644 trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py diff --git a/newsfragments/1630.bugfix.rst b/newsfragments/1630.bugfix.rst new file mode 100644 index 0000000000..0b33777705 --- /dev/null +++ b/newsfragments/1630.bugfix.rst @@ -0,0 +1,2 @@ +Trio can now be imported when `sys.excepthook` is a `functools.partial` instance, which might occur in a +``pytest-qt`` test function. diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 4ea22b8a80..9e0de7cf9f 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -477,7 +477,7 @@ def trio_show_traceback(self, etype, value, tb, tb_offset=None): # hook. # # More details: https://github.com/python-trio/trio/issues/1065 -if sys.excepthook.__name__ == "apport_excepthook": +if getattr(sys.excepthook, "__name__", None) == "apport_excepthook": import apport_python_hook assert sys.excepthook is apport_python_hook.apport_excepthook diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index d922f53a7b..e745ad7c0e 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -694,6 +694,15 @@ def test_ipython_imported_but_unused(): check_simple_excepthook(completed) +@slow +def test_partial_imported_but_unused(): + # Check that a functools.partial as sys.excepthook doesn't cause an exception when + # importing trio. This was a problem due to the lack of a .__name__ attribute and + # happens when inside a pytest-qt test case for example. + completed = run_script("simple_excepthook_partial.py") + completed.check_returncode() + + @slow @need_ipython def test_ipython_custom_exc_handler(): diff --git a/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py b/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py new file mode 100644 index 0000000000..e97fc39d57 --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py @@ -0,0 +1,13 @@ +import functools +import sys + +import _common + +# just making sure importing Trio doesn't fail if sys.excepthook doesn't have a +# .__name__ attribute + +sys.excepthook = functools.partial(sys.excepthook) + +assert not hasattr(sys.excepthook, "__name__") + +import trio From 09ad45cc8e18941ac653722f2e391f5194815c55 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 6 Jul 2020 23:03:17 +0000 Subject: [PATCH 0365/1498] Simplify instrumentation logic --- trio/_core/_instrumentation.py | 98 +++++++++--------------- trio/_core/_run.py | 34 ++++---- trio/_core/tests/test_instrumentation.py | 4 +- 3 files changed, 56 insertions(+), 80 deletions(-) diff --git a/trio/_core/_instrumentation.py b/trio/_core/_instrumentation.py index 80f8645a2b..e14c1ef1e0 100644 --- a/trio/_core/_instrumentation.py +++ b/trio/_core/_instrumentation.py @@ -17,67 +17,20 @@ def _public(fn: F) -> F: return fn -HookImpl = Callable[..., Any] - - -class Hook(Dict[Instrument, HookImpl]): - """Manages installed instruments for a single hook such as before_run(). - - The base dictionary maps each instrument to the method of that - instrument that will be called when the hook is invoked. We use - inheritance so that 'if hook:' is fast (no Python-level function - calls needed). - - """ - - __slots__ = ("_name", "_parent") - - def __init__(self, name: str, parent: "Instruments"): - self._name = name # "before_run" or similar - self._parent = parent - - def __call__(self, *args: Any): - """Invoke the instrumentation hook with the given arguments.""" - for instrument, method in list(self.items()): - try: - method(*args) - except: - self._parent.remove_instrument(instrument) - INSTRUMENT_LOGGER.exception( - "Exception raised when calling %r on instrument %r. " - "Instrument has been disabled.", - self._name, - instrument, - ) - - -class Instruments(Instrument): - """A collection of `trio.abc.Instrument` with some optimizations. +class Instruments(Dict[str, Dict[Instrument, None]]): + """A collection of `trio.abc.Instrument` organized by hook. Instrumentation calls are rather expensive, and we don't want a rarely-used instrument (like before_run()) to slow down hot operations (like before_task_step()). Thus, we cache the set of - handlers to be called for each hook, and skip the instrumentation + instruments to be called for each hook, and skip the instrumentation call if there's nothing currently installed for that hook. - - This inherits from `trio.abc.Instrument` for the benefit of - static type checking (to make sure you pass the right arguments - when calling an instrument). All of the class-level function - definitions are shadowed by instance-level Hooks. """ - # One Hook per instrument, with its same name - __slots__ = [name for name in Instrument.__dict__ if not name.startswith("_")] - - # Maps each installed instrument to the list of hook names that it implements. - _instruments: Dict[Instrument, List[str]] - __slots__.append("_instruments") + __slots__ = () def __init__(self, incoming: Sequence[Instrument]): - self._instruments = {} - for name in Instruments.__slots__: - if not hasattr(self, name): - setattr(self, name, Hook(name, self)) + self["_all"] = {} for instrument in incoming: self.add_instrument(instrument) @@ -91,9 +44,9 @@ def add_instrument(self, instrument: Instrument) -> None: If ``instrument`` is already active, does nothing. """ - if instrument in self._instruments: + if instrument in self["_all"]: return - hooknames = self._instruments[instrument] = [] + self["_all"][instrument] = None try: for name in dir(instrument): if name.startswith("_"): @@ -102,13 +55,11 @@ def add_instrument(self, instrument: Instrument) -> None: prototype = getattr(Instrument, name) except AttributeError: continue - impl: HookImpl = getattr(instrument, name) + impl = getattr(instrument, name) if isinstance(impl, types.MethodType) and impl.__func__ is prototype: # Inherited unchanged from _abc.Instrument continue - hook: Hook = getattr(self, name) - hook[instrument] = impl - hooknames.append(name) + self.setdefault(name, {})[instrument] = None except: self.remove_instrument(instrument) raise @@ -128,7 +79,30 @@ def remove_instrument(self, instrument: Instrument) -> None: """ # If instrument isn't present, the KeyError propagates out - hooknames = self._instruments.pop(instrument) - for name in hooknames: - hook: Hook = getattr(self, name) - del hook[instrument] + self["_all"].pop(instrument) + for hookname, instruments in list(self.items()): + if instrument in instruments: + del instruments[instrument] + if not instruments: + del self[hookname] + + def call(self, hookname: str, *args: Any) -> None: + """Call hookname(*args) on each applicable instrument. + + You must first check whether there are any instruments installed for + that hook, e.g.:: + + if "before_task_step" in instruments: + instruments.call("before_task_step", task) + """ + for instrument in list(self[hookname]): + try: + getattr(instrument, hookname)(*args) + except: + self.remove_instrument(instrument) + INSTRUMENT_LOGGER.exception( + "Exception raised when calling %r on instrument %r. " + "Instrument has been disabled.", + hookname, + instrument, + ) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 9d9725365e..6e90978585 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1306,7 +1306,8 @@ def force_guest_tick_asap(self): def close(self): self.io_manager.close() self.entry_queue.close() - self.instruments.after_run() + if "after_run" in self.instruments: + self.instruments.call("after_run") # This is where KI protection gets disabled, so we do it last self.ki_manager.close() @@ -1407,8 +1408,8 @@ def reschedule(self, task, next_send=_NO_SEND): if not self.runq and self.is_guest: self.force_guest_tick_asap() self.runq.append(task) - if self.instruments.task_scheduled: - self.instruments.task_scheduled(task) + if "task_scheduled" in self.instruments: + self.instruments.call("task_scheduled", task) def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): @@ -1465,8 +1466,8 @@ async def python_wrapper(orig_coro): nursery._children.add(task) task._activate_cancel_status(nursery._cancel_status) - if self.instruments.task_spawned: - self.instruments.task_spawned(task) + if "task_spawned" in self.instruments: + self.instruments.call("task_spawned", task) # Special case: normally next_send should be an Outcome, but for the # very first send we have to send a literal unboxed None. self.reschedule(task, None) @@ -1514,8 +1515,8 @@ def task_exited(self, task, outcome): else: task._parent_nursery._child_finished(task, outcome) - if self.instruments.task_exited: - self.instruments.task_exited(task) + if "task_exited" in self.instruments: + self.instruments.call("task_exited", task) ################ # System tasks and init @@ -2005,7 +2006,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): if not host_uses_signal_set_wakeup_fd: runner.entry_queue.wakeup.wakeup_on_signals() - runner.instruments.before_run() + if "before_run" in runner.instruments: + runner.instruments.call("before_run") runner.clock.start_clock() runner.init_task = runner.spawn_impl( runner.init, (async_fn, args), None, "", system_task=True, @@ -2034,16 +2036,16 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): timeout = runner.clock_autojump_threshold idle_primed = IdlePrimedTypes.AUTOJUMP_CLOCK - if runner.instruments.before_io_wait: - runner.instruments.before_io_wait(timeout) + if "before_io_wait" in runner.instruments: + runner.instruments.call("before_io_wait", timeout) # Driver will call io_manager.get_events(timeout) and pass it back # in throuh the yield events = yield timeout runner.io_manager.process_events(events) - if runner.instruments.after_io_wait: - runner.instruments.after_io_wait(timeout) + if "after_io_wait" in runner.instruments: + runner.instruments.call("after_io_wait", timeout) # Process cancellations due to deadline expiry now = runner.clock.current_time() @@ -2120,8 +2122,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): task = batch.pop() GLOBAL_RUN_CONTEXT.task = task - if runner.instruments.before_task_step: - runner.instruments.before_task_step(task) + if "before_task_step" in runner.instruments: + runner.instruments.call("before_task_step", task) next_send_fn = task._next_send_fn next_send = task._next_send @@ -2183,8 +2185,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): runner.reschedule(task, exc) task._next_send_fn = task.coro.throw - if runner.instruments.after_task_step: - runner.instruments.after_task_step(task) + if "after_task_step" in runner.instruments: + runner.instruments.call("after_task_step", task) del GLOBAL_RUN_CONTEXT.task except GeneratorExit: diff --git a/trio/_core/tests/test_instrumentation.py b/trio/_core/tests/test_instrumentation.py index 580937bab6..946caf7965 100644 --- a/trio/_core/tests/test_instrumentation.py +++ b/trio/_core/tests/test_instrumentation.py @@ -248,7 +248,7 @@ async def main(): # Make sure the instrument is fully removed from the per-method lists runner = _core.current_task()._runner - assert not runner.instruments.after_run - assert not runner.instruments.task_exited + assert "after_run" not in runner.instruments + assert "task_exited" not in runner.instruments _core.run(main) From 9a845c5d33e3309f67bf9d24549496468ae873db Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2020 08:24:03 +0000 Subject: [PATCH 0366/1498] Bump coverage from 5.1 to 5.2 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.1 to 5.2. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.1...coverage-5.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ac8999097a..b1780fc2b0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,7 +13,7 @@ backcall==0.2.0 # via ipython black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.0 # via cryptography click==7.1.2 # via black -coverage==5.1 # via pytest-cov +coverage==5.2 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.3 # via -r test-requirements.in From c3653b7ae581097eca114c6f3d12c6267d5d1e17 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 7 Jul 2020 21:04:50 +0000 Subject: [PATCH 0367/1498] Noop commit to poke codecov From 477672897aa4d5e72b9df678ea92153537d33d5d Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 7 Jul 2020 21:39:10 +0000 Subject: [PATCH 0368/1498] Fix flake8 --- trio/_core/tests/tutil.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index d6d768d0d4..4367338661 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -9,7 +9,6 @@ from contextlib import contextmanager import gc -import sys # See trio/tests/conftest.py for the other half of this from trio.tests.conftest import RUN_SLOW From 828726be4cc45bbb09708fe693ed4322fc5768fc Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 21 Jul 2020 14:37:34 -0700 Subject: [PATCH 0369/1498] CI: Fetch Komodia-based LSP from a source we control --- .github/workflows/ci.yml | 9 +++------ ci.sh | 13 +++++++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6d49fcc83..d11c153e9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: push: branches-ignore: - - "dependabot/*" + - "dependabot/*" pull_request: jobs: @@ -21,11 +21,8 @@ jobs: include: - python: '3.8' arch: 'x64' - # This is a SUPER sketchy source for this -- I assume it - # has some kind of embedded malware -- but the real - # download site has the downloads locked behind a - # registration wall. - lsp: 'http://fs3.softfamous.com/downloads/tname-110575cb0f191/software/astrill-setup-win.exe' + lsp: 'https://raw.githubusercontent.com/python-trio/trio-ci-assets/master/komodia-based-vpn-setup.zip' + lsp_extract_file: 'komodia-based-vpn-setup.exe' extra_name: ', with Komodia LSP' - python: '3.8' arch: 'x64' diff --git a/ci.sh b/ci.sh index c4e046b986..cad85d2032 100755 --- a/ci.sh +++ b/ci.sh @@ -420,10 +420,19 @@ else # installing some untrustworthy quasi-malware onto into a sandboxed # machine for testing. So MITM attacks are really the least of our # worries. - curl-harder --insecure -o lsp-installer.exe "$LSP" + if [ "$LSP_EXTRACT_FILE" != "" ]; then + # We host the Astrill VPN installer ourselves, and encrypt it + # so as to decrease the chances of becoming an inadvertent + # public redistributor. + curl-harder -o lsp-installer.zip "$LSP" + unzip -P "not very secret trio ci key" lsp-installer.zip "$LSP_EXTRACT_FILE" + mv "$LSP_EXTRACT_FILE" lsp-installer.exe + else + curl-harder --insecure -o lsp-installer.exe "$LSP" + fi # This is only needed for the Astrill LSP, but there's no harm in # doing it all the time. The cert was manually extracted by installing - # the package in a VPN, clicking "Always trust from this publisher" + # the package in a VM, clicking "Always trust from this publisher" # when installing, and then running 'certmgr.msc' and exporting the # certificate. See: # http://www.migee.com/2010/09/24/solution-for-unattendedsilent-installs-and-would-you-like-to-install-this-device-software/ From 3e85d9b8accdbf6c2218307a33acfbe27fac24ee Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 21 Jul 2020 14:41:04 -0700 Subject: [PATCH 0370/1498] Propagate an environment variable --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d11c153e9a..e340f1ee37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: shell: bash env: LSP: '${{ matrix.lsp }}' + LSP_EXTRACT_FILE: '${{ matrix.lsp_extract_file }}' # Should match 'name:' up above JOB_NAME: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' From 8fcf4ef5fb9a9e2416aa483fddfbc666e288c3e7 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 21 Jul 2020 14:54:43 -0700 Subject: [PATCH 0371/1498] Try harder --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e340f1ee37..57cf5a498f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ jobs: python: ['3.6', '3.7', '3.8'] arch: ['x86', 'x64'] lsp: [''] + lsp_extract_file: [''] extra_name: [''] include: - python: '3.8' @@ -27,10 +28,12 @@ jobs: - python: '3.8' arch: 'x64' lsp: 'http://www.proxifier.com/download/ProxifierSetup.exe' + lsp_extract_file: '' extra_name: ', with IFS LSP' - python: '3.8' arch: 'x64' lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' + lsp_extract_file: '' extra_name: ', with non-IFS LSP' steps: - name: Checkout From 560408761e0b21295427214cde20443749a5c06d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2020 06:42:29 +0000 Subject: [PATCH 0372/1498] Bump jedi from 0.17.1 to 0.17.2 Bumps [jedi](https://github.com/davidhalter/jedi) from 0.17.1 to 0.17.2. - [Release notes](https://github.com/davidhalter/jedi/releases) - [Changelog](https://github.com/davidhalter/jedi/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/jedi/compare/v0.17.1...v0.17.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b1780fc2b0..fe75c49ea7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets ipython==7.16.1 # via -r test-requirements.in isort==4.3.21 # via pylint -jedi==0.17.1 # via -r test-requirements.in, ipython +jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint more-itertools==8.4.0 # via pytest From b388e34ac06e8fec30405dad09763fe0db21c3da Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2020 06:43:37 +0000 Subject: [PATCH 0373/1498] Bump regex from 2020.6.8 to 2020.7.14 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.6.8 to 2020.7.14. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b1780fc2b0..c044f5dca2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.0 # via -r test-requirements.in pytest==5.4.3 # via -r test-requirements.in, pytest-cov -regex==2020.6.8 # via black +regex==2020.7.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in From 3432c7716178db91a1fea7f7b01ddc4df2051b5c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2020 06:25:48 +0000 Subject: [PATCH 0374/1498] Bump urllib3 from 1.25.9 to 1.25.10 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.25.9 to 1.25.10. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.25.10/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.25.9...1.25.10) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c5f72bd193..55ba344b9a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -39,4 +39,4 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in toml==0.10.1 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in -urllib3==1.25.9 # via requests +urllib3==1.25.10 # via requests From b746b6f530afd54eb9acce839131fe1554dd8076 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2020 06:38:51 +0000 Subject: [PATCH 0375/1498] Bump parso from 0.7.0 to 0.7.1 Bumps [parso](https://github.com/davidhalter/parso) from 0.7.0 to 0.7.1. - [Release notes](https://github.com/davidhalter/parso/releases) - [Changelog](https://github.com/davidhalter/parso/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/parso/compare/v0.7.0...v0.7.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 26b23cd139..15ae34a9da 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -30,7 +30,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-require mypy==0.782 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest -parso==0.7.0 # via jedi +parso==0.7.1 # via jedi pathspec==0.8.0 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython From 9868f25f746702f885eaf5388f05ddcd6a9ce115 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 24 Jul 2020 07:46:43 +0000 Subject: [PATCH 0376/1498] Respond to review comments; split asyncgens logic into a separate file --- docs/source/reference-core.rst | 10 +- trio/_core/_asyncgens.py | 191 ++++++++++++++++++++++++++++++ trio/_core/_run.py | 187 ++--------------------------- trio/_core/tests/test_asyncgen.py | 4 +- 4 files changed, 207 insertions(+), 185 deletions(-) create mode 100644 trio/_core/_asyncgens.py diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index e795088fbe..664a8d96c2 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1505,9 +1505,9 @@ by ``break``\ing out of the iteration, things aren't so simple. The async generator iterator object is still alive, waiting for you to resume iterating it so it can produce more values. At some point, Python will realize that you've dropped all references to the -iterator, and will call on Trio to help with executing any remaining -cleanup code inside the generator: ``finally`` blocks, ``__aexit__`` -handlers, and so on. +iterator, and will call on Trio to throw in a `GeneratorExit` exception +so that any remaining cleanup code inside the generator has a chance +to run: ``finally`` blocks, ``__aexit__`` handlers, and so on. So far, so good. Unfortunately, Python provides no guarantees about *when* this happens. It could be as soon as you break out of the @@ -1516,7 +1516,9 @@ even be after the entire Trio run has finished! Just about the only guarantee is that it *won't* happen in the task that was using the generator. That task will continue on with whatever else it's doing, and the async generator cleanup will happen "sometime later, -somewhere else". +somewhere else": potentially with different context variables, +not subject to timeouts, and/or after any nurseries you're using have +been closed. If you don't like that ambiguity, and you want to ensure that a generator's ``finally`` blocks and ``__aexit__`` handlers execute as diff --git a/trio/_core/_asyncgens.py b/trio/_core/_asyncgens.py new file mode 100644 index 0000000000..87f9d76904 --- /dev/null +++ b/trio/_core/_asyncgens.py @@ -0,0 +1,191 @@ +import attr +import logging +import sys +import warnings +import weakref + +from .._util import name_asyncgen +from . import _run +from .. import _core + +# Used to log exceptions in async generator finalizers +ASYNCGEN_LOGGER = logging.getLogger("trio.async_generator_errors") + + +@attr.s(eq=False, slots=True) +class AsyncGenerators: + # Async generators are added to this set when first iterated. Any + # left after the main task exits will be closed before trio.run() + # returns. During most of the run, this is a WeakSet so GC works. + # During shutdown, when we're finalizing all the remaining + # asyncgens after the system nursery has been closed, it's a + # regular set so we don't have to deal with GC firing at + # unexpected times. + alive = attr.ib(factory=weakref.WeakSet) + + # This collects async generators that get garbage collected during + # the one-tick window between the system nursery closing and the + # init task starting end-of-run asyncgen finalization. + trailing_needs_finalize = attr.ib(factory=set) + + prev_hooks = attr.ib(init=False) + + def install_hooks(self, runner): + def firstiter(agen): + if hasattr(_run.GLOBAL_RUN_CONTEXT, "task"): + self.alive.add(agen) + else: + # An async generator first iterated outside of a Trio + # task doesn't belong to Trio. Probably we're in guest + # mode and the async generator belongs to our host. + # The locals dictionary is the only good place to + # remember this fact, at least until + # https://bugs.python.org/issue40916 is implemented. + agen.ag_frame.f_locals["@trio_foreign_asyncgen"] = True + if self.prev_hooks.firstiter is not None: + self.prev_hooks.firstiter(agen) + + def finalize_in_trio_context(agen, agen_name): + try: + runner.spawn_system_task( + self._finalize_one, + agen, + agen_name, + name=f"close asyncgen {agen_name} (abandoned)", + ) + except RuntimeError: + # There is a one-tick window where the system nursery + # is closed but the init task hasn't yet made + # self.asyncgens a strong set to disable GC. We seem to + # have hit it. + self.trailing_needs_finalize.add(agen) + + def finalizer(agen): + agen_name = name_asyncgen(agen) + try: + is_ours = not agen.ag_frame.f_locals.get("@trio_foreign_asyncgen") + except AttributeError: # pragma: no cover + is_ours = True + + if is_ours: + runner.entry_queue.run_sync_soon( + finalize_in_trio_context, agen, agen_name + ) + + # Do this last, because it might raise an exception + # depending on the user's warnings filter. (That + # exception will be printed to the terminal and + # ignored, since we're running in GC context.) + warnings.warn( + f"Async generator {agen_name!r} was garbage collected before it " + f"had been exhausted. Surround its use in 'async with " + f"aclosing(...):' to ensure that it gets cleaned up as soon as " + f"you're done using it.", + ResourceWarning, + stacklevel=2, + source=agen, + ) + else: + # Not ours -> forward to the host loop's async generator finalizer + if self.prev_hooks.finalizer is not None: + self.prev_hooks.finalizer(agen) + else: + # Host has no finalizer. Reimplement the default + # Python behavior with no hooks installed: throw in + # GeneratorExit, step once, raise RuntimeError if + # it doesn't exit. + closer = agen.aclose() + try: + # If the next thing is a yield, this will raise RuntimeError + # which we allow to propagate + closer.send(None) + except StopIteration: + pass + else: + # If the next thing is an await, we get here. Give a nicer + # error than the default "async generator ignored GeneratorExit" + raise RuntimeError( + f"Non-Trio async generator {agen_name!r} awaited something " + f"during finalization; install a finalization hook to " + f"support this, or wrap it in 'async with aclosing(...):'" + ) + + self.prev_hooks = sys.get_asyncgen_hooks() + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer=finalizer) + + async def finalize_remaining(self, runner): + # This is called from init after shutting down the system nursery. + # The only tasks running at this point are init and + # the run_sync_soon task, and since the system nursery is closed, + # there's no way for user code to spawn more. + assert _core.current_task() is runner.init_task + assert len(runner.tasks) == 2 + + # To make async generator finalization easier to reason + # about, we'll shut down asyncgen garbage collection by turning + # the alive WeakSet into a regular set. + self.alive = set(self.alive) + + # Process all pending run_sync_soon callbacks, in case one of + # them was an asyncgen finalizer that snuck in under the wire. + runner.entry_queue.run_sync_soon(runner.reschedule, runner.init_task) + await _core.wait_task_rescheduled(lambda _: Abort.FAILED) # pragma: no cover + self.alive.update(self.trailing_needs_finalize) + self.trailing_needs_finalize.clear() + + # None of the still-living tasks use async generators, so + # every async generator must be suspended at a yield point -- + # there's no one to be doing the iteration. That's good, + # because aclose() only works on an asyncgen that's suspended + # at a yield point. (If it's suspended at an event loop trap, + # because someone is in the middle of iterating it, then you + # get a RuntimeError on 3.8+, and a nasty surprise on earlier + # versions due to https://bugs.python.org/issue32526.) + # + # However, once we start aclose() of one async generator, it + # might start fetching the next value from another, thus + # preventing us from closing that other (at least until + # aclose() of the first one is complete). This constraint + # effectively requires us to finalize the remaining asyncgens + # in arbitrary order, rather than doing all of them at the + # same time. On 3.8+ we could defer any generator with + # ag_running=True to a later batch, but that only catches + # the case where our aclose() starts after the user's + # asend()/etc. If our aclose() starts first, then the + # user's asend()/etc will raise RuntimeError, since they're + # probably not checking ag_running. + # + # It might be possible to allow some parallelized cleanup if + # we can determine that a certain set of asyncgens have no + # interdependencies, using gc.get_referents() and such. + # But just doing one at a time will typically work well enough + # (since each aclose() executes in a cancelled scope) and + # is much easier to reason about. + + # It's possible that that cleanup code will itself create + # more async generators, so we iterate repeatedly until + # all are gone. + while self.alive: + batch = self.alive + self.alive = set() + for agen in batch: + await self._finalize_one(agen, name_asyncgen(agen)) + + def close(self): + sys.set_asyncgen_hooks(*self.prev_hooks) + + async def _finalize_one(self, agen, name): + try: + # This shield ensures that finalize_asyncgen never exits + # with an exception, not even a Cancelled. The inside + # is cancelled so there's no deadlock risk. + with _core.CancelScope(shield=True) as cancel_scope: + cancel_scope.cancel() + await agen.aclose() + except BaseException: + ASYNCGEN_LOGGER.exception( + "Exception ignored during finalization of async generator %r -- " + "surround your use of the generator in 'async with aclosing(...):' " + "to raise exceptions like this in the context where they're generated", + name, + ) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 0c995788b2..d127dd54e6 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -43,10 +43,11 @@ PermanentlyDetachCoroutineObject, WaitTaskRescheduled, ) +from ._asyncgens import AsyncGenerators from ._thread_cache import start_thread_soon from .. import _core from .._deprecate import warn_deprecated -from .._util import Final, NoPublicConstructor, coroutine_or_error, name_asyncgen +from .._util import Final, NoPublicConstructor, coroutine_or_error DEADLINE_HEAP_MIN_PRUNE_THRESHOLD = 1000 @@ -67,9 +68,8 @@ def _public(fn): _ALLOW_DETERMINISTIC_SCHEDULING = False _r = random.Random() -# Used to log exceptions in instruments and async generator finalizers +# Used to log exceptions in instruments INSTRUMENT_LOGGER = logging.getLogger("trio.abc.Instrument") -ASYNCGEN_LOGGER = logging.getLogger("trio.async_generator_errors") # On 3.7+, Context.run() is implemented in C and doesn't show up in @@ -1288,6 +1288,7 @@ class Runner: entry_queue = attr.ib(factory=EntryQueue) trio_token = attr.ib(default=None) + asyncgens = attr.ib(factory=AsyncGenerators) # If everything goes idle for this long, we call clock._autojump() clock_autojump_threshold = attr.ib(default=inf) @@ -1296,24 +1297,8 @@ class Runner: is_guest = attr.ib(default=False) guest_tick_scheduled = attr.ib(default=False) - # Async generators are added to this set when first iterated. Any - # left after the main task exits will be closed before trio.run() - # returns. During most of the run, this is a WeakSet so GC works. - # During shutdown, when we're finalizign all the remaining - # asyncgens after the system nursery has been closed, it's a - # regular set so we don't have to deal with GC firing at - # unexpected times. - asyncgens = attr.ib(factory=weakref.WeakSet) - - # This collects async generators that get garbage collected during - # the one-tick window between the system nursery closing and the - # init task starting end-of-run asyncgen finalization. - trailing_finalizer_asyncgens = attr.ib(factory=set) - - prev_asyncgen_hooks = attr.ib(init=False) - def __attrs_post_init__(self): - self.setup_asyncgen_hooks() + self.asyncgens.install_hooks(self) def force_guest_tick_asap(self): if self.guest_tick_scheduled: @@ -1324,7 +1309,7 @@ def force_guest_tick_asap(self): def close(self): self.io_manager.close() self.entry_queue.close() - sys.set_asyncgen_hooks(*self.prev_asyncgen_hooks) + self.asyncgens.close() if self.instruments: self.instrument("after_run") # This is where KI protection gets disabled, so we do it last @@ -1540,147 +1525,6 @@ def task_exited(self, task, outcome): # Async generator finalization support ################ - async def finalize_asyncgen(self, agen, name): - try: - # This shield ensures that finalize_asyncgen never exits - # with an exception, not even a Cancelled. The inside - # is cancelled so there's no deadlock risk. - with CancelScope(shield=True) as cancel_scope: - cancel_scope.cancel() - await agen.aclose() - except BaseException: - ASYNCGEN_LOGGER.exception( - "Exception ignored during finalization of async generator %r -- " - "surround your use of the generator in 'async with aclosing(...):' " - "to raise exceptions like this in the context where they're generated", - name, - ) - - async def finalize_remaining_asyncgens(self): - # At the time this function is called, there are exactly two - # tasks running: init and the run_sync_soon task. (And we've - # shut down the system nursery, so no more can appear.) - # Neither one uses async generators, so every async generator - # must be suspended at a yield point -- there's no one to be - # doing the iteration. That's good, because aclose() only - # works on an asyncgen that's suspended at a yield point. - # (If it's suspended at an event loop trap, because someone - # is in the middle of iterating it, then you get a RuntimeError - # on 3.8+, and a nasty surprise on earlier versions due to - # https://bugs.python.org/issue32526.) - # - # However, once we start aclose() of one async generator, it - # might start fetching the next value from another, thus - # preventing us from closing that other (at least until - # aclose() of the first one is complete). This constraint - # effectively requires us to finalize the remaining asyncgens - # in arbitrary order, rather than doing all of them at the - # same time. On 3.8+ we could defer any generator with - # ag_running=True to a later batch, but that only catches - # the case where our aclose() starts after the user's - # asend()/etc. If our aclose() starts first, then the - # user's asend()/etc will raise RuntimeError, since they're - # probably not checking ag_running. - # - # It might be possible to allow some parallelized cleanup if - # we can determine that a certain set of asyncgens have no - # interdependencies, using gc.get_referents() and such. - # But just doing one at a time will typically work well enough - # (since each aclose() executes in a cancelled scope) and - # is much easier to reason about. - - # It's possible that that cleanup code will itself create - # more async generators, so we iterate repeatedly until - # all are gone. - while self.asyncgens: - batch = self.asyncgens - self.asyncgens = set() - for agen in batch: - await self.finalize_asyncgen(agen, name_asyncgen(agen)) - - def setup_asyncgen_hooks(self): - def firstiter(agen): - if hasattr(GLOBAL_RUN_CONTEXT, "task"): - self.asyncgens.add(agen) - else: - # An async generator first iterated outside of a Trio - # task doesn't belong to Trio. Probably we're in guest - # mode and the async generator belongs to our host. - # The locals dictionary is the only good place to - # remember this fact, at least until - # https://bugs.python.org/issue40916 is implemented. - agen.ag_frame.f_locals["@trio_foreign_asyncgen"] = True - if self.prev_asyncgen_hooks.firstiter is not None: - self.prev_asyncgen_hooks.firstiter(agen) - - def finalize_in_trio_context(agen, agen_name): - try: - self.spawn_system_task( - self.finalize_asyncgen, - agen, - agen_name, - name=f"close asyncgen {agen_name} (abandoned)", - ) - except RuntimeError: - # There is a one-tick window where the system nursery - # is closed but the init task hasn't yet made - # self.asyncgens a strong set to disable GC. We seem to - # have hit it. - self.trailing_finalizer_asyncgens.add(agen) - - def finalizer(agen): - agen_name = name_asyncgen(agen) - try: - is_ours = not agen.ag_frame.f_locals.get("@trio_foreign_asyncgen") - except AttributeError: # pragma: no cover - is_ours = True - - if is_ours: - self.entry_queue.run_sync_soon( - finalize_in_trio_context, agen, agen_name - ) - - # Do this last, because it might raise an exception - # depending on the user's warnings filter. (That - # exception will be printed to the terminal and - # ignored, since we're running in GC context.) - warnings.warn( - f"Async generator {agen_name!r} was garbage collected before it " - f"had been exhausted. Surround its use in 'async with " - f"aclosing(...):' to ensure that it gets cleaned up as soon as " - f"you're done using it.", - ResourceWarning, - stacklevel=2, - source=agen, - ) - else: - # Not ours -> forward to the host loop's async generator finalizer - if self.prev_asyncgen_hooks.finalizer is not None: - self.prev_asyncgen_hooks.finalizer(agen) - else: - # Host has no finalizer. Reimplement the default - # Python behavior with no hooks installed: throw in - # GeneratorExit, step once, raise RuntimeError if - # it doesn't exit. - closer = agen.aclose() - try: - # If the next thing is a yield, this will raise RuntimeError - # which we allow to propagate - closer.send(None) - except StopIteration: - pass - else: - # If the next thing is an await, we get here. Give a nicer - # error than the default "async generator ignored GeneratorExit" - raise RuntimeError( - f"async generator {agen_name!r} awaited during " - f"finalization; install a finalization hook to support " - f"this, or wrap it in 'async with aclosing(...):'" - ) - - self.prev_asyncgen_hooks = sys.get_asyncgen_hooks() - sys.set_asyncgen_hooks(firstiter=firstiter, finalizer=finalizer) - ################ # System tasks and init ################ @@ -1754,23 +1598,8 @@ async def init(self, async_fn, args): # Main task is done; start shutting down system tasks self.system_nursery.cancel_scope.cancel() - # The only tasks running at this point are init (this one) - # and the run_sync_soon task. We still need to finalize - # remaining async generators. To make that easier to reason - # about, we'll shut down their garbage collection by turning - # the asyncgens WeakSet into a regular set. - self.asyncgens = set(self.asyncgens) - - # Process all pending run_sync_soon callbacks, in case one of - # them was an asyncgen finalizer that snuck in under the wire. - self.entry_queue.run_sync_soon(self.reschedule, self.init_task) - await wait_task_rescheduled(lambda _: Abort.FAILED) # pragma: no cover - self.asyncgens.update(self.trailing_finalizer_asyncgens) - self.trailing_finalizer_asyncgens.clear() - - # The only async-colored user code left to run is the - # finalizers for the async generators that remain alive. - await self.finalize_remaining_asyncgens() + # System nursery is closed; finalize remaining async generators + await self.asyncgens.finalize_remaining(self) # There are no more asyncgens, which means no more user-provided # code except possibly run_sync_soon callbacks. It's finally safe diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index 829eb23db9..dd15da9bfd 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -184,7 +184,7 @@ async def agen(): def collect_at_opportune_moment(token): runner = _core._run.GLOBAL_RUN_CONTEXT.runner if runner.system_nursery._closed and isinstance( - runner.asyncgens, weakref.WeakSet + runner.asyncgens.alive, weakref.WeakSet ): saved.clear() record.append("final collection") @@ -277,7 +277,7 @@ async def awaits_after_yield(): await step_outside_async_context(awaits_after_yield()) gc_collect_harder() - assert "awaited during finalization" in capsys.readouterr().err + assert "awaited something during finalization" in capsys.readouterr().err @pytest.mark.skipif(buggy_pypy_asyncgens, reason="pypy 7.2.0 is buggy") From 2af69745fcf015098fa14069719b336c9318b11a Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 24 Jul 2020 07:49:39 +0000 Subject: [PATCH 0377/1498] Fix mypy --- trio/_core/_asyncgens.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trio/_core/_asyncgens.py b/trio/_core/_asyncgens.py index 87f9d76904..cb88a1d57b 100644 --- a/trio/_core/_asyncgens.py +++ b/trio/_core/_asyncgens.py @@ -129,7 +129,9 @@ async def finalize_remaining(self, runner): # Process all pending run_sync_soon callbacks, in case one of # them was an asyncgen finalizer that snuck in under the wire. runner.entry_queue.run_sync_soon(runner.reschedule, runner.init_task) - await _core.wait_task_rescheduled(lambda _: Abort.FAILED) # pragma: no cover + await _core.wait_task_rescheduled( + lambda _: _core.Abort.FAILED # pragma: no cover + ) self.alive.update(self.trailing_needs_finalize) self.trailing_needs_finalize.clear() From 46c105a2ef0e4cdea8f2646b64912f2c8af254f4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2020 07:01:09 +0000 Subject: [PATCH 0378/1498] Bump cffi from 1.14.0 to 1.14.1 Bumps [cffi](https://bitbucket.org/cffi/release-doc) from 1.14.0 to 1.14.1. - [Commits](https://bitbucket.org/cffi/release-doc/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 15ae34a9da..a6e1f68419 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ async-generator==1.10 # via -r test-requirements.in attrs==19.3.0 # via -r test-requirements.in, black, outcome, pytest backcall==0.2.0 # via ipython black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.0 # via cryptography +cffi==1.14.1 # via cryptography click==7.1.2 # via black coverage==5.2 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme From c78d7b15dbbcb51d94788886355e130bfc8fc3f0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2020 11:12:10 +0000 Subject: [PATCH 0379/1498] Bump coverage from 5.2 to 5.2.1 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.2 to 5.2.1. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.2...coverage-5.2.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a6e1f68419..70577c5b86 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,7 +13,7 @@ backcall==0.2.0 # via ipython black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.1 # via cryptography click==7.1.2 # via black -coverage==5.2 # via pytest-cov +coverage==5.2.1 # via pytest-cov cryptography==2.9.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.3 # via -r test-requirements.in From 4b3bbb83a96568ea2b16490e5ed2fc12e07e9d5a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2020 06:44:47 +0000 Subject: [PATCH 0380/1498] Bump pytest from 5.4.3 to 6.0.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.3 to 6.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.3...6.0.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 70577c5b86..6d1e1e136e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,6 +19,7 @@ decorator==4.4.2 # via ipython, traitlets flake8==3.8.3 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in +iniconfig==1.0.0 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.16.1 # via -r test-requirements.in isort==4.3.21 # via pylint @@ -46,15 +47,15 @@ pylint==2.5.3 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.0 # via -r test-requirements.in -pytest==5.4.3 # via -r test-requirements.in, pytest-cov +pytest==6.0.0 # via -r test-requirements.in, pytest-cov regex==2020.7.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in -toml==0.10.1 # via black, pylint +toml==0.10.1 # via black, pylint, pytest traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy typing-extensions==3.7.4.2 ; implementation_name == "cpython" # via -r test-requirements.in, mypy -wcwidth==0.2.5 # via prompt-toolkit, pytest +wcwidth==0.2.5 # via prompt-toolkit wrapt==1.12.1 # via astroid From cd1c4f4b260a2f6210838414e6c0ebd43602a285 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2020 07:01:20 +0000 Subject: [PATCH 0381/1498] Bump cryptography from 2.9.2 to 3.0 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.9.2 to 3.0. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.9.2...3.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6d1e1e136e..3c53649052 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,7 +14,7 @@ black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.1 # via cryptography click==7.1.2 # via black coverage==5.2.1 # via pytest-cov -cryptography==2.9.2 # via pyopenssl, trustme +cryptography==3.0 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.3 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme From f515e144876db689e5b7c350cc3300be8e64c724 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2020 16:35:59 +0000 Subject: [PATCH 0382/1498] Bump iniconfig from 1.0.0 to 1.0.1 Bumps [iniconfig](https://github.com/RonnyPfannschmidt/iniconfig) from 1.0.0 to 1.0.1. - [Release notes](https://github.com/RonnyPfannschmidt/iniconfig/releases) - [Changelog](https://github.com/RonnyPfannschmidt/iniconfig/blob/master/CHANGELOG) - [Commits](https://github.com/RonnyPfannschmidt/iniconfig/compare/v1.0.0...v1.0.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3c53649052..92384fabcc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ decorator==4.4.2 # via ipython, traitlets flake8==3.8.3 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in -iniconfig==1.0.0 # via pytest +iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.16.1 # via -r test-requirements.in isort==4.3.21 # via pylint From 6abe2d2a92c18ff2b07d9d9826cceb8088e3ce75 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2020 16:47:26 +0000 Subject: [PATCH 0383/1498] Bump pytest from 6.0.0 to 6.0.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.0.0 to 6.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.0.0...6.0.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 92384fabcc..5d6cee6fda 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pylint==2.5.3 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.0 # via -r test-requirements.in -pytest==6.0.0 # via -r test-requirements.in, pytest-cov +pytest==6.0.1 # via -r test-requirements.in, pytest-cov regex==2020.7.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets sniffio==1.1.0 # via -r test-requirements.in From 04419520bedc430d13f8421343c5d32d64455190 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 5 Aug 2020 02:09:27 +0000 Subject: [PATCH 0384/1498] Review responses --- trio/_core/_run.py | 8 +------- trio/_core/tests/test_instrumentation.py | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index f004f4cf39..55cc4a159d 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1301,9 +1301,6 @@ class Runner: is_guest = attr.ib(default=False) guest_tick_scheduled = attr.ib(default=False) - def __attrs_post_init__(self): - self.asyncgens.install_hooks(self) - def force_guest_tick_asap(self): if self.guest_tick_scheduled: return @@ -1525,10 +1522,6 @@ def task_exited(self, task, outcome): if "task_exited" in self.instruments: self.instruments.call("task_exited", task) - ################ - # Async generator finalization support - ################ - ################ # System tasks and init ################ @@ -1833,6 +1826,7 @@ def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints) system_context=system_context, ki_manager=ki_manager, ) + runner.asyncgens.install_hooks(runner) # This is where KI protection gets enabled, so we want to do it early - in # particular before we start modifying global state like GLOBAL_RUN_CONTEXT diff --git a/trio/_core/tests/test_instrumentation.py b/trio/_core/tests/test_instrumentation.py index 946caf7965..57d3461d3b 100644 --- a/trio/_core/tests/test_instrumentation.py +++ b/trio/_core/tests/test_instrumentation.py @@ -73,7 +73,6 @@ async def main(): + [("before", task), ("schedule", task), ("after", task)] * 5 + [("before", task), ("after", task), ("after_run",)] ) - assert len(r1.record) > len(r2.record) > len(r3.record) assert r1.record == r2.record + r3.record assert list(r1.filter_tasks([task])) == expected From 58912a88a42348c38bc7d2e771c1dd35a1ff869b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2020 07:22:10 +0000 Subject: [PATCH 0385/1498] Bump sphinx from 3.1.1 to 3.2.0 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.1.1 to 3.2.0. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.1.1...v3.2.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 55ba344b9a..60616d1869 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -29,7 +29,7 @@ sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.2.2 # via -r docs-requirements.in sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in -sphinx==3.1.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx==3.2.0 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From c9b86ef5111b8324767508c33317edf551649e4e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2020 10:28:09 +0000 Subject: [PATCH 0386/1498] Bump prompt-toolkit from 3.0.5 to 3.0.6 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.5 to 3.0.6. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.5...3.0.6) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5d6cee6fda..f1b364c3e2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -36,7 +36,7 @@ pathspec==0.8.0 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.5 # via ipython +prompt-toolkit==3.0.6 # via ipython ptyprocess==0.6.0 # via pexpect py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 From aa97006ead18341ef5ed5a21f478cf5e4193f757 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 08:26:52 +0000 Subject: [PATCH 0387/1498] Bump attrs from 19.3.0 to 20.1.0 Bumps [attrs](https://github.com/python-attrs/attrs) from 19.3.0 to 20.1.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/master/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/19.3.0...20.1.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 60616d1869..1acccd3af0 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -6,7 +6,7 @@ # alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in -attrs==19.3.0 # via -r docs-requirements.in, outcome +attrs==20.1.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx certifi==2020.6.20 # via requests chardet==3.0.4 # via requests diff --git a/test-requirements.txt b/test-requirements.txt index f1b364c3e2..47136f71d3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in astroid==2.4.2 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==19.3.0 # via -r test-requirements.in, black, outcome, pytest +attrs==20.1.0 # via -r test-requirements.in, black, outcome, pytest backcall==0.2.0 # via ipython black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.1 # via cryptography From 16e01f9b629df9d1775192385c8eedb0bab2ef94 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 08:37:10 +0000 Subject: [PATCH 0388/1498] Bump sphinx from 3.2.0 to 3.2.1 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.2.0 to 3.2.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.2.0...v3.2.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 1acccd3af0..84cd29ad72 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -29,7 +29,7 @@ sniffio==1.1.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.2.2 # via -r docs-requirements.in sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in -sphinx==3.2.0 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx==3.2.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 35e89fbcd646aa4fba702f59da545381b3a919a1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 08:37:18 +0000 Subject: [PATCH 0389/1498] Bump cffi from 1.14.1 to 1.14.2 Bumps [cffi](https://github.com/python-cffi/release-doc) from 1.14.1 to 1.14.2. - [Release notes](https://github.com/python-cffi/release-doc/releases) - [Commits](https://github.com/python-cffi/release-doc/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 47136f71d3..fad77e194b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ async-generator==1.10 # via -r test-requirements.in attrs==20.1.0 # via -r test-requirements.in, black, outcome, pytest backcall==0.2.0 # via ipython black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.1 # via cryptography +cffi==1.14.2 # via cryptography click==7.1.2 # via black coverage==5.2.1 # via pytest-cov cryptography==3.0 # via pyopenssl, trustme From 03ed3b4b53bbd347fd90477554effe19708d2c74 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 09:06:09 +0000 Subject: [PATCH 0390/1498] Bump pytest-cov from 2.10.0 to 2.10.1 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.10.0 to 2.10.1. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.10.0...v2.10.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fad77e194b..4d97cddb79 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -46,7 +46,7 @@ pygments==2.6.1 # via ipython pylint==2.5.3 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.10.0 # via -r test-requirements.in +pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.0.1 # via -r test-requirements.in, pytest-cov regex==2020.7.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets From ad2b5f4f1bf57d92eb3952bc8ae17b389e989438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Heissler?= Date: Sun, 23 Aug 2020 09:41:44 +0200 Subject: [PATCH 0391/1498] Fix typo \* in docs for Nursery.start --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 55cc4a159d..cec4357289 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -998,7 +998,7 @@ async def start(self, async_fn, *args, name=None): The conventional way to define ``async_fn`` is like:: - async def async_fn(arg1, arg2, \*, task_status=trio.TASK_STATUS_IGNORED): + async def async_fn(arg1, arg2, *, task_status=trio.TASK_STATUS_IGNORED): ... task_status.started() ... From 977fee10c92826e216a31ca492d693edf09f5ceb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 09:00:44 +0000 Subject: [PATCH 0392/1498] Bump typing-extensions from 3.7.4.2 to 3.7.4.3 Bumps [typing-extensions](https://github.com/python/typing) from 3.7.4.2 to 3.7.4.3. - [Release notes](https://github.com/python/typing/releases) - [Commits](https://github.com/python/typing/compare/3.7.4.2...3.7.4.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4d97cddb79..185a77c058 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -56,6 +56,6 @@ toml==0.10.1 # via black, pylint, pytest traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy -typing-extensions==3.7.4.2 ; implementation_name == "cpython" # via -r test-requirements.in, mypy +typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, mypy wcwidth==0.2.5 # via prompt-toolkit wrapt==1.12.1 # via astroid From 239fed595eb6eb6bb70dee795e639fdafb4a1a57 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 09:14:33 +0000 Subject: [PATCH 0393/1498] Bump pylint from 2.5.3 to 2.6.0 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.5.3 to 2.6.0. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.5.3...pylint-2.6.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 185a77c058..4a63434336 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,7 +43,7 @@ pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 pygments==2.6.1 # via ipython -pylint==2.5.3 # via -r test-requirements.in +pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in From a349c333ef0c00aa236a26a49a766ddf8dee3a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Heissler?= Date: Mon, 24 Aug 2020 12:35:24 +0200 Subject: [PATCH 0394/1498] Remove "Returns" section from Nursery.start_soon docs. Closes: #1691 --- trio/_core/_run.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index cec4357289..901d57b49a 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -975,9 +975,6 @@ def start_soon(self, async_fn, *args, name=None): original function as the ``name=`` to make debugging easier. - Returns: - True if successful, False otherwise. - Raises: RuntimeError: If this nursery is no longer open (i.e. its ``async with`` block has From 12eaf9910d1c4052e408086eda7d3a9c262b6d55 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2020 23:57:29 +0000 Subject: [PATCH 0395/1498] Bump isort from 4.3.21 to 5.4.2 Bumps [isort](https://github.com/timothycrosley/isort) from 4.3.21 to 5.4.2. - [Release notes](https://github.com/timothycrosley/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/timothycrosley/isort/compare/4.3.21...5.4.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4a63434336..e84eb075ce 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.16.1 # via -r test-requirements.in -isort==4.3.21 # via pylint +isort==5.4.2 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From da137bb524d316c991b80d74ce061f437a015f15 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2020 04:36:19 +0000 Subject: [PATCH 0396/1498] Bump cryptography from 3.0 to 3.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.0 to 3.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.0...3.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e84eb075ce..1dc9991a70 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,7 +14,7 @@ black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.2 # via cryptography click==7.1.2 # via black coverage==5.2.1 # via pytest-cov -cryptography==3.0 # via pyopenssl, trustme +cryptography==3.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython, traitlets flake8==3.8.3 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme From cb1b5f3ef345a88a94fcdc1179e03a7f3b53ad31 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 27 Aug 2020 09:40:07 +0400 Subject: [PATCH 0397/1498] Ignore dependabot pushes correctly dependabot opens a branch in the main repository and then opens a pull request, which triggers the same build twice. To avoid that, we had branches-ignore set to ignore the push. However the syntax was incorrect: since the branch name contains slashes, we have to use two stars to ignore it. See [0] for details on that syntax. [0]: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57cf5a498f..8b73f5d20b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: push: branches-ignore: - - "dependabot/*" + - "dependabot/**" pull_request: jobs: From 3c3cf31a2bce8b4dce94389d7d5714da2ae57fb1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 29 Aug 2020 02:15:46 +0000 Subject: [PATCH 0398/1498] Bump more-itertools from 8.4.0 to 8.5.0 Bumps [more-itertools](https://github.com/more-itertools/more-itertools) from 8.4.0 to 8.5.0. - [Release notes](https://github.com/more-itertools/more-itertools/releases) - [Commits](https://github.com/more-itertools/more-itertools/compare/v8.4.0...8.5.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1dc9991a70..96f8988490 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -26,7 +26,7 @@ isort==5.4.2 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint -more-itertools==8.4.0 # via pytest +more-itertools==8.5.0 # via pytest mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, mypy mypy==0.782 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in From f60fb6da83d9bec68a7850e08afe9f91fe7be17a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 31 Aug 2020 13:34:51 +0400 Subject: [PATCH 0399/1498] Fix setuptools 50 import on Python 3.6.1 pylint test --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 01582944db..876c6ece5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,8 @@ jobs: - "LINUX_VM_IMAGE=https://download.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-1.6.x86_64.qcow2" - python: 3.6.1 # earliest 3.6 version available on Travis + # https://github.com/pypa/setuptools/issues/2350 + env: SETUPTOOLS_USE_DISTUTILS=stdlib dist: bionic - python: 3.6-dev - python: 3.7-dev From c9234810c9cd5bb67360431fe68536f82ae76ebf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2020 12:26:44 +0000 Subject: [PATCH 0400/1498] Bump prompt-toolkit from 3.0.6 to 3.0.7 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.6 to 3.0.7. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.6...3.0.7) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 96f8988490..090ab7488a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -36,7 +36,7 @@ pathspec==0.8.0 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.6 # via ipython +prompt-toolkit==3.0.7 # via ipython ptyprocess==0.6.0 # via pexpect py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 From 6ba5ead12f804d827ff2b33c3f815ac4fab0432f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2020 14:04:20 +0000 Subject: [PATCH 0401/1498] Bump black from 19.10b0 to 20.8b1 Bumps [black](https://github.com/psf/black) from 19.10b0 to 20.8b1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/master/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 090ab7488a..4911eb3b91 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,9 +8,9 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in astroid==2.4.2 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==20.1.0 # via -r test-requirements.in, black, outcome, pytest +attrs==20.1.0 # via -r test-requirements.in, outcome, pytest backcall==0.2.0 # via ipython -black==19.10b0 ; implementation_name == "cpython" # via -r test-requirements.in +black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.2 # via cryptography click==7.1.2 # via black coverage==5.2.1 # via pytest-cov @@ -27,7 +27,7 @@ jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint more-itertools==8.5.0 # via pytest -mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, mypy +mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy mypy==0.782 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest @@ -56,6 +56,6 @@ toml==0.10.1 # via black, pylint, pytest traitlets==4.3.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy -typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, mypy +typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy wcwidth==0.2.5 # via prompt-toolkit wrapt==1.12.1 # via astroid From a540ff3f258ab3c500e546eaab6ee41c84a7160f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 1 Sep 2020 11:50:53 +0400 Subject: [PATCH 0402/1498] Remove useless trailing commas black 20.8b1 reformats trailing commas to include only a single element per line: removing the trailing comma allows keeping the previous behavior. --- trio/__init__.py | 2 +- trio/_core/_io_kqueue.py | 4 ++-- trio/_core/_io_windows.py | 4 ++-- trio/_core/_run.py | 4 ++-- trio/_core/_unbounded_queue.py | 2 +- trio/_core/tests/test_asyncgen.py | 2 +- trio/_core/tests/test_multierror.py | 2 +- trio/_core/tests/test_parking_lot.py | 8 ++++---- trio/_core/tests/test_run.py | 8 ++++---- trio/_core/tests/test_windows.py | 4 ++-- trio/_core/tests/tutil.py | 2 +- trio/_file_io.py | 2 +- trio/_highlevel_open_tcp_listeners.py | 4 ++-- trio/_highlevel_open_tcp_stream.py | 2 +- trio/_highlevel_open_unix_stream.py | 2 +- trio/_highlevel_serve_listeners.py | 2 +- trio/_highlevel_ssl_helpers.py | 11 ++++------- trio/_subprocess_platform/kqueue.py | 2 +- trio/_subprocess_platform/waitid.py | 2 +- trio/_sync.py | 4 ++-- trio/_tools/gen_exports.py | 2 +- trio/tests/module_with_deprecations.py | 4 ++-- trio/tests/test_highlevel_open_tcp_listeners.py | 6 +++--- trio/tests/test_highlevel_ssl_helpers.py | 6 +++--- trio/tests/test_socket.py | 4 ++-- trio/tests/test_ssl.py | 4 ++-- trio/tests/test_subprocess.py | 2 +- trio/tests/test_testing.py | 2 +- trio/tests/test_threads.py | 2 +- trio/tests/test_wait_for_object.py | 4 ++-- 30 files changed, 53 insertions(+), 56 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index 63e74e9da8..a7340e0819 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -111,7 +111,7 @@ # # https://github.com/python-trio/trio/pull/1484#issuecomment-622574499 "hazmat": _deprecate.DeprecatedAttribute( - lowlevel, "0.15.0", issue=476, instead="trio.lowlevel", + lowlevel, "0.15.0", issue=476, instead="trio.lowlevel" ), } diff --git a/trio/_core/_io_kqueue.py b/trio/_core/_io_kqueue.py index 7a723c5f98..31940d5694 100644 --- a/trio/_core/_io_kqueue.py +++ b/trio/_core/_io_kqueue.py @@ -44,7 +44,7 @@ def statistics(self): tasks_waiting += 1 else: monitors += 1 - return _KqueueStatistics(tasks_waiting=tasks_waiting, monitors=monitors,) + return _KqueueStatistics(tasks_waiting=tasks_waiting, monitors=monitors) def close(self): self._kqueue.close() @@ -153,7 +153,7 @@ def abort(_): # the fact... oh well, you can't have everything.) # # FreeBSD reports this using EBADF. macOS uses ENOENT. - if exc.errno in (errno.EBADF, errno.ENOENT,): # pragma: no branch + if exc.errno in (errno.EBADF, errno.ENOENT): # pragma: no branch pass else: # pragma: no cover # As far as we know, this branch can't happen. diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 618d0283b6..e7f2d08a4b 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -477,7 +477,7 @@ def get_events(self, timeout): try: _check( kernel32.GetQueuedCompletionStatusEx( - self._iocp, self._events, MAX_EVENTS, received, milliseconds, 0, + self._iocp, self._events, MAX_EVENTS, received, milliseconds, 0 ) ) except OSError as exc: @@ -557,7 +557,7 @@ def process_events(self, received): overlapped = int(ffi.cast("uintptr_t", entry.lpOverlapped)) transferred = entry.dwNumberOfBytesTransferred info = CompletionKeyEventInfo( - lpOverlapped=overlapped, dwNumberOfBytesTransferred=transferred, + lpOverlapped=overlapped, dwNumberOfBytesTransferred=transferred ) queue.put_nowait(info) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 901d57b49a..fbeb645d8f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1460,7 +1460,7 @@ async def python_wrapper(orig_coro): # Set up the Task object ###### task = Task._create( - coro=coro, parent_nursery=nursery, runner=self, name=name, context=context, + coro=coro, parent_nursery=nursery, runner=self, name=name, context=context ) self.tasks.add(task) @@ -2035,7 +2035,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): runner.instruments.call("before_run") runner.clock.start_clock() runner.init_task = runner.spawn_impl( - runner.init, (async_fn, args), None, "", system_task=True, + runner.init, (async_fn, args), None, "", system_task=True ) # You know how people talk about "event loops"? This 'while' loop right diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index 9830df4bea..14667af1e7 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -141,7 +141,7 @@ def statistics(self): """ return _UnboundedQueueStats( - qsize=len(self._data), tasks_waiting=self._lot.statistics().tasks_waiting, + qsize=len(self._data), tasks_waiting=self._lot.statistics().tasks_waiting ) def __aiter__(self): diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index dd15da9bfd..1f886e11ab 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -38,7 +38,7 @@ async def example(cause): async def async_main(): # GC'ed before exhausted with pytest.warns( - ResourceWarning, match="Async generator.*collected before.*exhausted", + ResourceWarning, match="Async generator.*collected before.*exhausted" ): assert 42 == await example("abandoned").asend(None) gc_collect_harder() diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index e745ad7c0e..14eab22df7 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -740,6 +740,6 @@ def test_apport_excepthook_monkeypatch_interaction(): # Proper traceback assert_match_in_seq( - ["Details of embedded", "KeyError", "Details of embedded", "ValueError",], + ["Details of embedded", "KeyError", "Details of embedded", "ValueError"], stdout, ) diff --git a/trio/_core/tests/test_parking_lot.py b/trio/_core/tests/test_parking_lot.py index 1d1ecd111a..13ffe0c066 100644 --- a/trio/_core/tests/test_parking_lot.py +++ b/trio/_core/tests/test_parking_lot.py @@ -2,8 +2,8 @@ from ... import _core from ...testing import wait_all_tasks_blocked -from .tutil import check_sequence_matches from .._parking_lot import ParkingLot +from .tutil import check_sequence_matches async def test_parking_lot_basic(): @@ -32,7 +32,7 @@ async def waiter(i, lot): assert len(record) == 6 check_sequence_matches( - record, [{"sleep 0", "sleep 1", "sleep 2"}, {"wake 0", "wake 1", "wake 2"},], + record, [{"sleep 0", "sleep 1", "sleep 2"}, {"wake 0", "wake 1", "wake 2"}] ) async with _core.open_nursery() as nursery: @@ -68,7 +68,7 @@ async def waiter(i, lot): lot.unpark(count=2) await wait_all_tasks_blocked() check_sequence_matches( - record, ["sleep 0", "sleep 1", "sleep 2", {"wake 0", "wake 1"},] + record, ["sleep 0", "sleep 1", "sleep 2", {"wake 0", "wake 1"}] ) lot.unpark_all() @@ -107,7 +107,7 @@ async def test_parking_lot_cancel(): assert len(record) == 6 check_sequence_matches( - record, ["sleep 1", "sleep 2", "sleep 3", "cancelled 2", {"wake 1", "wake 3"},], + record, ["sleep 1", "sleep 2", "sleep 3", "cancelled 2", {"wake 1", "wake 3"}] ) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 1591983d47..bc1f7867a0 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -132,7 +132,7 @@ async def looper(whoami, record): nursery.start_soon(looper, "b", record) check_sequence_matches( - record, [{("a", 0), ("b", 0)}, {("a", 1), ("b", 1)}, {("a", 2), ("b", 2)}], + record, [{("a", 0), ("b", 0)}, {("a", 1), ("b", 1)}, {("a", 2), ("b", 2)}] ) @@ -1562,7 +1562,7 @@ async def async_gen(arg): # pragma: no cover yield arg with pytest.raises( - TypeError, match="expected an async function but got an async generator", + TypeError, match="expected an async function but got an async generator" ): bad_call(async_gen, 0) @@ -1694,7 +1694,7 @@ async def just_started(task_status=_core.TASK_STATUS_IGNORED): # and if after the no-op started(), the child crashes, the error comes out # of start() - async def raise_keyerror_after_started(task_status=_core.TASK_STATUS_IGNORED,): + async def raise_keyerror_after_started(task_status=_core.TASK_STATUS_IGNORED): task_status.started() raise KeyError("whoopsiedaisy") @@ -1846,7 +1846,7 @@ def __aiter__(self): async def __anext__(self): nexts = self.nexts - items = [None,] * len(nexts) + items = [None] * len(nexts) got_stop = False def handle(exc): diff --git a/trio/_core/tests/test_windows.py b/trio/_core/tests/test_windows.py index 563e157e78..e6bab82204 100644 --- a/trio/_core/tests/test_windows.py +++ b/trio/_core/tests/test_windows.py @@ -81,7 +81,7 @@ async def test_readinto_overlapped(): async def read_region(start, end): await _core.readinto_overlapped( - handle, buffer_view[start:end], start, + handle, buffer_view[start:end], start ) _core.register_with_iocp(handle) @@ -123,7 +123,7 @@ async def main(): try: async with _core.open_nursery() as nursery: nursery.start_soon( - _core.readinto_overlapped, read_handle, target, name="xyz", + _core.readinto_overlapped, read_handle, target, name="xyz" ) await wait_all_tasks_blocked() nursery.cancel_scope.cancel() diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 4367338661..00669e883e 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -13,7 +13,7 @@ # See trio/tests/conftest.py for the other half of this from trio.tests.conftest import RUN_SLOW -slow = pytest.mark.skipif(not RUN_SLOW, reason="use --run-slow to run slow tests",) +slow = pytest.mark.skipif(not RUN_SLOW, reason="use --run-slow to run slow tests") # PyPy 7.2 was released with a bug that just never called the async # generator 'firstiter' hook at all. This impacts tests of end-of-run diff --git a/trio/_file_io.py b/trio/_file_io.py index 15e45711c5..3a6e28a0d0 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -157,7 +157,7 @@ async def open_file( """ _file = wrap_file( await trio.to_thread.run_sync( - io.open, file, mode, buffering, encoding, errors, newline, closefd, opener, + io.open, file, mode, buffering, encoding, errors, newline, closefd, opener ) ) return _file diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index 8e399a61b5..80f2c7a180 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -92,7 +92,7 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): backlog = _compute_backlog(backlog) addresses = await tsocket.getaddrinfo( - host, port, type=tsocket.SOCK_STREAM, flags=tsocket.AI_PASSIVE, + host, port, type=tsocket.SOCK_STREAM, flags=tsocket.AI_PASSIVE ) listeners = [] @@ -217,5 +217,5 @@ async def serve_tcp( """ listeners = await trio.open_tcp_listeners(port, host=host, backlog=backlog) await trio.serve_listeners( - handler, listeners, handler_nursery=handler_nursery, task_status=task_status, + handler, listeners, handler_nursery=handler_nursery, task_status=task_status ) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 0ff2e7900f..545fac8641 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -165,7 +165,7 @@ def format_host_port(host, port): # AF_INET6: "..."} # this might be simpler after async def open_tcp_stream( - host, port, *, happy_eyeballs_delay=DEFAULT_DELAY, local_address=None, + host, port, *, happy_eyeballs_delay=DEFAULT_DELAY, local_address=None ): """Connect to the given host and port over TCP. diff --git a/trio/_highlevel_open_unix_stream.py b/trio/_highlevel_open_unix_stream.py index eea1e77ffc..e5aba4695f 100644 --- a/trio/_highlevel_open_unix_stream.py +++ b/trio/_highlevel_open_unix_stream.py @@ -21,7 +21,7 @@ def close_on_error(obj): raise -async def open_unix_socket(filename,): +async def open_unix_socket(filename): """Opens a connection to the specified `Unix domain socket `__. diff --git a/trio/_highlevel_serve_listeners.py b/trio/_highlevel_serve_listeners.py index 55216c85c5..0585fa516f 100644 --- a/trio/_highlevel_serve_listeners.py +++ b/trio/_highlevel_serve_listeners.py @@ -49,7 +49,7 @@ async def _serve_one_listener(listener, handler_nursery, handler): async def serve_listeners( - handler, listeners, *, handler_nursery=None, task_status=trio.TASK_STATUS_IGNORED, + handler, listeners, *, handler_nursery=None, task_status=trio.TASK_STATUS_IGNORED ): r"""Listen for incoming connections on ``listeners``, and for each one start a task running ``handler(stream)``. diff --git a/trio/_highlevel_ssl_helpers.py b/trio/_highlevel_ssl_helpers.py index 61c463c65e..fb21f239e0 100644 --- a/trio/_highlevel_ssl_helpers.py +++ b/trio/_highlevel_ssl_helpers.py @@ -49,15 +49,12 @@ async def open_ssl_over_tcp_stream( """ tcp_stream = await trio.open_tcp_stream( - host, port, happy_eyeballs_delay=happy_eyeballs_delay, + host, port, happy_eyeballs_delay=happy_eyeballs_delay ) if ssl_context is None: ssl_context = ssl.create_default_context() return trio.SSLStream( - tcp_stream, - ssl_context, - server_hostname=host, - https_compatible=https_compatible, + tcp_stream, ssl_context, server_hostname=host, https_compatible=https_compatible ) @@ -78,7 +75,7 @@ async def open_ssl_over_tcp_listeners( """ tcp_listeners = await trio.open_tcp_listeners(port, host=host, backlog=backlog) ssl_listeners = [ - trio.SSLListener(tcp_listener, ssl_context, https_compatible=https_compatible,) + trio.SSLListener(tcp_listener, ssl_context, https_compatible=https_compatible) for tcp_listener in tcp_listeners ] return ssl_listeners @@ -150,5 +147,5 @@ async def serve_ssl_over_tcp( backlog=backlog, ) await trio.serve_listeners( - handler, listeners, handler_nursery=handler_nursery, task_status=task_status, + handler, listeners, handler_nursery=handler_nursery, task_status=task_status ) diff --git a/trio/_subprocess_platform/kqueue.py b/trio/_subprocess_platform/kqueue.py index 5907f8e963..412ccf8732 100644 --- a/trio/_subprocess_platform/kqueue.py +++ b/trio/_subprocess_platform/kqueue.py @@ -17,7 +17,7 @@ async def wait_child_exiting(process: "_subprocess.Process") -> None: KQ_NOTE_EXIT = 0x80000000 make_event = lambda flags: select.kevent( - process.pid, filter=select.KQ_FILTER_PROC, flags=flags, fflags=KQ_NOTE_EXIT, + process.pid, filter=select.KQ_FILTER_PROC, flags=flags, fflags=KQ_NOTE_EXIT ) try: diff --git a/trio/_subprocess_platform/waitid.py b/trio/_subprocess_platform/waitid.py index cba5a15163..91ba224546 100644 --- a/trio/_subprocess_platform/waitid.py +++ b/trio/_subprocess_platform/waitid.py @@ -77,7 +77,7 @@ async def _waitid_system_task(pid: int, event: Event) -> None: try: await to_thread_run_sync( - sync_wait_reapable, pid, cancellable=True, limiter=waitid_limiter, + sync_wait_reapable, pid, cancellable=True, limiter=waitid_limiter ) except OSError: # If waitid fails, waitpid will fail too, so it still makes diff --git a/trio/_sync.py b/trio/_sync.py index 9c4fad3a18..8c22db9ddc 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -572,7 +572,7 @@ def statistics(self): """ return _LockStatistics( - locked=self.locked(), owner=self._owner, tasks_waiting=len(self._lot), + locked=self.locked(), owner=self._owner, tasks_waiting=len(self._lot) ) @@ -786,5 +786,5 @@ def statistics(self): """ return _ConditionStatistics( - tasks_waiting=len(self._lot), lock_statistics=self._lock.statistics(), + tasks_waiting=len(self._lot), lock_statistics=self._lock.statistics() ) diff --git a/trio/_tools/gen_exports.py b/trio/_tools/gen_exports.py index e48114af6d..6da4ee02ff 100755 --- a/trio/_tools/gen_exports.py +++ b/trio/_tools/gen_exports.py @@ -170,7 +170,7 @@ def main(): # pragma: no cover description="Generate python code for public api wrappers" ) parser.add_argument( - "--test", "-t", action="store_true", help="test if code is still up to date", + "--test", "-t", action="store_true", help="test if code is still up to date" ) parsed_args = parser.parse_args() diff --git a/trio/tests/module_with_deprecations.py b/trio/tests/module_with_deprecations.py index b0f83b1540..73184d11e8 100644 --- a/trio/tests/module_with_deprecations.py +++ b/trio/tests/module_with_deprecations.py @@ -14,8 +14,8 @@ assert not hasattr(this_mod, "dep1") __deprecated_attributes__ = { - "dep1": _deprecate.DeprecatedAttribute("value1", "1.1", issue=1,), + "dep1": _deprecate.DeprecatedAttribute("value1", "1.1", issue=1), "dep2": _deprecate.DeprecatedAttribute( - "value2", "1.2", issue=1, instead="instead-string", + "value2", "1.2", issue=1, instead="instead-string" ), } diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index ee9672caf8..d5fc576ec5 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -215,17 +215,17 @@ async def handler(stream): @pytest.mark.parametrize( "try_families", - [{tsocket.AF_INET}, {tsocket.AF_INET6}, {tsocket.AF_INET, tsocket.AF_INET6},], + [{tsocket.AF_INET}, {tsocket.AF_INET6}, {tsocket.AF_INET, tsocket.AF_INET6}], ) @pytest.mark.parametrize( "fail_families", - [{tsocket.AF_INET}, {tsocket.AF_INET6}, {tsocket.AF_INET, tsocket.AF_INET6},], + [{tsocket.AF_INET}, {tsocket.AF_INET6}, {tsocket.AF_INET, tsocket.AF_INET6}], ) async def test_open_tcp_listeners_some_address_families_unavailable( try_families, fail_families ): fsf = FakeSocketFactory( - 10, raise_on_family={family: errno.EAFNOSUPPORT for family in fail_families}, + 10, raise_on_family={family: errno.EAFNOSUPPORT for family in fail_families} ) tsocket.set_custom_socket_factory(fsf) tsocket.set_custom_hostname_resolver( diff --git a/trio/tests/test_highlevel_ssl_helpers.py b/trio/tests/test_highlevel_ssl_helpers.py index e4afc6d72a..c00f5dc464 100644 --- a/trio/tests/test_highlevel_ssl_helpers.py +++ b/trio/tests/test_highlevel_ssl_helpers.py @@ -46,7 +46,7 @@ async def getnameinfo(self, *args): # pragma: no cover async def test_open_ssl_over_tcp_stream_and_everything_else(client_ctx): # noqa: F811 async with trio.open_nursery() as nursery: (listener,) = await nursery.start( - partial(serve_ssl_over_tcp, echo_handler, 0, SERVER_CTX, host="127.0.0.1",) + partial(serve_ssl_over_tcp, echo_handler, 0, SERVER_CTX, host="127.0.0.1") ) async with listener: sockaddr = listener.transport_listener.socket.getsockname() @@ -63,7 +63,7 @@ async def test_open_ssl_over_tcp_stream_and_everything_else(client_ctx): # noqa # We have the trust but not the hostname # (checks custom ssl_context + hostname checking) stream = await open_ssl_over_tcp_stream( - "xyzzy.example.org", 80, ssl_context=client_ctx, + "xyzzy.example.org", 80, ssl_context=client_ctx ) async with stream: with pytest.raises(trio.BrokenResourceError): @@ -71,7 +71,7 @@ async def test_open_ssl_over_tcp_stream_and_everything_else(client_ctx): # noqa # This one should work! stream = await open_ssl_over_tcp_stream( - "trio-test-1.example.org", 80, ssl_context=client_ctx, + "trio-test-1.example.org", 80, ssl_context=client_ctx ) async with stream: assert isinstance(stream, trio.SSLStream) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 6bd00930b8..3124c20d4a 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -509,7 +509,7 @@ def assert_eq(actual, expected): async def res(*args): return await getattr(sock, resolver)(*args) - assert_eq(await res((addrs.arbitrary, "http")), (addrs.arbitrary, 80,)) + assert_eq(await res((addrs.arbitrary, "http")), (addrs.arbitrary, 80)) if v6: # Check handling of different length ipv6 address tuples assert_eq(await res(("1::2", 80)), ("1::2", 80, 0, 0)) @@ -533,7 +533,7 @@ async def res(*args): assert_eq(await res(("1.2.3.4", "http")), ("::ffff:1.2.3.4", 80)) # Check the special case, because why not - assert_eq(await res(("", 123)), (addrs.broadcast, 123,)) + assert_eq(await res(("", 123)), (addrs.broadcast, 123)) # But not if it's true (at least on systems where getaddrinfo works # correctly) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 9b2279c958..2d920c7037 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -149,7 +149,7 @@ async def ssl_echo_server_raw(**kwargs): # nursery context manager to exit too. with a, b: nursery.start_soon( - trio.to_thread.run_sync, partial(ssl_echo_serve_sync, b, **kwargs), + trio.to_thread.run_sync, partial(ssl_echo_serve_sync, b, **kwargs) ) yield SocketStream(tsocket.from_stdlib_socket(a)) @@ -1245,7 +1245,7 @@ async def setup(**kwargs): transport_client = await open_tcp_stream(*listen_sock.getsockname()) ssl_client = SSLStream( - transport_client, client_ctx, server_hostname="trio-test-1.example.org", + transport_client, client_ctx, server_hostname="trio-test-1.example.org" ) return listen_sock, ssl_listener, ssl_client diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 17f1cb941c..d8e3224475 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -272,7 +272,7 @@ async def test_run_check(): @skip_if_fbsd_pipes_broken async def test_run_with_broken_pipe(): result = await run_process( - [sys.executable, "-c", "import sys; sys.stdin.close()"], stdin=b"x" * 131072, + [sys.executable, "-c", "import sys; sys.stdin.close()"], stdin=b"x" * 131072 ) assert result.returncode == 0 assert result.stdout is result.stderr is None diff --git a/trio/tests/test_testing.py b/trio/tests/test_testing.py index 706d0a5a13..b9fb8ac2bf 100644 --- a/trio/tests/test_testing.py +++ b/trio/tests/test_testing.py @@ -416,7 +416,7 @@ def close_hook(): record.append("close_hook") mss2 = MemorySendStream( - send_all_hook, wait_send_all_might_not_block_hook, close_hook, + send_all_hook, wait_send_all_might_not_block_hook, close_hook ) assert mss2.send_all_hook is send_all_hook diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 808690d6de..e90a856a59 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -334,7 +334,7 @@ def thread_fn(cancel_scope): async def run_thread(event): with _core.CancelScope() as cancel_scope: await to_thread_run_sync( - thread_fn, cancel_scope, limiter=limiter_arg, cancellable=cancel, + thread_fn, cancel_scope, limiter=limiter_arg, cancellable=cancel ) print("run_thread finished, cancelled:", cancel_scope.cancelled_caught) event.set() diff --git a/trio/tests/test_wait_for_object.py b/trio/tests/test_wait_for_object.py index ac507a26f9..38acfa802d 100644 --- a/trio/tests/test_wait_for_object.py +++ b/trio/tests/test_wait_for_object.py @@ -101,7 +101,7 @@ async def test_WaitForMultipleObjects_sync_slow(): t0 = _core.current_time() async with _core.open_nursery() as nursery: nursery.start_soon( - trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, handle2, + trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, handle2 ) await _timeouts.sleep(TIMEOUT) kernel32.SetEvent(handle1) @@ -117,7 +117,7 @@ async def test_WaitForMultipleObjects_sync_slow(): t0 = _core.current_time() async with _core.open_nursery() as nursery: nursery.start_soon( - trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, handle2, + trio.to_thread.run_sync, WaitForMultipleObjects_sync, handle1, handle2 ) await _timeouts.sleep(TIMEOUT) kernel32.SetEvent(handle2) From f872235a81049f8eac2dd9c229023090291f48e9 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 1 Sep 2020 11:54:59 +0400 Subject: [PATCH 0403/1498] Run black 20.8b1 --- trio/_abc.py | 12 +++-------- trio/_core/_exceptions.py | 4 +--- trio/_core/_parking_lot.py | 12 +++-------- trio/_core/_run.py | 6 ++---- trio/_core/_unbounded_queue.py | 4 +--- trio/_file_io.py | 4 +--- trio/_highlevel_generic.py | 16 ++++----------- trio/_highlevel_socket.py | 4 +--- trio/_socket.py | 20 +++++------------- trio/_ssl.py | 8 ++------ trio/_sync.py | 36 +++++++++------------------------ trio/_tools/gen_exports.py | 5 ++--- trio/_util.py | 4 +--- trio/_wait_for_object.py | 4 +--- trio/testing/_memory_streams.py | 16 ++++----------- trio/tests/test_deprecate.py | 16 ++++----------- 16 files changed, 44 insertions(+), 127 deletions(-) diff --git a/trio/_abc.py b/trio/_abc.py index e44e1affe7..b8e341fdaa 100644 --- a/trio/_abc.py +++ b/trio/_abc.py @@ -8,9 +8,7 @@ # We use ABCMeta instead of ABC, plus set __slots__=(), so as not to force a # __dict__ onto subclasses. class Clock(metaclass=ABCMeta): - """The interface for custom run loop clocks. - - """ + """The interface for custom run loop clocks.""" __slots__ = () @@ -70,14 +68,10 @@ class Instrument(metaclass=ABCMeta): __slots__ = () def before_run(self): - """Called at the beginning of :func:`trio.run`. - - """ + """Called at the beginning of :func:`trio.run`.""" def after_run(self): - """Called just before :func:`trio.run` returns. - - """ + """Called just before :func:`trio.run` returns.""" def task_spawned(self, task): """Called when the given task is created. diff --git a/trio/_core/_exceptions.py b/trio/_core/_exceptions.py index 81958bc762..6189c484b4 100644 --- a/trio/_core/_exceptions.py +++ b/trio/_core/_exceptions.py @@ -25,9 +25,7 @@ class RunFinishedError(RuntimeError): class WouldBlock(Exception): - """Raised by ``X_nowait`` functions if ``X`` would block. - - """ + """Raised by ``X_nowait`` functions if ``X`` would block.""" class Cancelled(BaseException, metaclass=NoPublicConstructor): diff --git a/trio/_core/_parking_lot.py b/trio/_core/_parking_lot.py index c55d298133..4afc7f631a 100644 --- a/trio/_core/_parking_lot.py +++ b/trio/_core/_parking_lot.py @@ -104,15 +104,11 @@ class ParkingLot(metaclass=SubclassingDeprecatedIn_v0_15_0): _parked = attr.ib(factory=OrderedDict, init=False) def __len__(self): - """Returns the number of parked tasks. - - """ + """Returns the number of parked tasks.""" return len(self._parked) def __bool__(self): - """True if there are parked tasks, False otherwise. - - """ + """True if there are parked tasks, False otherwise.""" return bool(self._parked) # XX this currently returns None @@ -158,9 +154,7 @@ def unpark(self, *, count=1): return tasks def unpark_all(self): - """Unpark all parked tasks. - - """ + """Unpark all parked tasks.""" return self.unpark(count=len(self)) @_core.enable_ki_protection diff --git a/trio/_core/_run.py b/trio/_core/_run.py index fbeb645d8f..7012000ac3 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -983,7 +983,7 @@ def start_soon(self, async_fn, *args, name=None): GLOBAL_RUN_CONTEXT.runner.spawn_impl(async_fn, args, self, name) async def start(self, async_fn, *args, name=None): - r""" Creates and initalizes a child task. + r"""Creates and initalizes a child task. Like :meth:`start_soon`, but blocks until the new task has finished initializing itself, and optionally returns some @@ -1361,9 +1361,7 @@ def current_time(self): @_public def current_clock(self): - """Returns the current :class:`~trio.abc.Clock`. - - """ + """Returns the current :class:`~trio.abc.Clock`.""" return self.clock @_public diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index 14667af1e7..ceb30ca761 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -57,9 +57,7 @@ def __repr__(self): return "".format(len(self._data)) def qsize(self): - """Returns the number of items currently in the queue. - - """ + """Returns the number of items currently in the queue.""" return len(self._data) def empty(self): diff --git a/trio/_file_io.py b/trio/_file_io.py index 3a6e28a0d0..8c8425c775 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -63,9 +63,7 @@ def __init__(self, file): @property def wrapped(self): - """object: A reference to the wrapped file object - - """ + """object: A reference to the wrapped file object""" return self._wrapped diff --git a/trio/_highlevel_generic.py b/trio/_highlevel_generic.py index d4091942db..8915bdedfb 100644 --- a/trio/_highlevel_generic.py +++ b/trio/_highlevel_generic.py @@ -76,15 +76,11 @@ class StapledStream(HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15 receive_stream = attr.ib() async def send_all(self, data): - """Calls ``self.send_stream.send_all``. - - """ + """Calls ``self.send_stream.send_all``.""" return await self.send_stream.send_all(data) async def wait_send_all_might_not_block(self): - """Calls ``self.send_stream.wait_send_all_might_not_block``. - - """ + """Calls ``self.send_stream.wait_send_all_might_not_block``.""" return await self.send_stream.wait_send_all_might_not_block() async def send_eof(self): @@ -100,15 +96,11 @@ async def send_eof(self): return await self.send_stream.aclose() async def receive_some(self, max_bytes=None): - """Calls ``self.receive_stream.receive_some``. - - """ + """Calls ``self.receive_stream.receive_some``.""" return await self.receive_stream.receive_some(max_bytes) async def aclose(self): - """Calls ``aclose`` on both underlying streams. - - """ + """Calls ``aclose`` on both underlying streams.""" try: await self.send_stream.aclose() finally: diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index a707f0cac6..e676691243 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -377,8 +377,6 @@ async def accept(self): return SocketStream(sock) async def aclose(self): - """Close this listener and its underlying socket. - - """ + """Close this listener and its underlying socket.""" self.socket.close() await trio.lowlevel.checkpoint() diff --git a/trio/_socket.py b/trio/_socket.py index be4e611fb3..fcf26e072b 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -240,9 +240,7 @@ def from_stdlib_socket(sock): @_wraps(_stdlib_socket.fromfd, assigned=(), updated=()) def fromfd(fd, family, type, proto=0): - """Like :func:`socket.fromfd`, but returns a Trio socket object. - - """ + """Like :func:`socket.fromfd`, but returns a Trio socket object.""" family, type, proto = _sniff_sockopts_for_fileno(family, type, proto, fd) return from_stdlib_socket(_stdlib_socket.fromfd(fd, family, type, proto)) @@ -290,9 +288,7 @@ def socket( def _sniff_sockopts_for_fileno(family, type, proto, fileno): - """Correct SOCKOPTS for given fileno, falling back to provided values. - - """ + """Correct SOCKOPTS for given fileno, falling back to provided values.""" # Wrap the raw fileno into a Python socket object # This object might have the wrong metadata, but it lets us easily call getsockopt # and then we'll throw it away and construct a new one with the correct metadata. @@ -439,9 +435,7 @@ def __repr__(self): return repr(self._sock).replace("socket.socket", "trio.socket.socket") def dup(self): - """Same as :meth:`socket.socket.dup`. - - """ + """Same as :meth:`socket.socket.dup`.""" return _SocketType(self._sock.dup()) def close(self): @@ -609,9 +603,7 @@ async def _nonblocking_helper(self, fn, args, kwargs, wait_fn): _accept = _make_simple_sock_method_wrapper("accept", _core.wait_readable) async def accept(self): - """Like :meth:`socket.socket.accept`, but async. - - """ + """Like :meth:`socket.socket.accept`, but async.""" sock, addr = await self._accept() return from_stdlib_socket(sock), addr @@ -745,9 +737,7 @@ async def connect(self, address): @_wraps(_stdlib_socket.socket.sendto, assigned=(), updated=()) async def sendto(self, *args): - """Similar to :meth:`socket.socket.sendto`, but async. - - """ + """Similar to :meth:`socket.socket.sendto`, but async.""" # args is: data[, flags], address) # and kwargs are not accepted args = list(args) diff --git a/trio/_ssl.py b/trio/_ssl.py index 41242f45c2..1c96654561 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -830,9 +830,7 @@ async def aclose(self): self._state = _State.CLOSED async def wait_send_all_might_not_block(self): - """See :meth:`trio.abc.SendStream.wait_send_all_might_not_block`. - - """ + """See :meth:`trio.abc.SendStream.wait_send_all_might_not_block`.""" # This method's implementation is deceptively simple. # # First, we take the outer send lock, because of Trio's standard @@ -923,7 +921,5 @@ async def accept(self): ) async def aclose(self): - """Close the transport listener. - - """ + """Close the transport listener.""" await self.transport_listener.aclose() diff --git a/trio/_sync.py b/trio/_sync.py index 8c22db9ddc..ec62a1ab79 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -41,16 +41,12 @@ class Event(metaclass=SubclassingDeprecatedIn_v0_15_0): _flag = attr.ib(default=False, init=False) def is_set(self): - """Return the current value of the internal flag. - - """ + """Return the current value of the internal flag.""" return self._flag @enable_ki_protection def set(self): - """Set the internal flag value to True, and wake any waiting tasks. - - """ + """Set the internal flag value to True, and wake any waiting tasks.""" self._flag = True self._lot.unpark_all() @@ -202,16 +198,12 @@ def _wake_waiters(self): @property def borrowed_tokens(self): - """The amount of capacity that's currently in use. - - """ + """The amount of capacity that's currently in use.""" return len(self._borrowers) @property def available_tokens(self): - """The amount of capacity that's available to use. - - """ + """The amount of capacity that's available to use.""" return self.total_tokens - self.borrowed_tokens @enable_ki_protection @@ -406,16 +398,12 @@ def __repr__(self): @property def value(self): - """The current value of the semaphore. - - """ + """The current value of the semaphore.""" return self._value @property def max_value(self): - """The maximum allowed value. May be None to indicate no limit. - - """ + """The maximum allowed value. May be None to indicate no limit.""" return self._max_value @enable_ki_protection @@ -529,9 +517,7 @@ def acquire_nowait(self): @enable_ki_protection async def acquire(self): - """Acquire the lock, blocking if necessary. - - """ + """Acquire the lock, blocking if necessary.""" await trio.lowlevel.checkpoint_if_cancelled() try: self.acquire_nowait() @@ -702,15 +688,11 @@ def acquire_nowait(self): return self._lock.acquire_nowait() async def acquire(self): - """Acquire the underlying lock, blocking if necessary. - - """ + """Acquire the underlying lock, blocking if necessary.""" await self._lock.acquire() def release(self): - """Release the underlying lock. - - """ + """Release the underlying lock.""" self._lock.release() @enable_ki_protection diff --git a/trio/_tools/gen_exports.py b/trio/_tools/gen_exports.py index 6da4ee02ff..d7e6326ce6 100755 --- a/trio/_tools/gen_exports.py +++ b/trio/_tools/gen_exports.py @@ -46,8 +46,7 @@ def is_function(node): def is_public(node): - """Check if the AST node has a _public decorator - """ + """Check if the AST node has a _public decorator""" if not is_function(node): return False for decorator in node.decorator_list: @@ -57,7 +56,7 @@ def is_public(node): def get_public_methods(tree): - """ Return a list of methods marked as public. + """Return a list of methods marked as public. The function walks the given tree and extracts all objects that are functions which are marked public. diff --git a/trio/_util.py b/trio/_util.py index f1e8f9eb44..6d117503c2 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -198,9 +198,7 @@ def __exit__(self, *args): def async_wraps(cls, wrapped_cls, attr_name): - """Similar to wraps, but for async wrappers of non-async functions. - - """ + """Similar to wraps, but for async wrappers of non-async functions.""" def decorator(func): func.__name__ = attr_name diff --git a/trio/_wait_for_object.py b/trio/_wait_for_object.py index 3231e4d551..07c0461429 100644 --- a/trio/_wait_for_object.py +++ b/trio/_wait_for_object.py @@ -51,9 +51,7 @@ async def WaitForSingleObject(obj): def WaitForMultipleObjects_sync(*handles): - """Wait for any of the given Windows handles to be signaled. - - """ + """Wait for any of the given Windows handles to be signaled.""" n = len(handles) handle_arr = ffi.new("HANDLE[{}]".format(n)) for i in range(n): diff --git a/trio/testing/_memory_streams.py b/trio/testing/_memory_streams.py index 66f7f25d97..68adce06e5 100644 --- a/trio/testing/_memory_streams.py +++ b/trio/testing/_memory_streams.py @@ -155,9 +155,7 @@ def close(self): self.close_hook() async def aclose(self): - """Same as :meth:`close`, but async. - - """ + """Same as :meth:`close`, but async.""" self.close() await _core.checkpoint() @@ -251,22 +249,16 @@ def close(self): self.close_hook() async def aclose(self): - """Same as :meth:`close`, but async. - - """ + """Same as :meth:`close`, but async.""" self.close() await _core.checkpoint() def put_data(self, data): - """Appends the given data to the internal buffer. - - """ + """Appends the given data to the internal buffer.""" self._incoming.put(data) def put_eof(self): - """Adds an end-of-file marker to the internal buffer. - - """ + """Adds an end-of-file marker to the internal buffer.""" self._incoming.close() diff --git a/trio/tests/test_deprecate.py b/trio/tests/test_deprecate.py index 421278f0d3..e5e1da8c5f 100644 --- a/trio/tests/test_deprecate.py +++ b/trio/tests/test_deprecate.py @@ -160,30 +160,22 @@ def test_deprecated_alias_method(recwarn_always): @deprecated("2.1", issue=1, instead="hi") def docstring_test1(): # pragma: no cover - """Hello! - - """ + """Hello!""" @deprecated("2.1", issue=None, instead="hi") def docstring_test2(): # pragma: no cover - """Hello! - - """ + """Hello!""" @deprecated("2.1", issue=1, instead=None) def docstring_test3(): # pragma: no cover - """Hello! - - """ + """Hello!""" @deprecated("2.1", issue=None, instead=None) def docstring_test4(): # pragma: no cover - """Hello! - - """ + """Hello!""" def test_deprecated_docstring_munging(): From c747376cdc0efd9cb8de3fc17060bcc796fc7b33 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 1 Sep 2020 13:00:27 +0400 Subject: [PATCH 0404/1498] Generate sources after black update --- trio/_core/_generated_run.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index 6b41f3bfb4..8c03bf2e09 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -56,9 +56,7 @@ def current_time(): def current_clock(): - """Returns the current :class:`~trio.abc.Clock`. - - """ + """Returns the current :class:`~trio.abc.Clock`.""" locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: return GLOBAL_RUN_CONTEXT.runner.current_clock() From c52d0305d17f4d40e2951eb6f76a7267c31c030c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2020 13:05:20 +0000 Subject: [PATCH 0405/1498] Bump ipython from 7.16.1 to 7.18.1 Bumps [ipython](https://github.com/ipython/ipython) from 7.16.1 to 7.18.1. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.16.1...7.18.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4911eb3b91..6f059c9a01 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.16.1 # via -r test-requirements.in +ipython==7.18.1 # via -r test-requirements.in isort==5.4.2 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From 2a7b3dd6571bdc13b61c37f8c59fc358b6cb0452 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2020 01:25:21 +0000 Subject: [PATCH 0406/1498] Bump traitlets from 4.3.3 to 5.0.0 Bumps [traitlets](https://github.com/ipython/traitlets) from 4.3.3 to 5.0.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4911eb3b91..9bf8e05d75 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,7 +15,7 @@ cffi==1.14.2 # via cryptography click==7.1.2 # via black coverage==5.2.1 # via pytest-cov cryptography==3.1 # via pyopenssl, trustme -decorator==4.4.2 # via ipython, traitlets +decorator==4.4.2 # via ipython flake8==3.8.3 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in @@ -49,11 +49,11 @@ pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.0.1 # via -r test-requirements.in, pytest-cov regex==2020.7.14 # via black -six==1.15.0 # via astroid, cryptography, packaging, pyopenssl, traitlets +six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint, pytest -traitlets==4.3.3 # via ipython +traitlets==5.0.0 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy From 86ac43146b6fd9b80034f8a22957cb6f7adf0553 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 21:43:49 -0400 Subject: [PATCH 0407/1498] Hack in ipython==7.16.1 for Python < 3.7 --- ci.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci.sh b/ci.sh index cad85d2032..64760b316f 100755 --- a/ci.sh +++ b/ci.sh @@ -394,6 +394,10 @@ python setup.py sdist --formats=zip python -m pip install dist/*.zip if [ "$CHECK_FORMATTING" = "1" ]; then + if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then + # Python < 3.7, select last ipython with 3.6 support + sed -i 's/ipython==[^ ]+/ipython==7.16.1/' + fi python -m pip install -r test-requirements.txt source check.sh else From 620a1ca9d5e66b3fd71e99d156d233959cd75d74 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 21:47:10 -0400 Subject: [PATCH 0408/1498] Tell sed what file to tweak --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6f059c9a01..4911eb3b91 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.18.1 # via -r test-requirements.in +ipython==7.16.1 # via -r test-requirements.in isort==5.4.2 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From 694f90814a15707236b8459905bc25e1103427d8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 21:52:21 -0400 Subject: [PATCH 0409/1498] Revert "Tell sed what file to tweak" This reverts commit 620a1ca9d5e66b3fd71e99d156d233959cd75d74. --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4911eb3b91..6f059c9a01 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.16.1 # via -r test-requirements.in +ipython==7.18.1 # via -r test-requirements.in isort==5.4.2 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From d6ab6ef7ec36bcd373f732f08a6b3a40fab53649 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 21:52:39 -0400 Subject: [PATCH 0410/1498] and traitlets --- ci.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci.sh b/ci.sh index 64760b316f..f9baecff88 100755 --- a/ci.sh +++ b/ci.sh @@ -393,11 +393,13 @@ python -m pip --version python setup.py sdist --formats=zip python -m pip install dist/*.zip +if python dc 'import sys; sys.exit(sys.version_info >= (3, 7))'; then + # Python < 3.7, select last ipython with 3.6 support + sed -i 's/ipython==[^ ]+/ipython==7.16.1/' test-requirements.txt + sed -i 's/traitlets==[^ ]+/traitlets==4.3.3/' test-requirements.txt +fi + if [ "$CHECK_FORMATTING" = "1" ]; then - if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then - # Python < 3.7, select last ipython with 3.6 support - sed -i 's/ipython==[^ ]+/ipython==7.16.1/' - fi python -m pip install -r test-requirements.txt source check.sh else From 493407830049979724f0eb05ee6ceaec5c461186 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 21:54:12 -0400 Subject: [PATCH 0411/1498] Add a git diff for reference in CI --- ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci.sh b/ci.sh index f9baecff88..718d0cbf09 100755 --- a/ci.sh +++ b/ci.sh @@ -397,6 +397,7 @@ if python dc 'import sys; sys.exit(sys.version_info >= (3, 7))'; then # Python < 3.7, select last ipython with 3.6 support sed -i 's/ipython==[^ ]+/ipython==7.16.1/' test-requirements.txt sed -i 's/traitlets==[^ ]+/traitlets==4.3.3/' test-requirements.txt + git diff test-requirements.txt fi if [ "$CHECK_FORMATTING" = "1" ]; then From 232b4bc0bbb2454dc1013bd75eb430bd7000ee36 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 21:56:42 -0400 Subject: [PATCH 0412/1498] correct to python -c --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 718d0cbf09..5799124f42 100755 --- a/ci.sh +++ b/ci.sh @@ -393,7 +393,7 @@ python -m pip --version python setup.py sdist --formats=zip python -m pip install dist/*.zip -if python dc 'import sys; sys.exit(sys.version_info >= (3, 7))'; then +if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then # Python < 3.7, select last ipython with 3.6 support sed -i 's/ipython==[^ ]+/ipython==7.16.1/' test-requirements.txt sed -i 's/traitlets==[^ ]+/traitlets==4.3.3/' test-requirements.txt From 9fb4d56d382cb1137f662f7cc7b8dc3399972ec0 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 21:59:13 -0400 Subject: [PATCH 0413/1498] bash handles * better --- ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci.sh b/ci.sh index 5799124f42..8325f3ff3e 100755 --- a/ci.sh +++ b/ci.sh @@ -395,8 +395,8 @@ python -m pip install dist/*.zip if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then # Python < 3.7, select last ipython with 3.6 support - sed -i 's/ipython==[^ ]+/ipython==7.16.1/' test-requirements.txt - sed -i 's/traitlets==[^ ]+/traitlets==4.3.3/' test-requirements.txt + sed -i 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt + sed -i 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt git diff test-requirements.txt fi From ea6ceb2e4f53fc4a4187d409dd13cd08fb4a1e33 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 22:06:31 -0400 Subject: [PATCH 0414/1498] --in-place='.bak' --- ci.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci.sh b/ci.sh index 8325f3ff3e..3b58ab126e 100755 --- a/ci.sh +++ b/ci.sh @@ -395,8 +395,9 @@ python -m pip install dist/*.zip if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then # Python < 3.7, select last ipython with 3.6 support - sed -i 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt - sed -i 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt + # macOS requires the suffix for --in-place or you get an undefined label error + sed --in-place='.bak' 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt + sed --in-place='.bak' 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt git diff test-requirements.txt fi From d45a972bd97bdfb89270407dffdd3a5189e037c1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 22:11:32 -0400 Subject: [PATCH 0415/1498] macOS only supports -i, not --in-place apparently --- ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci.sh b/ci.sh index 3b58ab126e..57bcef7c00 100755 --- a/ci.sh +++ b/ci.sh @@ -396,8 +396,8 @@ python -m pip install dist/*.zip if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then # Python < 3.7, select last ipython with 3.6 support # macOS requires the suffix for --in-place or you get an undefined label error - sed --in-place='.bak' 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt - sed --in-place='.bak' 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt + sed -i '.bak' 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt + sed -i '.bak' 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt git diff test-requirements.txt fi From 3cd6ecc54517ce95c1790069696976f2f0b71281 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 2 Sep 2020 22:17:52 -0400 Subject: [PATCH 0416/1498] maybe no space? --- ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci.sh b/ci.sh index 57bcef7c00..1386e15280 100755 --- a/ci.sh +++ b/ci.sh @@ -396,8 +396,8 @@ python -m pip install dist/*.zip if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then # Python < 3.7, select last ipython with 3.6 support # macOS requires the suffix for --in-place or you get an undefined label error - sed -i '.bak' 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt - sed -i '.bak' 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt + sed -i'.bak' 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt + sed -i'.bak' 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt git diff test-requirements.txt fi From 73717b328bbe9551d64a3b6a317a568d08dffd7d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2020 03:12:36 +0000 Subject: [PATCH 0417/1498] Bump traitlets from 4.3.3 to 5.0.2 Bumps [traitlets](https://github.com/ipython/traitlets) from 4.3.3 to 5.0.2. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/4.3.3...5.0.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a6b965acea..fc187b783f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,7 +53,7 @@ six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint, pytest -traitlets==5.0.0 # via ipython +traitlets==5.0.2 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy From 4cc77bb1dc09ba7b26d0e0e51a7d0c60b430b5d5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2020 04:28:14 +0000 Subject: [PATCH 0418/1498] Bump isort from 5.4.2 to 5.5.0 Bumps [isort](https://github.com/pycqa/isort) from 5.4.2 to 5.5.0. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.4.2...5.5.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fc187b783f..b4b0809f45 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.18.1 # via -r test-requirements.in -isort==5.4.2 # via pylint +isort==5.5.0 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From 1c1be91f666bfc0e972e9311410f8e77f8abe5fe Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2020 04:39:09 +0000 Subject: [PATCH 0419/1498] Bump traitlets from 5.0.2 to 5.0.3 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.0.2 to 5.0.3. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.0.2...5.0.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b4b0809f45..833616b567 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,7 +53,7 @@ six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint, pytest -traitlets==5.0.2 # via ipython +traitlets==5.0.3 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy From 05ed422078eb6d9526a4d5fa7230e345bb99ff2d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2020 04:31:29 +0000 Subject: [PATCH 0420/1498] Bump isort from 5.5.0 to 5.5.1 Bumps [isort](https://github.com/pycqa/isort) from 5.5.0 to 5.5.1. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.5.0...5.5.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 833616b567..749b2b418d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.18.1 # via -r test-requirements.in -isort==5.5.0 # via pylint +isort==5.5.1 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From 634e2a5b1c8e2141f0081e76a7f0d0b462ef56d6 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 4 Sep 2020 23:32:30 -0700 Subject: [PATCH 0421/1498] Bump up the timeout in test_pipes to hopefully reduce spurious CI failures --- trio/tests/test_subprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index d8e3224475..7ba794a428 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -134,8 +134,8 @@ async def check_output(stream, expected): assert seen == expected async with _core.open_nursery() as nursery: - # fail quickly if something is broken - nursery.cancel_scope.deadline = _core.current_time() + 3.0 + # fail eventually if something is broken + nursery.cancel_scope.deadline = _core.current_time() + 30.0 nursery.start_soon(feed_input) nursery.start_soon(check_output, proc.stdout, msg) nursery.start_soon(check_output, proc.stderr, msg[::-1]) From 8dedd95743e5e32398e6a13f30d78c527caf8cee Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 08:57:49 +0000 Subject: [PATCH 0422/1498] Bump attrs from 20.1.0 to 20.2.0 Bumps [attrs](https://github.com/python-attrs/attrs) from 20.1.0 to 20.2.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/master/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/20.1.0...20.2.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 84cd29ad72..e02d328fd4 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -6,7 +6,7 @@ # alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in -attrs==20.1.0 # via -r docs-requirements.in, outcome +attrs==20.2.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx certifi==2020.6.20 # via requests chardet==3.0.4 # via requests diff --git a/test-requirements.txt b/test-requirements.txt index 749b2b418d..42857fbf7b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in astroid==2.4.2 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==20.1.0 # via -r test-requirements.in, outcome, pytest +attrs==20.2.0 # via -r test-requirements.in, outcome, pytest backcall==0.2.0 # via ipython black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.2 # via cryptography From 15494b5c0edef1aabceca88769fedfdd05b32a0f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 08:58:50 +0000 Subject: [PATCH 0423/1498] Bump traitlets from 5.0.3 to 5.0.4 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.0.3 to 5.0.4. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.0.3...5.0.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 749b2b418d..cf6296d8be 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,7 +53,7 @@ six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint, pytest -traitlets==5.0.3 # via ipython +traitlets==5.0.4 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy From 26384e218b29986732da59585cc1036037026d9d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2020 02:16:33 +0000 Subject: [PATCH 0424/1498] Bump isort from 5.5.1 to 5.5.2 Bumps [isort](https://github.com/pycqa/isort) from 5.5.1 to 5.5.2. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.5.1...5.5.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1d79ec0a59..9125e840ac 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.18.1 # via -r test-requirements.in -isort==5.5.1 # via pylint +isort==5.5.2 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From 7a28996443c96a7f6b889c52fddfc9ed5ec5dbe7 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 11 Sep 2020 10:35:10 +0400 Subject: [PATCH 0425/1498] Remove deprecated trio.hazmat --- trio/__init__.py | 10 ---------- trio/hazmat.py | 13 ------------- 2 files changed, 23 deletions(-) delete mode 100644 trio/hazmat.py diff --git a/trio/__init__.py b/trio/__init__.py index a7340e0819..d66ffceea9 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -105,16 +105,6 @@ from . import _deprecate _deprecate.enable_attribute_deprecations(__name__) -__deprecated_attributes__ = { - # NOTE: when you remove this, you should also remove the file - # trio/hazmat.py. For details on why we have both, see: - # - # https://github.com/python-trio/trio/pull/1484#issuecomment-622574499 - "hazmat": _deprecate.DeprecatedAttribute( - lowlevel, "0.15.0", issue=476, instead="trio.lowlevel" - ), -} - # Having the public path in .__module__ attributes is important for: # - exception names in printed tracebacks diff --git a/trio/hazmat.py b/trio/hazmat.py deleted file mode 100644 index 1d079ae8f8..0000000000 --- a/trio/hazmat.py +++ /dev/null @@ -1,13 +0,0 @@ -# This file is a hack to work around a quirk in Python's import machinery: -# -# https://github.com/python-trio/trio/pull/1484#issuecomment-622574499 -# -# It should be removed at the same time as the _DeprecatedAttribute("hazmat", -# ...) inside trio/__init__.py. - -from ._deprecate import warn_deprecated - -warn_deprecated("trio.hazmat", "0.15.0", issue=476, instead="trio.lowlevel") -del warn_deprecated - -from .lowlevel import * From 7e9c5b75d4273e464c75d2eaa1898826911d239e Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sat, 12 Sep 2020 06:51:09 +1000 Subject: [PATCH 0426/1498] docs: Fix simple typo, renegotation -> renegotiation There is a small typo in trio/_ssl.py, trio/tests/test_ssl.py. Should read `renegotiation` rather than `renegotation`. --- trio/_ssl.py | 4 ++-- trio/tests/test_ssl.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/trio/_ssl.py b/trio/_ssl.py index 1c96654561..dd120beef7 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -69,7 +69,7 @@ # able to use this to figure out the key. Is this a real practical problem? I # have no idea, I'm not a cryptographer. In any case, some people worry that # it's a problem, so their TLS libraries are designed to automatically trigger -# a renegotation every once in a while on some sort of timer. +# a renegotiation every once in a while on some sort of timer. # # The end result is that you might be going along, minding your own business, # and then *bam*! a wild renegotiation appears! And you just have to cope. @@ -542,7 +542,7 @@ async def _retry(self, fn, *args, ignore_want_read=False, is_handshake=False): # We could do something tricky to keep track of whether a # receive_some happens while we're sending, but the case where # we have to do both is very unusual (only during a - # renegotation), so it's better to keep things simple. So we + # renegotiation), so it's better to keep things simple. So we # do just one potentially-blocking operation, then check again # for fresh information. # diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 2d920c7037..a4e22716ae 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -231,7 +231,7 @@ def renegotiate_pending(self): return self._conn.renegotiate_pending() def renegotiate(self): - # Returns false if a renegotation is already in progress, meaning + # Returns false if a renegotiation is already in progress, meaning # nothing happens. assert self._conn.renegotiate() @@ -323,7 +323,7 @@ async def receive_some(self, nbytes=None): async def test_PyOpenSSLEchoStream_gives_resource_busy_errors(): # Make sure that PyOpenSSLEchoStream complains if two tasks call send_all # at the same time, or ditto for receive_some. The tricky cases where SSLStream - # might accidentally do this are during renegotation, which we test using + # might accidentally do this are during renegotiation, which we test using # PyOpenSSLEchoStream, so this makes sure that if we do have a bug then # PyOpenSSLEchoStream will notice and complain. From 54514bc7ef7f597853d0f4f41811e068541d9791 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2020 03:52:52 +0000 Subject: [PATCH 0427/1498] Bump pytest from 6.0.1 to 6.0.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.0.1...6.0.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9125e840ac..13b9cfcbc7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in -pytest==6.0.1 # via -r test-requirements.in, pytest-cov +pytest==6.0.2 # via -r test-requirements.in, pytest-cov regex==2020.7.14 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in From c14a6e391a93aad1df8e5e4084bdda3c1de9bd4f Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Sat, 12 Sep 2020 15:41:19 +0400 Subject: [PATCH 0428/1498] Add newsfragment --- newsfragments/1722.deprecated.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1722.deprecated.rst diff --git a/newsfragments/1722.deprecated.rst b/newsfragments/1722.deprecated.rst new file mode 100644 index 0000000000..0c963d5a4a --- /dev/null +++ b/newsfragments/1722.deprecated.rst @@ -0,0 +1 @@ +Remove the deprecated ``trio.hazmat`` module. From c13fb467af7e2282c05b406bbe1b3ce820a36e9a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 09:10:16 +0000 Subject: [PATCH 0429/1498] Bump pygments from 2.6.1 to 2.7.0 Bumps [pygments](https://github.com/pygments/pygments) from 2.6.1 to 2.7.0. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.6.1...2.7.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index e02d328fd4..510f437e37 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -20,7 +20,7 @@ jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.0.1 # via -r docs-requirements.in packaging==20.4 # via sphinx -pygments==2.6.1 # via sphinx +pygments==2.7.0 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.1 # via babel requests==2.24.0 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index 13b9cfcbc7..5bc1fc046c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -42,7 +42,7 @@ py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 -pygments==2.6.1 # via ipython +pygments==2.7.0 # via ipython pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging From 79341691a13eb522a46940c1234eed8f124ea69d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 09:29:42 +0000 Subject: [PATCH 0430/1498] Bump coverage from 5.2.1 to 5.3 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.2.1 to 5.3. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.2.1...coverage-5.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5bc1fc046c..8603c01762 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,7 +13,7 @@ backcall==0.2.0 # via ipython black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.2 # via cryptography click==7.1.2 # via black -coverage==5.2.1 # via pytest-cov +coverage==5.3 # via pytest-cov cryptography==3.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython flake8==3.8.3 # via -r test-requirements.in From e8f052eca8bfcf39b04f60bc0e5de9861b8715c6 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 14 Sep 2020 20:57:09 +0400 Subject: [PATCH 0431/1498] Stop allowing subclassing public classes It was deprecated in Trio 0.15.0. --- newsfragments/1726.deprecated.rst | 1 + trio/_core/_local.py | 4 ++-- trio/_core/_mock_clock.py | 4 ++-- trio/_core/_parking_lot.py | 4 ++-- trio/_core/_unbounded_queue.py | 4 ++-- trio/_highlevel_generic.py | 4 ++-- trio/_highlevel_socket.py | 6 +++--- trio/_path.py | 4 ++-- trio/_ssl.py | 6 +++--- trio/_sync.py | 14 +++++++------- trio/_unix_pipes.py | 4 ++-- trio/_util.py | 14 -------------- trio/testing/_memory_streams.py | 6 ++---- trio/testing/_sequencer.py | 2 +- trio/tests/test_exports.py | 4 +--- trio/tests/test_util.py | 11 ----------- 16 files changed, 32 insertions(+), 60 deletions(-) create mode 100644 newsfragments/1726.deprecated.rst diff --git a/newsfragments/1726.deprecated.rst b/newsfragments/1726.deprecated.rst new file mode 100644 index 0000000000..14cb51b9a7 --- /dev/null +++ b/newsfragments/1726.deprecated.rst @@ -0,0 +1 @@ +Stop allowing subclassing public classes. This behavior was deprecated in 0.15.0. diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 352caa5682..1f64d4ce85 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -1,7 +1,7 @@ # Runvar implementations from . import _run -from .._util import SubclassingDeprecatedIn_v0_15_0 +from .._util import Final class _RunVarToken: @@ -19,7 +19,7 @@ def __init__(self, var, value): self.redeemed = False -class RunVar(metaclass=SubclassingDeprecatedIn_v0_15_0): +class RunVar(metaclass=Final): """The run-local variant of a context variable. :class:`RunVar` objects are similar to context variable objects, diff --git a/trio/_core/_mock_clock.py b/trio/_core/_mock_clock.py index 6d9332fd23..0e95e4e5c5 100644 --- a/trio/_core/_mock_clock.py +++ b/trio/_core/_mock_clock.py @@ -4,7 +4,7 @@ from .. import _core from ._run import GLOBAL_RUN_CONTEXT from .._abc import Clock -from .._util import SubclassingDeprecatedIn_v0_15_0 +from .._util import Final ################################################################ # The glorious MockClock @@ -14,7 +14,7 @@ # Prior art: # https://twistedmatrix.com/documents/current/api/twisted.internet.task.Clock.html # https://github.com/ztellman/manifold/issues/57 -class MockClock(Clock, metaclass=SubclassingDeprecatedIn_v0_15_0): +class MockClock(Clock, metaclass=Final): """A user-controllable clock suitable for writing tests. Args: diff --git a/trio/_core/_parking_lot.py b/trio/_core/_parking_lot.py index 4afc7f631a..8b114b5230 100644 --- a/trio/_core/_parking_lot.py +++ b/trio/_core/_parking_lot.py @@ -75,7 +75,7 @@ from collections import OrderedDict from .. import _core -from .._util import SubclassingDeprecatedIn_v0_15_0 +from .._util import Final _counter = count() @@ -86,7 +86,7 @@ class _ParkingLotStatistics: @attr.s(eq=False, hash=False) -class ParkingLot(metaclass=SubclassingDeprecatedIn_v0_15_0): +class ParkingLot(metaclass=Final): """A fair wait queue with cancellation and requeueing. This class encapsulates the tricky parts of implementing a wait diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index ceb30ca761..f877e42a0c 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -2,7 +2,7 @@ from .. import _core from .._deprecate import deprecated -from .._util import SubclassingDeprecatedIn_v0_15_0 +from .._util import Final @attr.s(frozen=True) @@ -11,7 +11,7 @@ class _UnboundedQueueStats: tasks_waiting = attr.ib() -class UnboundedQueue(metaclass=SubclassingDeprecatedIn_v0_15_0): +class UnboundedQueue(metaclass=Final): """An unbounded queue suitable for certain unusual forms of inter-task communication. diff --git a/trio/_highlevel_generic.py b/trio/_highlevel_generic.py index 8915bdedfb..c31b4fdbf3 100644 --- a/trio/_highlevel_generic.py +++ b/trio/_highlevel_generic.py @@ -3,7 +3,7 @@ import trio from .abc import HalfCloseableStream -from trio._util import SubclassingDeprecatedIn_v0_15_0 +from trio._util import Final async def aclose_forcefully(resource): @@ -37,7 +37,7 @@ async def aclose_forcefully(resource): @attr.s(eq=False, hash=False) -class StapledStream(HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0): +class StapledStream(HalfCloseableStream, metaclass=Final): """This class `staples `__ together two unidirectional streams to make single bidirectional stream. diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index e676691243..0d9dbc0e92 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -5,7 +5,7 @@ import trio from . import socket as tsocket -from ._util import ConflictDetector, SubclassingDeprecatedIn_v0_15_0 +from ._util import ConflictDetector, Final from .abc import HalfCloseableStream, Listener # XX TODO: this number was picked arbitrarily. We should do experiments to @@ -35,7 +35,7 @@ def _translate_socket_errors_to_stream_errors(): ) from exc -class SocketStream(HalfCloseableStream, metaclass=SubclassingDeprecatedIn_v0_15_0): +class SocketStream(HalfCloseableStream, metaclass=Final): """An implementation of the :class:`trio.abc.HalfCloseableStream` interface based on a raw network socket. @@ -315,7 +315,7 @@ def getsockopt(self, level, option, buffersize=0): pass -class SocketListener(Listener[SocketStream], metaclass=SubclassingDeprecatedIn_v0_15_0): +class SocketListener(Listener[SocketStream], metaclass=Final): """A :class:`~trio.abc.Listener` that uses a listening socket to accept incoming connections as :class:`SocketStream` objects. diff --git a/trio/_path.py b/trio/_path.py index d2442c89f6..4077c449d7 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -6,7 +6,7 @@ import pathlib import trio -from trio._util import async_wraps, SubclassingDeprecatedIn_v0_15_0 +from trio._util import async_wraps, Final # re-wrap return value from methods that return new instances of pathlib.Path @@ -77,7 +77,7 @@ async def wrapper(cls, *args, **kwargs): return wrapper -class AsyncAutoWrapperType(SubclassingDeprecatedIn_v0_15_0): +class AsyncAutoWrapperType(Final): def __init__(cls, name, bases, attrs): super().__init__(name, bases, attrs) diff --git a/trio/_ssl.py b/trio/_ssl.py index dd120beef7..9c346d268b 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -158,7 +158,7 @@ from .abc import Stream, Listener from ._highlevel_generic import aclose_forcefully from . import _sync -from ._util import ConflictDetector, SubclassingDeprecatedIn_v0_15_0 +from ._util import ConflictDetector, Final from ._deprecate import warn_deprecated ################################################################ @@ -224,7 +224,7 @@ def done(self): _State = _Enum("_State", ["OK", "BROKEN", "CLOSED"]) -class SSLStream(Stream, metaclass=SubclassingDeprecatedIn_v0_15_0): +class SSLStream(Stream, metaclass=Final): r"""Encrypted communication using SSL/TLS. :class:`SSLStream` wraps an arbitrary :class:`~trio.abc.Stream`, and @@ -870,7 +870,7 @@ async def wait_send_all_might_not_block(self): await self.transport_stream.wait_send_all_might_not_block() -class SSLListener(Listener[SSLStream], metaclass=SubclassingDeprecatedIn_v0_15_0): +class SSLListener(Listener[SSLStream], metaclass=Final): """A :class:`~trio.abc.Listener` for SSL/TLS-encrypted servers. :class:`SSLListener` wraps around another Listener, and converts diff --git a/trio/_sync.py b/trio/_sync.py index ec62a1ab79..bed339ef6b 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -7,11 +7,11 @@ from ._core import enable_ki_protection, ParkingLot from ._deprecate import deprecated -from ._util import SubclassingDeprecatedIn_v0_15_0 +from ._util import Final @attr.s(repr=False, eq=False, hash=False) -class Event(metaclass=SubclassingDeprecatedIn_v0_15_0): +class Event(metaclass=Final): """A waitable boolean value useful for inter-task synchronization, inspired by :class:`threading.Event`. @@ -99,7 +99,7 @@ class _CapacityLimiterStatistics: @async_cm -class CapacityLimiter(metaclass=SubclassingDeprecatedIn_v0_15_0): +class CapacityLimiter(metaclass=Final): """An object for controlling access to a resource with limited capacity. Sometimes you need to put a limit on how many tasks can do something at @@ -342,7 +342,7 @@ def statistics(self): @async_cm -class Semaphore(metaclass=SubclassingDeprecatedIn_v0_15_0): +class Semaphore(metaclass=Final): """A `semaphore `__. A semaphore holds an integer value, which can be incremented by @@ -562,7 +562,7 @@ def statistics(self): ) -class Lock(_LockImpl, metaclass=SubclassingDeprecatedIn_v0_15_0): +class Lock(_LockImpl, metaclass=Final): """A classic `mutex `__. @@ -576,7 +576,7 @@ class Lock(_LockImpl, metaclass=SubclassingDeprecatedIn_v0_15_0): """ -class StrictFIFOLock(_LockImpl, metaclass=SubclassingDeprecatedIn_v0_15_0): +class StrictFIFOLock(_LockImpl, metaclass=Final): r"""A variant of :class:`Lock` where tasks are guaranteed to acquire the lock in strict first-come-first-served order. @@ -646,7 +646,7 @@ class _ConditionStatistics: @async_cm -class Condition(metaclass=SubclassingDeprecatedIn_v0_15_0): +class Condition(metaclass=Final): """A classic `condition variable `__, similar to :class:`threading.Condition`. diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index faf4162551..0d2d11c53c 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -2,7 +2,7 @@ import errno from ._abc import Stream -from ._util import ConflictDetector, SubclassingDeprecatedIn_v0_15_0 +from ._util import ConflictDetector, Final import trio @@ -74,7 +74,7 @@ async def aclose(self): await trio.lowlevel.checkpoint() -class FdStream(Stream, metaclass=SubclassingDeprecatedIn_v0_15_0): +class FdStream(Stream, metaclass=Final): """ Represents a stream given the file descriptor to a pipe, TTY, etc. diff --git a/trio/_util.py b/trio/_util.py index 6d117503c2..0775a107fd 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -309,20 +309,6 @@ def __new__(cls, name, bases, cls_namespace): return super().__new__(cls, name, bases, cls_namespace) -class SubclassingDeprecatedIn_v0_15_0(BaseMeta): - def __new__(cls, name, bases, cls_namespace): - for base in bases: - if isinstance(base, SubclassingDeprecatedIn_v0_15_0): - warn_deprecated( - f"subclassing {base.__module__}.{base.__qualname__}", - "0.15.0", - issue=1044, - instead="composition or delegation", - ) - break - return super().__new__(cls, name, bases, cls_namespace) - - T = t.TypeVar("T") diff --git a/trio/testing/_memory_streams.py b/trio/testing/_memory_streams.py index 68adce06e5..99ad7dfcaf 100644 --- a/trio/testing/_memory_streams.py +++ b/trio/testing/_memory_streams.py @@ -73,7 +73,7 @@ async def get(self, max_bytes=None): return self._get_impl(max_bytes) -class MemorySendStream(SendStream, metaclass=_util.SubclassingDeprecatedIn_v0_15_0): +class MemorySendStream(SendStream, metaclass=_util.Final): """An in-memory :class:`~trio.abc.SendStream`. Args: @@ -187,9 +187,7 @@ def get_data_nowait(self, max_bytes=None): return self._outgoing.get_nowait(max_bytes) -class MemoryReceiveStream( - ReceiveStream, metaclass=_util.SubclassingDeprecatedIn_v0_15_0 -): +class MemoryReceiveStream(ReceiveStream, metaclass=_util.Final): """An in-memory :class:`~trio.abc.ReceiveStream`. Args: diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index abecf396ce..a7e6e50ff0 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -12,7 +12,7 @@ @attr.s(eq=False, hash=False) -class Sequencer(metaclass=_util.SubclassingDeprecatedIn_v0_15_0): +class Sequencer(metaclass=_util.Final): """A convenience class for forcing code in different tasks to run in an explicit linear order. diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 40c1586578..43fd8a2282 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -141,6 +141,4 @@ def test_classes_are_final(): continue # ... insert other special cases here ... - assert isinstance( - class_, (_util.Final, _util.SubclassingDeprecatedIn_v0_15_0) - ) + assert isinstance(class_, (_util.Final, _util.Final)) diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index b08676e622..2ea0a1e287 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -12,7 +12,6 @@ generic_function, Final, NoPublicConstructor, - SubclassingDeprecatedIn_v0_15_0, ) from ..testing import wait_all_tasks_blocked @@ -171,16 +170,6 @@ class SubClass(FinalClass): pass -def test_subclassing_deprecated_metaclass(): - class Blah(metaclass=SubclassingDeprecatedIn_v0_15_0): - pass - - with pytest.warns(trio.TrioDeprecationWarning): - - class Blah2(Blah): - pass - - def test_no_public_constructor_metaclass(): class SpecialClass(metaclass=NoPublicConstructor): pass From d0362a1a0c1d765f2e2f0e3db22df7ffe5d4380d Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 15 Sep 2020 14:55:02 +0400 Subject: [PATCH 0432/1498] Bump version to 0.17.0 --- docs/source/history.rst | 62 +++++++++++++++++++++++++++++++ newsfragments/1340.misc.rst | 6 --- newsfragments/1595.feature.rst | 3 -- newsfragments/1596.deprecated.rst | 1 - newsfragments/1613.feature.rst | 1 - newsfragments/1621.bugfix.rst | 2 - newsfragments/1629.feature.rst | 2 - newsfragments/1630.bugfix.rst | 2 - newsfragments/1638.bugfix.rst | 1 - newsfragments/1659.bugfix.rst | 7 ---- newsfragments/265.headline.rst | 6 --- newsfragments/275.feature.rst | 3 -- trio/_version.py | 2 +- 13 files changed, 63 insertions(+), 35 deletions(-) delete mode 100644 newsfragments/1340.misc.rst delete mode 100644 newsfragments/1595.feature.rst delete mode 100644 newsfragments/1596.deprecated.rst delete mode 100644 newsfragments/1613.feature.rst delete mode 100644 newsfragments/1621.bugfix.rst delete mode 100644 newsfragments/1629.feature.rst delete mode 100644 newsfragments/1630.bugfix.rst delete mode 100644 newsfragments/1638.bugfix.rst delete mode 100644 newsfragments/1659.bugfix.rst delete mode 100644 newsfragments/265.headline.rst delete mode 100644 newsfragments/275.feature.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 33f72d8681..830e3acafc 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,68 @@ Release history .. towncrier release notes start +Trio 0.17.0 (2020-09-15) +------------------------ + +Headline features +~~~~~~~~~~~~~~~~~ + +- Trio now supports automatic :ref:`async generator finalization + `, so more async generators will work even if you + don't wrap them in ``async with async_generator.aclosing():`` + blocks. Please see the documentation for important caveats; in + particular, yielding within a nursery or cancel scope remains + unsupported. (`#265 `__) + + +Features +~~~~~~~~ + +- `trio.open_tcp_stream` has a new ``local_address=`` keyword argument + that can be used on machines with multiple IP addresses to control + which IP is used for the outgoing connection. (`#275 `__) +- If you pass a raw IP address into ``sendto``, it no longer spends any + time trying to resolve the hostname. If you're using UDP, this should + substantially reduce your per-packet overhead. (`#1595 `__) +- `trio.lowlevel.checkpoint` is now much faster. (`#1613 `__) +- We switched to a new, lower-overhead data structure to track upcoming + timeouts, which should make your programs faster. (`#1629 `__) + + +Bugfixes +~~~~~~~~ + +- On macOS and BSDs, explicitly close our wakeup socketpair when we're + done with it. (`#1621 `__) +- Trio can now be imported when `sys.excepthook` is a `functools.partial` instance, which might occur in a + ``pytest-qt`` test function. (`#1630 `__) +- The thread cache didn't release its reference to the previous job. (`#1638 `__) +- On Windows, Trio now works around the buggy behavior of certain + Layered Service Providers (system components that can intercept + network activity) that are built on top of a commercially available + library called Komodia Redirector. This benefits users of products + such as Astrill VPN and Qustodio parental controls. Previously, Trio + would crash on startup when run on a system where such a product was + installed. (`#1659 `__) + + +Deprecations and removals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Remove ``wait_socket_*``, ``notify_socket_closing``, ``notify_fd_closing``, ``run_sync_in_worker_thread`` and ``current_default_worker_thread_limiter``. They were deprecated in 0.12.0. (`#1596 `__) + + +Miscellaneous internal changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- When using :ref:`instruments `, you now only "pay for what you use": + if there are no instruments installed that override a particular hook such as + :meth:`~trio.abc.Instrument.before_task_step`, then Trio doesn't waste any effort + on checking its instruments when the event corresponding to that hook occurs. + Previously, installing any instrument would incur all the instrumentation overhead, + even for hooks no one was interested in. (`#1340 `__) + + Trio 0.16.0 (2020-06-10) ------------------------ diff --git a/newsfragments/1340.misc.rst b/newsfragments/1340.misc.rst deleted file mode 100644 index 9899505e2f..0000000000 --- a/newsfragments/1340.misc.rst +++ /dev/null @@ -1,6 +0,0 @@ -When using :ref:`instruments `, you now only "pay for what you use": -if there are no instruments installed that override a particular hook such as -:meth:`~trio.abc.Instrument.before_task_step`, then Trio doesn't waste any effort -on checking its instruments when the event corresponding to that hook occurs. -Previously, installing any instrument would incur all the instrumentation overhead, -even for hooks no one was interested in. diff --git a/newsfragments/1595.feature.rst b/newsfragments/1595.feature.rst deleted file mode 100644 index b7a0df3938..0000000000 --- a/newsfragments/1595.feature.rst +++ /dev/null @@ -1,3 +0,0 @@ -If you pass a raw IP address into ``sendto``, it no longer spends any -time trying to resolve the hostname. If you're using UDP, this should -substantially reduce your per-packet overhead. diff --git a/newsfragments/1596.deprecated.rst b/newsfragments/1596.deprecated.rst deleted file mode 100644 index 9736419485..0000000000 --- a/newsfragments/1596.deprecated.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``wait_socket_*``, ``notify_socket_closing``, ``notify_fd_closing``, ``run_sync_in_worker_thread`` and ``current_default_worker_thread_limiter``. They were deprecated in 0.12.0. diff --git a/newsfragments/1613.feature.rst b/newsfragments/1613.feature.rst deleted file mode 100644 index 2971bb054d..0000000000 --- a/newsfragments/1613.feature.rst +++ /dev/null @@ -1 +0,0 @@ -`trio.lowlevel.checkpoint` is now much faster. diff --git a/newsfragments/1621.bugfix.rst b/newsfragments/1621.bugfix.rst deleted file mode 100644 index 2eb869c969..0000000000 --- a/newsfragments/1621.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -On macOS and BSDs, explicitly close our wakeup socketpair when we're -done with it. diff --git a/newsfragments/1629.feature.rst b/newsfragments/1629.feature.rst deleted file mode 100644 index 6047ae4ccf..0000000000 --- a/newsfragments/1629.feature.rst +++ /dev/null @@ -1,2 +0,0 @@ -We switched to a new, lower-overhead data structure to track upcoming -timeouts, which should make your programs faster. diff --git a/newsfragments/1630.bugfix.rst b/newsfragments/1630.bugfix.rst deleted file mode 100644 index 0b33777705..0000000000 --- a/newsfragments/1630.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Trio can now be imported when `sys.excepthook` is a `functools.partial` instance, which might occur in a -``pytest-qt`` test function. diff --git a/newsfragments/1638.bugfix.rst b/newsfragments/1638.bugfix.rst deleted file mode 100644 index 6c6a9a6a5d..0000000000 --- a/newsfragments/1638.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -The thread cache didn't release its reference to the previous job. diff --git a/newsfragments/1659.bugfix.rst b/newsfragments/1659.bugfix.rst deleted file mode 100644 index 395a369002..0000000000 --- a/newsfragments/1659.bugfix.rst +++ /dev/null @@ -1,7 +0,0 @@ -On Windows, Trio now works around the buggy behavior of certain -Layered Service Providers (system components that can intercept -network activity) that are built on top of a commercially available -library called Komodia Redirector. This benefits users of products -such as Astrill VPN and Qustodio parental controls. Previously, Trio -would crash on startup when run on a system where such a product was -installed. diff --git a/newsfragments/265.headline.rst b/newsfragments/265.headline.rst deleted file mode 100644 index 57a1ae30b4..0000000000 --- a/newsfragments/265.headline.rst +++ /dev/null @@ -1,6 +0,0 @@ -Trio now supports automatic :ref:`async generator finalization -`, so more async generators will work even if you -don't wrap them in ``async with async_generator.aclosing():`` -blocks. Please see the documentation for important caveats; in -particular, yielding within a nursery or cancel scope remains -unsupported. diff --git a/newsfragments/275.feature.rst b/newsfragments/275.feature.rst deleted file mode 100644 index 26b8ebb521..0000000000 --- a/newsfragments/275.feature.rst +++ /dev/null @@ -1,3 +0,0 @@ -`trio.open_tcp_stream` has a new ``local_address=`` keyword argument -that can be used on machines with multiple IP addresses to control -which IP is used for the outgoing connection. diff --git a/trio/_version.py b/trio/_version.py index 4d1671b9ec..16ee802331 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.16.0+dev" +__version__ = "0.17.0" From c37aa8c935fa9e81ba2e17e3a5c9a477178c3707 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 15 Sep 2020 15:12:22 +0400 Subject: [PATCH 0433/1498] Bump version to 0.17.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index 16ee802331..cd6e4e1649 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.17.0" +__version__ = "0.17.0+dev" From 934cfe56ab6c2bad532e49423a773f6210f76c7a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 15 Sep 2020 15:16:03 +0400 Subject: [PATCH 0434/1498] Stop using RawGit URL in PyPI RawGit has shut down, but GitHub fixed the original issue so it's no longer needed. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f76c36e378..726da99365 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ exec(open("trio/_version.py", encoding="utf-8").read()) LONG_DESC = """\ -.. image:: https://cdn.rawgit.com/python-trio/trio/9b0bec646a31e0d0f67b8b6ecc6939726faf3e17/logo/logo-with-background.svg +.. image:: https://raw.githubusercontent.com/python-trio/trio/9b0bec646a31e0d0f67b8b6ecc6939726faf3e17/logo/logo-with-background.svg :width: 200px :align: right From 8a4cb08f53a653e760b8b8db1aade2a927cdd8c2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2020 02:59:07 +0000 Subject: [PATCH 0435/1498] Bump pygments from 2.7.0 to 2.7.1 Bumps [pygments](https://github.com/pygments/pygments) from 2.7.0 to 2.7.1. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.7.0...2.7.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 510f437e37..f2e9fffbf3 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -20,7 +20,7 @@ jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.0.1 # via -r docs-requirements.in packaging==20.4 # via sphinx -pygments==2.7.0 # via sphinx +pygments==2.7.1 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.1 # via babel requests==2.24.0 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index 8603c01762..d54ea642b1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -42,7 +42,7 @@ py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 -pygments==2.7.0 # via ipython +pygments==2.7.1 # via ipython pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging From 397f67e03e8a03ba262ad6e683e78d1926bddd93 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 18 Sep 2020 14:04:29 +0400 Subject: [PATCH 0436/1498] Simplify redundant isinstance check --- trio/tests/test_exports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 43fd8a2282..374ce8c044 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -141,4 +141,4 @@ def test_classes_are_final(): continue # ... insert other special cases here ... - assert isinstance(class_, (_util.Final, _util.Final)) + assert isinstance(class_, _util.Final) From 48eee1c54808ecbd4d79634256e1d72ed25f8efc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2020 08:54:03 +0000 Subject: [PATCH 0437/1498] Bump isort from 5.5.2 to 5.5.3 Bumps [isort](https://github.com/pycqa/isort) from 5.5.2 to 5.5.3. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.5.2...5.5.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d54ea642b1..69c946351a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.18.1 # via -r test-requirements.in -isort==5.5.2 # via pylint +isort==5.5.3 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From a444f2635ff6383219f53bf9b1d22aed30ffb87c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2020 09:19:01 +0000 Subject: [PATCH 0438/1498] Bump cffi from 1.14.2 to 1.14.3 Bumps [cffi](https://github.com/python-cffi/release-doc) from 1.14.2 to 1.14.3. - [Release notes](https://github.com/python-cffi/release-doc/releases) - [Commits](https://github.com/python-cffi/release-doc/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 69c946351a..615ff014b4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ async-generator==1.10 # via -r test-requirements.in attrs==20.2.0 # via -r test-requirements.in, outcome, pytest backcall==0.2.0 # via ipython black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.2 # via cryptography +cffi==1.14.3 # via cryptography click==7.1.2 # via black coverage==5.3 # via pytest-cov cryptography==3.1 # via pyopenssl, trustme From 39a2926d3e9967e05f073aabbd7daca6a83a970a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 09:09:38 +0000 Subject: [PATCH 0439/1498] Bump regex from 2020.7.14 to 2020.9.27 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.7.14 to 2020.9.27. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 615ff014b4..6434610516 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -48,7 +48,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.0.2 # via -r test-requirements.in, pytest-cov -regex==2020.7.14 # via black +regex==2020.9.27 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in From 314fa105347a45c7af67ee1c3f4b21cb22bdaa27 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 09:25:44 +0000 Subject: [PATCH 0440/1498] Bump pytest from 6.0.2 to 6.1.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.0.2 to 6.1.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.0.2...6.1.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6434610516..c04739cc67 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -26,7 +26,6 @@ isort==5.5.3 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint -more-itertools==8.5.0 # via pytest mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy mypy==0.782 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in @@ -47,7 +46,7 @@ pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in -pytest==6.0.2 # via -r test-requirements.in, pytest-cov +pytest==6.1.0 # via -r test-requirements.in, pytest-cov regex==2020.9.27 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in From bad1095b14274230fecd275c5e06ad19daefda1d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 09:53:03 +0000 Subject: [PATCH 0441/1498] Bump cryptography from 3.1 to 3.1.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.1 to 3.1.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.1...3.1.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c04739cc67..26e316ec19 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,7 +14,7 @@ black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.3 # via cryptography click==7.1.2 # via black coverage==5.3 # via pytest-cov -cryptography==3.1 # via pyopenssl, trustme +cryptography==3.1.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython flake8==3.8.3 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme From dcb98ec6bc846272293a1b872581937df74f9463 Mon Sep 17 00:00:00 2001 From: LUXEY Adrien Date: Tue, 29 Sep 2020 08:52:32 +0200 Subject: [PATCH 0442/1498] clarified the 'networking with trio' section of the tutorial: the echo client does not work without a server listening on the designated address --- docs/source/tutorial.rst | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index e435966f3b..7cc1816c54 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -836,19 +836,25 @@ Networking with Trio Now let's take what we've learned and use it to do some I/O, which is where async/await really shines. - -An echo client -~~~~~~~~~~~~~~ - -The traditional application for demonstrating network APIs is an "echo -server": a program that accepts arbitrary data from a client, and then -sends that same data right back. (Probably a more relevant example +The traditional toy application for demonstrating network APIs is an +"echo server": a program that awaits arbitrary data from remote clients, +and then sends that same data right back. (Probably a more relevant example these days would be an application that does lots of concurrent HTTP requests, but for that `you need an HTTP library `__ such as `asks `__, so we'll stick with the echo server tradition.) +In this tutorial, we present both ends of the pipe: the client, and the +server. The client periodically sends data to the server, and displays its +answers. The server awaits connections; when a client connects, it recopies +the received data back on the pipe. + + +An echo client +~~~~~~~~~~~~~~ + + To start with, here's an example echo *client*, i.e., the program that will send some data at our echo server and get responses back: @@ -857,6 +863,9 @@ will send some data at our echo server and get responses back: .. literalinclude:: tutorial/echo-client.py :linenos: +Note that this code will not work without a TCP server listening at +``127.0.0.1:12345``. + The overall structure here should be familiar, because it's just like our :ref:`last example `: we have a parent task, which spawns two child tasks to do the actual work, and From e0210293d0754a72d9213d52700c8bce81ece62a Mon Sep 17 00:00:00 2001 From: Adrien Luxey Date: Tue, 29 Sep 2020 09:42:30 +0200 Subject: [PATCH 0443/1498] Update docs/source/tutorial.rst Co-authored-by: Quentin Pradet --- docs/source/tutorial.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 7cc1816c54..676ff91b23 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -863,8 +863,8 @@ will send some data at our echo server and get responses back: .. literalinclude:: tutorial/echo-client.py :linenos: -Note that this code will not work without a TCP server listening at -``127.0.0.1:12345``. +Note that this code will not work without a TCP server such as the one +we'll implement below. The overall structure here should be familiar, because it's just like our :ref:`last example `: we have a From eaf19199689e7556dbaf7d5756078d0383986e16 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2020 01:44:01 +0000 Subject: [PATCH 0444/1498] Bump isort from 5.5.3 to 5.5.4 Bumps [isort](https://github.com/pycqa/isort) from 5.5.3 to 5.5.4. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.5.3...5.5.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 26e316ec19..a64f48a4ea 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.18.1 # via -r test-requirements.in -isort==5.5.3 # via pylint +isort==5.5.4 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From 784e76d743ef299b7c5c9f50e965e12bcfc9ac49 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Thu, 1 Oct 2020 11:32:19 +0900 Subject: [PATCH 0445/1498] add Python 3.9-rc to CI build --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b73f5d20b..0276a1ea8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8'] + python: ['3.6', '3.7', '3.8', '3.9.0-rc.2'] arch: ['x86', 'x64'] lsp: [''] lsp_extract_file: [''] @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8'] + python: ['3.6', '3.7', '3.8', '3.9.0-rc.2'] check_formatting: ['0'] extra_name: [''] include: @@ -87,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8'] + python: ['3.6', '3.7', '3.8', '3.9.0-rc.2'] steps: - name: Checkout uses: actions/checkout@v2 From 5b2d17d54e0bbb53c03f60527ad45a439725414b Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Thu, 1 Oct 2020 11:51:23 +0900 Subject: [PATCH 0446/1498] upgrade to setup-python@v2 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0276a1ea8e..c302766fad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '${{ matrix.python }}' architecture: '${{ matrix.arch }}' @@ -70,7 +70,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '${{ matrix.python }}' - name: Run tests @@ -92,7 +92,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '${{ matrix.python }}' - name: Run tests From 3a8a2473f0040ba68774a254ad5adeacfbe3e546 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Thu, 1 Oct 2020 11:53:49 +0900 Subject: [PATCH 0447/1498] fix `current_task()` call in example (#1739) --- docs/source/reference-lowlevel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 36b37e3e37..ebd0f57c03 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -455,7 +455,7 @@ this does serve to illustrate the basic structure of the async def acquire(self): while self._held: - task = trio.current_task() + task = trio.lowlevel.current_task() self._blocked_tasks.append(task) def abort_fn(_): self._blocked_tasks.remove(task) From f6c332592af7912ac1910160d5e4b84cfb05180b Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 30 Sep 2020 22:07:21 -0800 Subject: [PATCH 0448/1498] Make trio.from_thread.run() raise RunFinishedError if the system nursery is closed --- newsfragments/1738.bugfix.rst | 6 ++++++ trio/_core/_run.py | 9 +++++++++ trio/_threads.py | 12 ++++++++++-- trio/tests/test_threads.py | 22 ++++++++++++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 newsfragments/1738.bugfix.rst diff --git a/newsfragments/1738.bugfix.rst b/newsfragments/1738.bugfix.rst new file mode 100644 index 0000000000..ddb42050b4 --- /dev/null +++ b/newsfragments/1738.bugfix.rst @@ -0,0 +1,6 @@ +:func:`trio.from_thread.run` no longer crashes the Trio run if it is +executed after the system nursery has been closed but before the run +has finished. Calls made at this time will now raise +`trio.RunFinishedError`. This fixes a regression introduced in +Trio 0.17.0. The window in question is only one scheduler tick long in +most cases, but may be longer if async generators need to be cleaned up. diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 7012000ac3..fe51f7bfc8 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1546,6 +1546,15 @@ def spawn_system_task(self, async_fn, *args, name=None): * System tasks do not inherit context variables from their creator. + Towards the end of a call to :meth:`trio.run`, after the main + task and all system tasks have exited, the system nursery + becomes closed. At this point, new calls to + :func:`spawn_system_task` will raise ``RuntimeError("Nursery + is closed to new arrivals")`` instead of creating a system + task. It's possible to encounter this state either in + a ``finally`` block in an async generator, or in a callback + passed to :meth:`TrioToken.run_sync_soon` at the right moment. + Args: async_fn: An async callable. args: Positional arguments for ``async_fn``. If you want to pass diff --git a/trio/_threads.py b/trio/_threads.py index 1f90268aa0..648b87d801 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -1,3 +1,5 @@ +# coding: utf-8 + import threading import queue as stdlib_queue from itertools import count @@ -248,7 +250,8 @@ def from_thread_run(afn, *args, trio_token=None): Raises: RunFinishedError: if the corresponding call to :func:`trio.run` has - already completed. + already completed, or if the run has started its final cleanup phase + and can no longer spawn new system tasks. Cancelled: if the corresponding call to :func:`trio.run` completes while ``afn(*args)`` is running, then ``afn`` is likely to raise :exc:`trio.Cancelled`, and this will propagate out into @@ -279,7 +282,12 @@ async def unprotected_afn(): async def await_in_trio_thread_task(): q.put_nowait(await outcome.acapture(unprotected_afn)) - trio.lowlevel.spawn_system_task(await_in_trio_thread_task, name=afn) + try: + trio.lowlevel.spawn_system_task(await_in_trio_thread_task, name=afn) + except RuntimeError: # system nursery is closed + q.put_nowait( + outcome.Error(trio.RunFinishedError("system nursery is closed")) + ) return _run_fn_as_system_task(callback, afn, *args, trio_token=trio_token) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index e90a856a59..9da2838cbd 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -7,6 +7,7 @@ from .. import _core from .. import Event, CapacityLimiter, sleep from ..testing import wait_all_tasks_blocked +from .._core.tests.tutil import buggy_pypy_asyncgens from .._threads import ( to_thread_run_sync, current_default_thread_limiter, @@ -554,3 +555,24 @@ def not_called(): # pragma: no cover trio_token = _core.current_trio_token() with pytest.raises(RuntimeError): from_thread_run_sync(not_called, trio_token=trio_token) + + +@pytest.mark.skipif(buggy_pypy_asyncgens, reason="pypy 7.2.0 is buggy") +def test_from_thread_run_during_shutdown(): + save = [] + record = [] + + async def agen(): + try: + yield + finally: + with pytest.raises(_core.RunFinishedError), _core.CancelScope(shield=True): + await to_thread_run_sync(from_thread_run, sleep, 0) + record.append("ok") + + async def main(): + save.append(agen()) + await save[-1].asend(None) + + _core.run(main) + assert record == ["ok"] From 5d940e01ea9afed99db255edc9b7e2e5e2de8bf7 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Wed, 30 Sep 2020 22:16:28 -0800 Subject: [PATCH 0449/1498] Rerun gen_exports --- trio/_core/_generated_run.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index 8c03bf2e09..0da2c4f48e 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -126,6 +126,15 @@ def spawn_system_task(async_fn, *args, name=None): * System tasks do not inherit context variables from their creator. + Towards the end of a call to :meth:`trio.run`, after the main + task and all system tasks have exited, the system nursery + becomes closed. At this point, new calls to + :func:`spawn_system_task` will raise ``RuntimeError("Nursery + is closed to new arrivals")`` instead of creating a system + task. It's possible to encounter this state either in + a ``finally`` block in an async generator, or in a callback + passed to :meth:`TrioToken.run_sync_soon` at the right moment. + Args: async_fn: An async callable. args: Positional arguments for ``async_fn``. If you want to pass From e4ac375c4f3edbd38ab8f0cf60df619a9093cce6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2020 01:39:55 +0000 Subject: [PATCH 0450/1498] Bump flake8 from 3.8.3 to 3.8.4 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.8.3 to 3.8.4. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.8.3...3.8.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a64f48a4ea..cc56e5561f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -16,7 +16,7 @@ click==7.1.2 # via black coverage==5.3 # via pytest-cov cryptography==3.1.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython -flake8==3.8.3 # via -r test-requirements.in +flake8==3.8.4 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest From f3df00c84940a420422aaf1d8b2fe4a41071fc3a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 14 Oct 2020 14:27:58 +0400 Subject: [PATCH 0451/1498] Fix FreeBSD build by busting the cache My hope is that this is going to reinstall/rebuild and stop depending on an old libffi version. https://forums.freebsd.org/threads/certbot-python3-7-dependency-error.76808/ --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 876c6ece5c..35f85357d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ jobs: - "FREEBSD_INSTALLER_ISO_XZ=https://download.freebsd.org/ftp/releases/amd64/amd64/ISO-IMAGES/12.1/FreeBSD-12.1-RELEASE-amd64-disc1.iso.xz" # Increment this each time you change the image build code in # ci.sh and want to intentionally bust the cache. - - "CACHE_GEN=2" + - "CACHE_GEN=3" cache: directories: - travis-cache From 563792f485e9b52313f17c6d8f84e0abfdb45bcc Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 14 Oct 2020 09:51:00 -0400 Subject: [PATCH 0452/1498] Use .png for favicon --- docs/source/_static/favicon-32.ico | Bin 4286 -> 0 bytes docs/source/conf.py | 9 +-------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 docs/source/_static/favicon-32.ico diff --git a/docs/source/_static/favicon-32.ico b/docs/source/_static/favicon-32.ico deleted file mode 100644 index 1ec20bf8a49972877dd2a5651dd8f0f28680c656..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4286 zcmcIndu&r>6u)EY5`#h_F=*n31VzQTwzu1EyGOgIQDOv*A}R?IAh2%h*4oEj-KYi( z;bkO*wQL=Gv}@Nr5geKzS?~}4LGTaB*nD9$5*6l1#@PGwckb=y?w%O|ww!+V@qOoa zzH@%(obL)k9R63V5a?HkEER-D1wnWQf=7ZGLeJ$tygRf=5#_B`3V*RrdyIrq*y-))Uc^Ox9eQ$?88|cWtk~;%bY>G}=~{b*8mEJALMgg*|Do zNxGmTt+l}+T?;y;k)YGi+vqa11zm=%jSllNt|jU+FOL+bv!>zs-*A_V9GhT3ae3JLa+_Z$58y8(A07( zXldqaWCNb1VQO2NiZV{MmAv=}e9}*L(L2Nv>tguWBlg>0nXB1yEKkd-@^lQgFtshRPhihv zeBhhe;{(o04Dn&GMcym$uD6tC+KC$+pWYL%>gFYWNjHFbOz$dx|i!xzL$;s4$L1>+cK3aN_^xza!3*2UnP9XSB?+<+~D{M2gFNWulYXMaGu*x znWKGIjjz7;+r-DJ3&av|rF$N(^flNGtAKx#YL$#HbI$=9OeU5CYw(2AB$;FyN@zQZyYJIg`C)ebe_~4*JIy(&?yw@fZ#<9HQr-AufVxWo_QDA}bJnX{BTls8K5-U`@K zHgR1o<>r$+I!^sdb=vFSJwo`6PQxeDa{y~A_!>U?^0ZmLZMs*1y9bz262B$61>^T* zgMuB)Szb4kdNJTH5|@x3?B5;rWcM9DxU(l*oH?p+P_j(V0g4~TRgeB0ZL#{EU;Fbk zE;&!JZ>m{x9>;2xT)Ul88qIAl&lbS}xi`depbPbCJ=M@@Inc{>^RY7Lh39=an&0`o zdJI_ftEo;+#h&aNAy?W{d_M47kOQ|OB^gIgxYHjByV5gxANaoxqYecM^{e?B>T~e% zS;!2#Xik+?zi%dXEcYE!H`g=U7%I+KDc50DA0Yo>Kl}N7_}xM=_T?m>;m`PpSnWjK zPsqKB`V{Ih`I727_BGMylwRj^XB+8!0Q~V8Ibanh{JFXYU#>0)>@gYNqTS1NkWawV zB4CeG-J-h3`!wkypAxtE{Q1U^+t?%HQ!nN@2p&-nB3>)l*gtZB^iUqw6o{c2Jd^h= zpHa|rqyuE*e7O7_`as?E9H4ko9&>!kf3H_qRF$9lBw}-M5_^(o%DxT5A-5r)>t4u+ z^VCMpL5iQ;2fzWsr(R6^uguYGsAOE?WqudJgq7;Quwf z4pUAMK59;9pg{K~um|Vj8GPuEdd-W~dT-|%wr`tq7NHzK4E%YzpR26e&A=X;gJ<2S zqq$+Xp*37=>_Ttf4t~gI$=kuFx%Ey%2{=5a;^6tpe7za59+(AtH0Us#M65f3KS5_Y zs%dn-L466|E;gSZvZt)9v!!ki*z{|o8#crPyLmF!Jh!DmIXzNwtUr|?WZ(-AhE<=hQN2dJmnn(D`ajqjiA^>>2H*HZo> zA8x7YnR1>4E#ui`-kTUUxUI8y(^@^vV5*bvLl- z`)&~X{tn;XyIab$*DMr^u^|&<{YzMvCZ2hPxGRt@EcU7u(37|!2#XVzUP%y^3-LlC z>UfeTPDu2`c^7-)hZYyd4+&cm#so7&NP?IkyC!fA&Lly166=@6#D;`j5XQ#h*?$LB Bvd91c diff --git a/docs/source/conf.py b/docs/source/conf.py index 1d0665f347..6045ffd828 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -118,14 +118,7 @@ def setup(app): # The full version, including alpha/beta/rc tags. release = version -# It would be nicer to make this a .png; literally every browser that -# supports favicons at all now supports png: -# https://caniuse.com/#feat=link-icon-png -# But sphinx won't let me: -# https://github.com/sphinx-doc/sphinx/pull/3715 -# Oh well. 'convert favicon-32.png favicon-32.ico' it is. And it's only 2x -# bigger... -html_favicon = "_static/favicon-32.ico" +html_favicon = "_static/favicon-32.png" html_logo = "../../logo/wordmark-transparent.svg" # & down below in html_theme_options we set logo_only=True From 3fa9c8f939e6eb40e47240130da58a5c996808b5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 18:28:27 +0000 Subject: [PATCH 0453/1498] Bump prompt-toolkit from 3.0.7 to 3.0.8 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.7 to 3.0.8. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index cc56e5561f..09bf9daa7a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -35,7 +35,7 @@ pathspec==0.8.0 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.7 # via ipython +prompt-toolkit==3.0.8 # via ipython ptyprocess==0.6.0 # via pexpect py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 From 4fe9b92b3058bec9a49421e8e8534573798b22ef Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 18:48:57 +0000 Subject: [PATCH 0454/1498] Bump pytest from 6.1.0 to 6.1.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.1.0 to 6.1.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.1.0...6.1.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 09bf9daa7a..bedd9f2e34 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -46,7 +46,7 @@ pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in -pytest==6.1.0 # via -r test-requirements.in, pytest-cov +pytest==6.1.1 # via -r test-requirements.in, pytest-cov regex==2020.9.27 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in From a9179f7918107780c14ca9f38ec84616f5d86fbb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 18:59:05 +0000 Subject: [PATCH 0455/1498] Bump isort from 5.5.4 to 5.6.4 Bumps [isort](https://github.com/pycqa/isort) from 5.5.4 to 5.6.4. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.5.4...5.6.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 09bf9daa7a..57fd62c7e7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.0.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.18.1 # via -r test-requirements.in -isort==5.5.4 # via pylint +isort==5.6.4 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From fdcc60d2dbcb9f4e42718b148da6e277ce968774 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 19:28:20 +0000 Subject: [PATCH 0456/1498] Bump regex from 2020.9.27 to 2020.10.11 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.9.27 to 2020.10.11. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index bedd9f2e34..7b5b3ecbd6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.1 # via -r test-requirements.in, pytest-cov -regex==2020.9.27 # via black +regex==2020.10.11 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.1.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in From b6abc1a4cc940dcefecf06dd6a707b5894e147ae Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 19:57:27 +0000 Subject: [PATCH 0457/1498] Bump sniffio from 1.1.0 to 1.2.0 Bumps [sniffio](https://github.com/python-trio/sniffio) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/python-trio/sniffio/releases) - [Commits](https://github.com/python-trio/sniffio/compare/v1.1.0...v1.2.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f2e9fffbf3..83b1270acc 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -25,7 +25,7 @@ pyparsing==2.4.7 # via packaging pytz==2020.1 # via babel requests==2.24.0 # via sphinx six==1.15.0 # via packaging -sniffio==1.1.0 # via -r docs-requirements.in +sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.2.2 # via -r docs-requirements.in sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in diff --git a/test-requirements.txt b/test-requirements.txt index f92e3e4925..d529876273 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -49,7 +49,7 @@ pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.1 # via -r test-requirements.in, pytest-cov regex==2020.10.11 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl -sniffio==1.1.0 # via -r test-requirements.in +sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint, pytest traitlets==5.0.4 # via ipython From d8d3e486ea90fa65f4332ea3f658822deb3ca528 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2020 01:10:22 +0000 Subject: [PATCH 0458/1498] Bump mypy from 0.782 to 0.790 Bumps [mypy](https://github.com/python/mypy) from 0.782 to 0.790. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.782...v0.790) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d529876273..bc6ecd4461 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -27,7 +27,7 @@ jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy -mypy==0.782 ; implementation_name == "cpython" # via -r test-requirements.in +mypy==0.790 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest parso==0.7.1 # via jedi From 508ead5908b497e0e156c1b8fa4e4e35b4196a30 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2020 03:06:21 +0000 Subject: [PATCH 0459/1498] Bump iniconfig from 1.0.1 to 1.1.1 Bumps [iniconfig](https://github.com/RonnyPfannschmidt/iniconfig) from 1.0.1 to 1.1.1. - [Release notes](https://github.com/RonnyPfannschmidt/iniconfig/releases) - [Changelog](https://github.com/RonnyPfannschmidt/iniconfig/blob/master/CHANGELOG) - [Commits](https://github.com/RonnyPfannschmidt/iniconfig/compare/v1.0.1...v1.1.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index bc6ecd4461..a7ddb73995 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ decorator==4.4.2 # via ipython flake8==3.8.4 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in -iniconfig==1.0.1 # via pytest +iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.18.1 # via -r test-requirements.in isort==5.6.4 # via pylint From 4ca3a88e71e4d73e3ecef5facb449b05a4edc2cd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2020 12:05:11 +0000 Subject: [PATCH 0460/1498] Bump traitlets from 5.0.4 to 5.0.5 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.0.4 to 5.0.5. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.0.4...5.0.5) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a7ddb73995..1e36b30ec7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -52,7 +52,7 @@ six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in toml==0.10.1 # via black, pylint, pytest -traitlets==5.0.4 # via ipython +traitlets==5.0.5 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy From ad67d332d8d4b10467519842310c5a397e1cf3b2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2020 12:05:58 +0000 Subject: [PATCH 0461/1498] Bump regex from 2020.10.11 to 2020.10.15 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.10.11 to 2020.10.15. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a7ddb73995..dc8ae8e796 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.1 # via -r test-requirements.in, pytest-cov -regex==2020.10.11 # via black +regex==2020.10.15 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in From 9da626701289791690490b2272b25e1f4f0e1db0 Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Mon, 19 Oct 2020 14:01:44 -0700 Subject: [PATCH 0462/1498] Fix a crash in pypy. Apparently, in pypy, "signal.set_wakeup_fd" doesn't have the keyword argument warn_on_full_buffer. Here is a simple script to reproduce this problem: import trio async def handler(stream: trio.abc.Stream): await stream.receive_some(1) async def trio_main(): socket_listeners = await trio.serve_tcp(handler) trio.run(trio_main) --- trio/_core/_wakeup_socketpair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 86eaeb6355..d78b1a506b 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -6,7 +6,7 @@ from .. import _core from .._util import is_main_thread -if sys.version_info >= (3, 7): +if sys.version_info >= (3, 7) and '__pypy__' not in sys.builtin_module_names: HAVE_WARN_ON_FULL_BUFFER = True else: HAVE_WARN_ON_FULL_BUFFER = False From 79ebffe51713466b969ff9270a2f8bf007ceb45e Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Mon, 19 Oct 2020 17:41:36 -0700 Subject: [PATCH 0463/1498] Update format. --- trio/_core/_wakeup_socketpair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index d78b1a506b..1f6e0191ae 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -6,7 +6,7 @@ from .. import _core from .._util import is_main_thread -if sys.version_info >= (3, 7) and '__pypy__' not in sys.builtin_module_names: +if sys.version_info >= (3, 7) and "__pypy__" not in sys.builtin_module_names: HAVE_WARN_ON_FULL_BUFFER = True else: HAVE_WARN_ON_FULL_BUFFER = False From 7015e8e99caee1f27f2b8089cd0f7c3a814903fc Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Tue, 20 Oct 2020 00:30:49 -0700 Subject: [PATCH 0464/1498] Improve warn_on_full_buffer support inspection. pypy 3.7 doesn't support warn_on_full_buffer for now. https://foss.heptapod.net/pypy/pypy/-/issues/3227 So, when running in pypy 3.7, inspect whether this is still the case. --- trio/_core/_wakeup_socketpair.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 1f6e0191ae..7c03c8a8d9 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -6,10 +6,22 @@ from .. import _core from .._util import is_main_thread -if sys.version_info >= (3, 7) and "__pypy__" not in sys.builtin_module_names: - HAVE_WARN_ON_FULL_BUFFER = True -else: - HAVE_WARN_ON_FULL_BUFFER = False + +def _has_warn_on_full_buffer(): + if sys.version_info < (3, 7): + return False + + if "__pypy__" not in sys.builtin_module_names: + # CPython has warn_on_full_buffer. Don't need to inspect. + # Also, CPython doesn't support inspecting built-in functions. + return True + + import inspect + + return "warn_on_full_buffer" in inspect.getfullargspec(set_wakeup_fd).kwonlyargs + + +HAVE_WARN_ON_FULL_BUFFER = _has_warn_on_full_buffer() class WakeupSocketpair: From 7931b05d25d21e65bf6181dd26ce07952770b33b Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Tue, 20 Oct 2020 00:33:34 -0700 Subject: [PATCH 0465/1498] Oops, an extra whitespace. --- trio/_core/_wakeup_socketpair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 7c03c8a8d9..2228960244 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -8,7 +8,7 @@ def _has_warn_on_full_buffer(): - if sys.version_info < (3, 7): + if sys.version_info < (3, 7): return False if "__pypy__" not in sys.builtin_module_names: From c67636ce0daebf2ec6895122bdfb915263db6043 Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Tue, 20 Oct 2020 00:36:54 -0700 Subject: [PATCH 0466/1498] Extract args_spec. --- trio/_core/_wakeup_socketpair.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 2228960244..121cec584e 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -18,7 +18,8 @@ def _has_warn_on_full_buffer(): import inspect - return "warn_on_full_buffer" in inspect.getfullargspec(set_wakeup_fd).kwonlyargs + args_spec = inspect.getfullargspec(signal.set_wakeup_fd) + return "warn_on_full_buffer" in args_spec.kwonlyargs HAVE_WARN_ON_FULL_BUFFER = _has_warn_on_full_buffer() From c8d49bd4da3f6c1271b1e2b7e00df98df699bc76 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2020 17:20:44 +0000 Subject: [PATCH 0467/1498] Bump regex from 2020.10.15 to 2020.10.23 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.10.15 to 2020.10.23. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 141b5c702d..c53a4201a1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.1 # via -r test-requirements.in, pytest-cov -regex==2020.10.15 # via black +regex==2020.10.23 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in From be276b424d79cd3d24b0d7231f4cd708a6f600e6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2020 00:55:30 +0000 Subject: [PATCH 0468/1498] Bump pygments from 2.7.1 to 2.7.2 Bumps [pygments](https://github.com/pygments/pygments) from 2.7.1 to 2.7.2. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.7.1...2.7.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 83b1270acc..abf332fccc 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -20,7 +20,7 @@ jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.0.1 # via -r docs-requirements.in packaging==20.4 # via sphinx -pygments==2.7.1 # via sphinx +pygments==2.7.2 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.1 # via babel requests==2.24.0 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index c53a4201a1..53c4f1e9ec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 -pygments==2.7.1 # via ipython +pygments==2.7.2 # via ipython pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging From c5ae26ac15b42d92a2f3506d7bc622d2e874e8a6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2020 13:29:28 +0000 Subject: [PATCH 0469/1498] Bump cryptography from 3.1.1 to 3.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.1.1 to 3.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.1.1...3.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 53c4f1e9ec..2bfac680dd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,7 +14,7 @@ black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.3 # via cryptography click==7.1.2 # via black coverage==5.3 # via pytest-cov -cryptography==3.1.1 # via pyopenssl, trustme +cryptography==3.2 # via pyopenssl, trustme decorator==4.4.2 # via ipython flake8==3.8.4 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme From a76f57cf2f61fe9562d8b9e00e8b1f41b799b183 Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Mon, 26 Oct 2020 20:47:10 -0700 Subject: [PATCH 0470/1498] Create 1765.bugfix.rst --- newsfragments/1765.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1765.bugfix.rst diff --git a/newsfragments/1765.bugfix.rst b/newsfragments/1765.bugfix.rst new file mode 100644 index 0000000000..7d771c13e5 --- /dev/null +++ b/newsfragments/1765.bugfix.rst @@ -0,0 +1 @@ +Fix a crash in pypy-3.7 From a2dee140bcab1d1241410c565e5e78d466ebe086 Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Tue, 27 Oct 2020 00:58:41 -0700 Subject: [PATCH 0471/1498] Add pypy3.7 to travis. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 35f85357d9..911834d76d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ jobs: dist: bionic - language: generic env: PYPY_NIGHTLY_BRANCH=py3.6 + - language: generic + env: PYPY_NIGHTLY_BRANCH=py3.7 # Qemu tests are also slow # FreeBSD: - language: generic From 5163a253c2b89a106a7e326e2339ff9a87b973fd Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Tue, 27 Oct 2020 22:26:50 +0900 Subject: [PATCH 0472/1498] suggest @trio_util.trio_async_generator in docs on yield from nursery --- docs/source/reference-core.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 664a8d96c2..d3c6a02f71 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1632,12 +1632,12 @@ an exception, where can that exception be reraised? If you have an async generator that wants to ``yield`` from within a nursery or cancel scope, your best bet is to refactor it to be a separate task -that communicates over memory channels. +that communicates over memory channels. The ``trio_util`` package offers a +`decorator that does this for you automatically +`_. -For more discussion and some experimental partial workarounds, see +For more discussion, see Trio issues `264 `__ -(especially `this comment -`__) and `638 `__. From 003a66bdde1dba3972c507c1c95c27dbfa79a540 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Tue, 27 Oct 2020 22:33:38 +0900 Subject: [PATCH 0473/1498] Update reference-core.rst --- docs/source/reference-core.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index d3c6a02f71..023fd04f25 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1634,10 +1634,12 @@ If you have an async generator that wants to ``yield`` from within a nursery or cancel scope, your best bet is to refactor it to be a separate task that communicates over memory channels. The ``trio_util`` package offers a `decorator that does this for you automatically -`_. +`__. For more discussion, see Trio issues `264 `__ +(especially `this comment +`__) and `638 `__. From 674dd8da9aa84b0db988e648f8f69c905646382c Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Tue, 27 Oct 2020 22:39:19 +0900 Subject: [PATCH 0474/1498] Update reference-core.rst --- docs/source/reference-core.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 023fd04f25..7d2cf44a61 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1633,7 +1633,7 @@ an exception, where can that exception be reraised? If you have an async generator that wants to ``yield`` from within a nursery or cancel scope, your best bet is to refactor it to be a separate task that communicates over memory channels. The ``trio_util`` package offers a -`decorator that does this for you automatically +`decorator that does this for you transparently `__. For more discussion, see From c5141dbb1fdde3df43d928fc7e282b96cf5a64e9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2020 10:59:11 +0000 Subject: [PATCH 0475/1498] Bump regex from 2020.10.23 to 2020.10.28 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.10.23 to 2020.10.28. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2bfac680dd..c467830ea0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.1 # via -r test-requirements.in, pytest-cov -regex==2020.10.23 # via black +regex==2020.10.28 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in From 5933f55fd0de649f1813ae1e8a4d55c59ea21242 Mon Sep 17 00:00:00 2001 From: leojay <130761+leojay@users.noreply.github.com> Date: Thu, 29 Oct 2020 19:54:50 -0700 Subject: [PATCH 0476/1498] test_guest_mode_asyncgens is expected to fail in pypy. --- trio/_core/tests/test_guest_mode.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 381d966f81..c9701e7cdd 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -502,6 +502,10 @@ async def trio_main(in_host): @pytest.mark.skipif(buggy_pypy_asyncgens, reason="PyPy 7.2 is buggy") +@pytest.mark.xfail( + sys.implementation.name == "pypy" and sys.version_info >= (3, 7), + reason="async generator issue under investigation", +) def test_guest_mode_asyncgens(): import sniffio From 3e69c3f8b9a05a95abe8dd5ddbda196a5797b11b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 30 Oct 2020 16:03:55 -0700 Subject: [PATCH 0477/1498] Try using an older version of Proxifier Apparently the latest one no longer uses an LSP? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c302766fad..afc612c301 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: extra_name: ', with Komodia LSP' - python: '3.8' arch: 'x64' - lsp: 'http://www.proxifier.com/download/ProxifierSetup.exe' + lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' lsp_extract_file: '' extra_name: ', with IFS LSP' - python: '3.8' From 1cd39c46bea8e2c659363b40113664a2e4921ec8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2020 06:48:56 +0000 Subject: [PATCH 0478/1498] Bump ipython from 7.18.1 to 7.19.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.18.1 to 7.19.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.18.1...7.19.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c467830ea0..c83aa0b7cc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ idna==2.10 # via -r test-requirements.in, trustme immutables==0.14 # via -r test-requirements.in iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.18.1 # via -r test-requirements.in +ipython==7.19.0 # via -r test-requirements.in isort==5.6.4 # via pylint jedi==0.17.2 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid From bb5954ea97b02d86efcea21d7a5e483d0a56124a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2020 07:10:24 +0000 Subject: [PATCH 0479/1498] Bump pytest from 6.1.1 to 6.1.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.1.1 to 6.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.1.1...6.1.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c83aa0b7cc..dcb22d8d73 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -46,7 +46,7 @@ pylint==2.6.0 # via -r test-requirements.in pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in -pytest==6.1.1 # via -r test-requirements.in, pytest-cov +pytest==6.1.2 # via -r test-requirements.in, pytest-cov regex==2020.10.28 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in From 5e96b51e87bd0df87e155c0e58955f8620388f63 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2020 07:28:13 +0000 Subject: [PATCH 0480/1498] Bump cryptography from 3.2 to 3.2.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.2 to 3.2.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.2...3.2.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index dcb22d8d73..5767ea27ec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,7 +14,7 @@ black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.3 # via cryptography click==7.1.2 # via black coverage==5.3 # via pytest-cov -cryptography==3.2 # via pyopenssl, trustme +cryptography==3.2.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython flake8==3.8.4 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme From b61e5d7d4d76958906cb1f4672efa15c89fcde0b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2020 07:51:04 +0000 Subject: [PATCH 0481/1498] Bump urllib3 from 1.25.10 to 1.25.11 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.25.10 to 1.25.11. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/master/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.25.10...1.25.11) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index abf332fccc..38b3839afd 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -39,4 +39,4 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in toml==0.10.1 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in -urllib3==1.25.10 # via requests +urllib3==1.25.11 # via requests From 7bc9dc2764283f4d03a4954851a2c98c789d1404 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 05:39:52 +0000 Subject: [PATCH 0482/1498] Bump toml from 0.10.1 to 0.10.2 Bumps [toml](https://github.com/uiri/toml) from 0.10.1 to 0.10.2. - [Release notes](https://github.com/uiri/toml/releases) - [Changelog](https://github.com/uiri/toml/blob/master/RELEASE.rst) - [Commits](https://github.com/uiri/toml/commits) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 38b3839afd..8c1ab89b70 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -37,6 +37,6 @@ sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -toml==0.10.1 # via towncrier +toml==0.10.2 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in urllib3==1.25.11 # via requests diff --git a/test-requirements.txt b/test-requirements.txt index 5767ea27ec..cd0b8675ab 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -51,7 +51,7 @@ regex==2020.10.28 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.2.2 # via -r test-requirements.in -toml==0.10.1 # via black, pylint, pytest +toml==0.10.2 # via black, pylint, pytest traitlets==5.0.5 # via ipython trustme==0.6.0 # via -r test-requirements.in typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy From 28e8c036a58474f74cb8ad9583c9f1f335ba53bc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 05:40:32 +0000 Subject: [PATCH 0483/1498] Bump pytz from 2020.1 to 2020.4 Bumps [pytz](https://github.com/stub42/pytz) from 2020.1 to 2020.4. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2020.1...release_2020.4) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 38b3839afd..7d5cb9d84c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -22,7 +22,7 @@ outcome==1.0.1 # via -r docs-requirements.in packaging==20.4 # via sphinx pygments==2.7.2 # via sphinx pyparsing==2.4.7 # via packaging -pytz==2020.1 # via babel +pytz==2020.4 # via babel requests==2.24.0 # via sphinx six==1.15.0 # via packaging sniffio==1.2.0 # via -r docs-requirements.in From d6b075afbbaa318af73f270f54e64be8700a4f15 Mon Sep 17 00:00:00 2001 From: Kevin Tewouda Date: Tue, 3 Nov 2020 01:57:12 +0100 Subject: [PATCH 0484/1498] add pyscalpel to awesome trio libraries (#1788) Co-authored-by: le_woudar --- docs/source/awesome-trio-libraries.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 7ab0379447..e31ba608b5 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -34,6 +34,7 @@ Web and HTML * `httpx `__ - HTTPX is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. * `DeFramed `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi `__ GUI library. * `pura `__ - A simple web framework for embedding realtime graphical visualization into Trio apps, enabling inspection and manipulation of program state during development. +* `pyscalpel `__ - A fast and powerful webscraping library. Database From 697b009d0b8b5f2f63d59207589a8ee83b656136 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2020 05:22:33 +0000 Subject: [PATCH 0485/1498] Bump sphinx from 3.2.1 to 3.3.0 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.2.1 to 3.3.0. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.2.1...v3.3.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index d7cde4f916..3f023b5de6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -29,7 +29,7 @@ sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.2.2 # via -r docs-requirements.in sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in -sphinx==3.2.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx==3.3.0 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 6aa7e90c91e7472113dd1595abc701c21a98f7c3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2020 05:34:12 +0000 Subject: [PATCH 0486/1498] Bump attrs from 20.2.0 to 20.3.0 Bumps [attrs](https://github.com/python-attrs/attrs) from 20.2.0 to 20.3.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/master/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/20.2.0...20.3.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 3f023b5de6..103625a24b 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -6,7 +6,7 @@ # alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in -attrs==20.2.0 # via -r docs-requirements.in, outcome +attrs==20.3.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx certifi==2020.6.20 # via requests chardet==3.0.4 # via requests diff --git a/test-requirements.txt b/test-requirements.txt index cd0b8675ab..7a373112fc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in astroid==2.4.2 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==20.2.0 # via -r test-requirements.in, outcome, pytest +attrs==20.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.2.0 # via ipython black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.3 # via cryptography From 818af97d20b5e91038238c2e887d48ee614271b8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 05:43:46 +0000 Subject: [PATCH 0487/1498] Bump certifi from 2020.6.20 to 2020.11.8 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.6.20 to 2020.11.8. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.06.20...2020.11.08) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 103625a24b..6a6f52412f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in attrs==20.3.0 # via -r docs-requirements.in, outcome babel==2.8.0 # via sphinx -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests chardet==3.0.4 # via requests click==7.1.2 # via towncrier docutils==0.16 # via sphinx From cd874be0e2ae11a00553943c04f46a970a81b76f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 05:45:35 +0000 Subject: [PATCH 0488/1498] Bump pathspec from 0.8.0 to 0.8.1 Bumps [pathspec](https://github.com/cpburnz/python-path-specification) from 0.8.0 to 0.8.1. - [Release notes](https://github.com/cpburnz/python-path-specification/releases) - [Changelog](https://github.com/cpburnz/python-path-specification/blob/master/CHANGES.rst) - [Commits](https://github.com/cpburnz/python-path-specification/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7a373112fc..08c293208e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -31,7 +31,7 @@ mypy==0.790 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.0.1 # via -r test-requirements.in packaging==20.4 # via pytest parso==0.7.1 # via jedi -pathspec==0.8.0 # via black +pathspec==0.8.1 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest From 471ff6c9bb2e4981db8fcf30d86755d5cc5ebb35 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 05:46:25 +0000 Subject: [PATCH 0489/1498] Bump sortedcontainers from 2.2.2 to 2.3.0 Bumps [sortedcontainers](https://github.com/grantjenks/python-sortedcontainers) from 2.2.2 to 2.3.0. - [Release notes](https://github.com/grantjenks/python-sortedcontainers/releases) - [Changelog](https://github.com/grantjenks/python-sortedcontainers/blob/master/HISTORY.rst) - [Commits](https://github.com/grantjenks/python-sortedcontainers/compare/v2.2.2...v2.3.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 103625a24b..33921f167d 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -27,7 +27,7 @@ requests==2.24.0 # via sphinx six==1.15.0 # via packaging sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx -sortedcontainers==2.2.2 # via -r docs-requirements.in +sortedcontainers==2.3.0 # via -r docs-requirements.in sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in sphinx==3.3.0 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index 7a373112fc..a4bd483224 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -50,7 +50,7 @@ pytest==6.1.2 # via -r test-requirements.in, pytest-cov regex==2020.10.28 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in -sortedcontainers==2.2.2 # via -r test-requirements.in +sortedcontainers==2.3.0 # via -r test-requirements.in toml==0.10.2 # via black, pylint, pytest traitlets==5.0.5 # via ipython trustme==0.6.0 # via -r test-requirements.in From fbc9673e93e50f4eb2cc8a9587e0bd91e9ada6de Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Nov 2020 10:54:26 +0000 Subject: [PATCH 0490/1498] Synchronous .close() method for memory channels --- trio/_channel.py | 30 ++++++++++++++++++--- trio/tests/test_channel.py | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/trio/_channel.py b/trio/_channel.py index dac7935c0c..07a7f963d3 100644 --- a/trio/_channel.py +++ b/trio/_channel.py @@ -207,9 +207,16 @@ def clone(self): return MemorySendChannel._create(self._state) @enable_ki_protection - async def aclose(self): + def close(self): + """Close this send channel object synchronously. + + All channel objects have an asynchronous `~.AsyncResource.aclose` method. + Memory channels can also be closed synchronously. This has the same + effect on the channel and other tasks using it, but `close` is not a + trio checkpoint. This simplifies cleaning up in cancelled tasks. + + """ if self._closed: - await trio.lowlevel.checkpoint() return self._closed = True for task in self._tasks: @@ -223,6 +230,10 @@ async def aclose(self): task.custom_sleep_data._tasks.remove(task) trio.lowlevel.reschedule(task, Error(trio.EndOfChannel())) self._state.receive_tasks.clear() + + @enable_ki_protection + async def aclose(self): + self.close() await trio.lowlevel.checkpoint() @@ -326,9 +337,16 @@ def clone(self): return MemoryReceiveChannel._create(self._state) @enable_ki_protection - async def aclose(self): + def close(self): + """Close this receive channel object synchronously. + + All channel objects have an asynchronous `~.AsyncResource.aclose` method. + Memory channels can also be closed synchronously. This has the same + effect on the channel and other tasks using it, but `close` is not a + trio checkpoint. This simplifies cleaning up in cancelled tasks. + + """ if self._closed: - await trio.lowlevel.checkpoint() return self._closed = True for task in self._tasks: @@ -343,4 +361,8 @@ async def aclose(self): trio.lowlevel.reschedule(task, Error(trio.BrokenResourceError())) self._state.send_tasks.clear() self._state.data.clear() + + @enable_ki_protection + async def aclose(self): + self.close() await trio.lowlevel.checkpoint() diff --git a/trio/tests/test_channel.py b/trio/tests/test_channel.py index b43466dd7d..296aecc3f1 100644 --- a/trio/tests/test_channel.py +++ b/trio/tests/test_channel.py @@ -157,6 +157,61 @@ async def receive_block(r): await r.receive() +async def test_close_sync(): + async def send_block(s, expect): + with pytest.raises(expect): + await s.send(None) + + # closing send -> other send gets ClosedResourceError + s, r = open_memory_channel(0) + async with trio.open_nursery() as nursery: + nursery.start_soon(send_block, s, trio.ClosedResourceError) + await wait_all_tasks_blocked() + s.close() + + # and it's persistent + with pytest.raises(trio.ClosedResourceError): + s.send_nowait(None) + with pytest.raises(trio.ClosedResourceError): + await s.send(None) + + # and receive gets EndOfChannel + with pytest.raises(EndOfChannel): + r.receive_nowait() + with pytest.raises(EndOfChannel): + await r.receive() + + # closing receive -> send gets BrokenResourceError + s, r = open_memory_channel(0) + async with trio.open_nursery() as nursery: + nursery.start_soon(send_block, s, trio.BrokenResourceError) + await wait_all_tasks_blocked() + r.close() + + # and it's persistent + with pytest.raises(trio.BrokenResourceError): + s.send_nowait(None) + with pytest.raises(trio.BrokenResourceError): + await s.send(None) + + # closing receive -> other receive gets ClosedResourceError + async def receive_block(r): + with pytest.raises(trio.ClosedResourceError): + await r.receive() + + s, r = open_memory_channel(0) + async with trio.open_nursery() as nursery: + nursery.start_soon(receive_block, r) + await wait_all_tasks_blocked() + r.close() + + # and it's persistent + with pytest.raises(trio.ClosedResourceError): + r.receive_nowait() + with pytest.raises(trio.ClosedResourceError): + await r.receive() + + async def test_receive_channel_clone_and_close(): s, r = open_memory_channel(10) From bc52535697bdd732b8e16dd4dafdaee752782cb7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Nov 2020 11:11:58 +0000 Subject: [PATCH 0491/1498] Implement synchronous context manager protocol for memory channels --- trio/_channel.py | 18 ++++++++++++++++++ trio/tests/test_channel.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/trio/_channel.py b/trio/_channel.py index 07a7f963d3..1cecc55621 100644 --- a/trio/_channel.py +++ b/trio/_channel.py @@ -206,6 +206,12 @@ def clone(self): raise trio.ClosedResourceError return MemorySendChannel._create(self._state) + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + @enable_ki_protection def close(self): """Close this send channel object synchronously. @@ -215,6 +221,9 @@ def close(self): effect on the channel and other tasks using it, but `close` is not a trio checkpoint. This simplifies cleaning up in cancelled tasks. + Using ``with send_channel:`` will close the channel object on leaving + the with block. + """ if self._closed: return @@ -336,6 +345,12 @@ def clone(self): raise trio.ClosedResourceError return MemoryReceiveChannel._create(self._state) + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + @enable_ki_protection def close(self): """Close this receive channel object synchronously. @@ -345,6 +360,9 @@ def close(self): effect on the channel and other tasks using it, but `close` is not a trio checkpoint. This simplifies cleaning up in cancelled tasks. + Using ``with receive_channel:`` will close the channel object on + leaving the with block. + """ if self._closed: return diff --git a/trio/tests/test_channel.py b/trio/tests/test_channel.py index 296aecc3f1..5255c58801 100644 --- a/trio/tests/test_channel.py +++ b/trio/tests/test_channel.py @@ -285,7 +285,7 @@ async def test_inf_capacity(): s, r = open_memory_channel(float("inf")) # It's accepted, and we can send all day without blocking - async with s: + with s: for i in range(10): s.send_nowait(i) From 4ced3af7ee8d4742a6c25d6a5b3b8b25410ab440 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Nov 2020 11:29:41 +0000 Subject: [PATCH 0492/1498] Add news fragment --- newsfragments/1797.feature.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/1797.feature.rst diff --git a/newsfragments/1797.feature.rst b/newsfragments/1797.feature.rst new file mode 100644 index 0000000000..f71c36ae75 --- /dev/null +++ b/newsfragments/1797.feature.rst @@ -0,0 +1,2 @@ +Add synchronous ``.close()`` methods and context manager (``with x``) support +for `.MemorySendChannel` and `.MemoryReceiveChannel`. From 9ea8ca1ab91057defe6c1cac37d9401a6404997a Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Nov 2020 11:30:03 +0000 Subject: [PATCH 0493/1498] Cover MemoryReceiveChannel __enter__ and __exit__ in tests --- trio/tests/test_channel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_channel.py b/trio/tests/test_channel.py index 5255c58801..fd990fb3e3 100644 --- a/trio/tests/test_channel.py +++ b/trio/tests/test_channel.py @@ -220,7 +220,7 @@ async def test_receive_channel_clone_and_close(): s.send_nowait(None) await r.aclose() - async with r2: + with r2: pass with pytest.raises(trio.ClosedResourceError): From 4e3f1d29b054832d60e4c5e188dbe3e155e9c899 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2020 05:38:24 +0000 Subject: [PATCH 0494/1498] Bump babel from 2.8.0 to 2.8.1 Bumps [babel](https://github.com/python-babel/babel) from 2.8.0 to 2.8.1. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES) - [Commits](https://github.com/python-babel/babel/compare/v2.8.0...v2.8.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index fa43c883f9..ac69f4fb3e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -7,7 +7,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in attrs==20.3.0 # via -r docs-requirements.in, outcome -babel==2.8.0 # via sphinx +babel==2.8.1 # via sphinx certifi==2020.11.8 # via requests chardet==3.0.4 # via requests click==7.1.2 # via towncrier From 11f14237ce2a7d0c87e65fbcf12531b7ecfaf024 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2020 05:38:57 +0000 Subject: [PATCH 0495/1498] Bump regex from 2020.10.28 to 2020.11.11 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.10.28 to 2020.11.11. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c30087736c..191c81c97f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.2 # via -r test-requirements.in, pytest-cov -regex==2020.10.28 # via black +regex==2020.11.11 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.3.0 # via -r test-requirements.in From 49ca9e800386ba6cf0063cb95472a33f0e52c300 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2020 05:55:45 +0000 Subject: [PATCH 0496/1498] Bump requests from 2.24.0 to 2.25.0 Bumps [requests](https://github.com/psf/requests) from 2.24.0 to 2.25.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.24.0...v2.25.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index ac69f4fb3e..7724bc8456 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -23,7 +23,7 @@ packaging==20.4 # via sphinx pygments==2.7.2 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.4 # via babel -requests==2.24.0 # via sphinx +requests==2.25.0 # via sphinx six==1.15.0 # via packaging sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx From 9970eb39a834608622fa85fca649e078e888efab Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2020 05:22:58 +0000 Subject: [PATCH 0497/1498] Bump regex from 2020.11.11 to 2020.11.13 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.11.11 to 2020.11.13. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 191c81c97f..0bf484ee20 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ pyopenssl==19.1.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.2 # via -r test-requirements.in, pytest-cov -regex==2020.11.11 # via black +regex==2020.11.13 # via black six==1.15.0 # via astroid, cryptography, packaging, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.3.0 # via -r test-requirements.in From 37495d2a2d93c3ac05a5bb82a16ce6c4981e54be Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2020 05:23:49 +0000 Subject: [PATCH 0498/1498] Bump babel from 2.8.1 to 2.9.0 Bumps [babel](https://github.com/python-babel/babel) from 2.8.1 to 2.9.0. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES) - [Commits](https://github.com/python-babel/babel/commits) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 7724bc8456..11bd46855e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -7,7 +7,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in attrs==20.3.0 # via -r docs-requirements.in, outcome -babel==2.8.1 # via sphinx +babel==2.9.0 # via sphinx certifi==2020.11.8 # via requests chardet==3.0.4 # via requests click==7.1.2 # via towncrier From 25feca2a31256681fa510a2f857dab354871e562 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2020 05:24:52 +0000 Subject: [PATCH 0499/1498] Bump urllib3 from 1.25.11 to 1.26.2 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.25.11 to 1.26.2. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/master/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.25.11...1.26.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 7724bc8456..c45e0c678a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -39,4 +39,4 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in toml==0.10.2 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in -urllib3==1.25.11 # via requests +urllib3==1.26.2 # via requests From 3a8e06fc124dc9dba7ca87f31e434605a90065fe Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2020 05:51:28 +0000 Subject: [PATCH 0500/1498] Bump sphinx from 3.3.0 to 3.3.1 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.3.0...v3.3.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 93b0d753b5..3e7b21aea3 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -29,7 +29,7 @@ sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.3.0 # via -r docs-requirements.in sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in -sphinx==3.3.0 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx==3.3.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 58132b4149bccacafdca3760d749989ca0d45cfd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2020 05:26:03 +0000 Subject: [PATCH 0501/1498] Bump outcome from 1.0.1 to 1.1.0 Bumps [outcome](https://github.com/python-trio/outcome) from 1.0.1 to 1.1.0. - [Release notes](https://github.com/python-trio/outcome/releases) - [Commits](https://github.com/python-trio/outcome/compare/v1.0.1...v1.1.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 3e7b21aea3..81d8fb9211 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,7 +18,7 @@ immutables==0.14 # via -r docs-requirements.in incremental==17.5.0 # via towncrier jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 -outcome==1.0.1 # via -r docs-requirements.in +outcome==1.1.0 # via -r docs-requirements.in packaging==20.4 # via sphinx pygments==2.7.2 # via sphinx pyparsing==2.4.7 # via packaging diff --git a/test-requirements.txt b/test-requirements.txt index 0bf484ee20..c0e9b0af49 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -28,7 +28,7 @@ lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy mypy==0.790 ; implementation_name == "cpython" # via -r test-requirements.in -outcome==1.0.1 # via -r test-requirements.in +outcome==1.1.0 # via -r test-requirements.in packaging==20.4 # via pytest parso==0.7.1 # via jedi pathspec==0.8.1 # via black From 5229abb1be5b71912b4eebbe77435cbd4c14c494 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Nov 2020 05:41:47 +0000 Subject: [PATCH 0502/1498] Bump pyopenssl from 19.1.0 to 20.0.0 Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 19.1.0 to 20.0.0. - [Release notes](https://github.com/pyca/pyopenssl/releases) - [Changelog](https://github.com/pyca/pyopenssl/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/pyopenssl/compare/19.1.0...20.0.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c0e9b0af49..8defe3509c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,7 +43,7 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 pygments==2.7.2 # via ipython pylint==2.6.0 # via -r test-requirements.in -pyopenssl==19.1.0 # via -r test-requirements.in +pyopenssl==20.0.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.2 # via -r test-requirements.in, pytest-cov From c3a7053661f98f5a50fafa8dcfcd94b5f394c68f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Nov 2020 05:51:14 +0000 Subject: [PATCH 0503/1498] Bump cffi from 1.14.3 to 1.14.4 Bumps [cffi](https://github.com/python-cffi/release-doc) from 1.14.3 to 1.14.4. - [Release notes](https://github.com/python-cffi/release-doc/releases) - [Commits](https://github.com/python-cffi/release-doc/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8defe3509c..898e9716fd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,7 @@ async-generator==1.10 # via -r test-requirements.in attrs==20.3.0 # via -r test-requirements.in, outcome, pytest backcall==0.2.0 # via ipython black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.3 # via cryptography +cffi==1.14.4 # via cryptography click==7.1.2 # via black coverage==5.3 # via pytest-cov cryptography==3.2.1 # via pyopenssl, trustme From 9cfd4b91e9014396b4b7dd4890610a33c156a0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 5 Dec 2020 23:04:11 +0700 Subject: [PATCH 0504/1498] Retire obsolete distros on GitHub Actions & Travis --- .github/workflows/ci.yml | 18 --- .travis.yml | 22 --- ci.sh | 322 --------------------------------------- 3 files changed, 362 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afc612c301..6c1cd9d061 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,21 +100,3 @@ jobs: env: # Should match 'name:' up above JOB_NAME: 'macOS (${{ matrix.python }})' - - Alpine: - name: 'Alpine' - timeout-minutes: 10 - runs-on: 'ubuntu-latest' - container: 'alpine' - strategy: - fail-fast: false - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install bash - run: apk add --no-cache bash - - name: Run tests - run: ./ci.sh - env: - # Should match 'name:' up above - JOB_NAME: 'Alpine' diff --git a/.travis.yml b/.travis.yml index 911834d76d..43e24c8b9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,28 +11,6 @@ jobs: env: PYPY_NIGHTLY_BRANCH=py3.6 - language: generic env: PYPY_NIGHTLY_BRANCH=py3.7 - # Qemu tests are also slow - # FreeBSD: - - language: generic - env: - - "JOB_NAME='FreeBSD 12.1-RELEASE, full VM'" - - "FREEBSD_INSTALLER_ISO_XZ=https://download.freebsd.org/ftp/releases/amd64/amd64/ISO-IMAGES/12.1/FreeBSD-12.1-RELEASE-amd64-disc1.iso.xz" - # Increment this each time you change the image build code in - # ci.sh and want to intentionally bust the cache. - - "CACHE_GEN=3" - cache: - directories: - - travis-cache - # More recent Ubuntu: - # The unique thing this provides is testing on the given distro's - # kernel, which is important when we use new kernel features. This - # is also good for testing the latest openssl etc., and getting - # early warning of any issues that might happen in the next Ubuntu - # LTS. - - language: generic - env: - - "JOB_NAME='Fedora 32, full VM'" - - "LINUX_VM_IMAGE=https://download.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-1.6.x86_64.qcow2" - python: 3.6.1 # earliest 3.6 version available on Travis # https://github.com/pypa/setuptools/issues/2350 diff --git a/ci.sh b/ci.sh index 1386e15280..c48062a501 100755 --- a/ci.sh +++ b/ci.sh @@ -29,14 +29,6 @@ function curl-harder() { # Bootstrap python environment, if necessary ################################################################ -### Alpine ### - -if [ -e /etc/alpine-release ]; then - apk add --no-cache gcc musl-dev libffi-dev openssl-dev python3-dev curl - python3 -m venv venv - source venv/bin/activate -fi - ### PyPy nightly (currently on Travis) ### if [ "$PYPY_NIGHTLY_BRANCH" != "" ]; then @@ -67,320 +59,6 @@ if [ "$PYPY_NIGHTLY_BRANCH" != "" ]; then source testenv/bin/activate fi -### FreeBSD-in-Qemu virtual-machine inception, on Travis - -# This is complex, because none of the pre-made images are set up to be -# controlled by an automatic process – making them do anything requires a -# human to look at the screen and type stuff. So we hack up an install CD, run -# it to build our own custom image, and use that. But that's slow and we don't -# want to do it every run, so we try to get Travis to cache the image for us. -# -# Additional subtlety: we actually re-use the same image in-place, and let -# Travis re-cache it every time. The point of this is that it saves our system -# packages + pip cache, so we don't have to re-fetch them from scratch every -# time. (In particular, this means that the first time we install a given -# version of cryptography, we have to build it from source, but on subsequent -# runs we'll have a pre-built wheel sitting in our disk image.) -# -# You can run this locally for testing. But some things to watch out for: -# -# - It'll modify /etc/exports on your host machine. You might want to clean -# it up after. -# - It'll create a symlink at /host-files on your host machine. You might want -# to clean it up after. -# - If you don't want to keep downloading the installer ISO over and over, -# then drop an unpacked copy at ./local-freebsd-installer.iso - -if [ "$FREEBSD_INSTALLER_ISO_XZ" != "" ]; then - sudo apt update - sudo apt install qemu-system-x86 qemu-utils nfs-kernel-server - - if [ ! -e travis-cache/image.qcow2 ]; then - echo "--- No cached FreeBSD VM image; recreating from scratch ---" - sudo apt install growisofs genisoimage - rm -rf scratch - mkdir scratch - qemu-img create -f qcow2 scratch/image.qcow2 10G - if [ -e local-freebsd-installer.iso ]; then - cp local-freebsd-installer.iso scratch/installer.iso - else - curl-harder "$FREEBSD_INSTALLER_ISO_XZ" -o scratch/installer.iso.xz - unxz scratch/installer.iso.xz - fi - - # Files that we want to add to the ISO, to convert it into an - # unattended-installer: - mkdir -p scratch/overlay/etc - mkdir -p scratch/overlay/boot - # Use serial console, and disable the normal 10 second pause before - # booting - cat >scratch/overlay/boot/loader.conf.local <scratch/overlay/etc/rc.local <scratch/overlay/etc/installerconfig <> /etc/rc.conf -/bin/cp /usr/share/zoneinfo/UTC /etc/localtime - -echo 'autoboot_delay=0' >> /boot/loader.conf.local -echo 'console=comconsole' >> /boot/loader.conf.local - -dhclient em0 -export ASSUME_ALWAYS_YES=true -pkg install bash curl python38 py38-sqlite3 - -mkdir /host-files -cat >>/etc/rc.local <freebsd-wrapper.sh < Date: Sat, 5 Dec 2020 22:22:30 +0700 Subject: [PATCH 0505/1498] Employ builds.sr.ht for Alpine, Fedora and FreeBSD --- .builds/alpine.yml | 19 +++++++++++++++++++ .builds/fedora.yml | 14 ++++++++++++++ .builds/freebsd.yml | 17 +++++++++++++++++ ci.sh | 1 + 4 files changed, 51 insertions(+) create mode 100644 .builds/alpine.yml create mode 100644 .builds/fedora.yml create mode 100644 .builds/freebsd.yml diff --git a/.builds/alpine.yml b/.builds/alpine.yml new file mode 100644 index 0000000000..f69280d852 --- /dev/null +++ b/.builds/alpine.yml @@ -0,0 +1,19 @@ +image: alpine/latest +packages: + - curl + - gcc + - libffi-dev + - musl-dev + - openssl-dev + - python3-dev +sources: + - https://github.com/python-trio/trio +tasks: + - test: | + python3 -m venv venv + source venv/bin/activate + cd trio + CI_BUILD_ID=$JOB_ID CI_BUILD_URL=$JOB_URL ./ci.sh +environment: + CODECOV_TOKEN: 87cefb17-c44b-4f2f-8b30-1fff5769ce46 + JOB_NAME: Alpine diff --git a/.builds/fedora.yml b/.builds/fedora.yml new file mode 100644 index 0000000000..133cf3d40f --- /dev/null +++ b/.builds/fedora.yml @@ -0,0 +1,14 @@ +image: fedora/rawhide +packages: + - python3-pip +sources: + - https://github.com/python-trio/trio +tasks: + - test: | + python3 -m venv venv + source venv/bin/activate + cd trio + CI_BUILD_ID=$JOB_ID CI_BUILD_URL=$JOB_URL ./ci.sh +environment: + CODECOV_TOKEN: 87cefb17-c44b-4f2f-8b30-1fff5769ce46 + JOB_NAME: Fedora diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml new file mode 100644 index 0000000000..34e95cb099 --- /dev/null +++ b/.builds/freebsd.yml @@ -0,0 +1,17 @@ +image: freebsd/latest +packages: + - curl + - python38 + - py38-sqlite3 +sources: + - https://github.com/python-trio/trio +tasks: + - setup: sudo ln -s /usr/local/bin/bash /bin/bash + - test: | + python3.8 -m venv venv + source venv/bin/activate + cd trio + CI_BUILD_ID=$JOB_ID CI_BUILD_URL=$JOB_URL ./ci.sh +environment: + CODECOV_TOKEN: 87cefb17-c44b-4f2f-8b30-1fff5769ce46 + JOB_NAME: FreeBSD diff --git a/ci.sh b/ci.sh index c48062a501..430bfcff00 100755 --- a/ci.sh +++ b/ci.sh @@ -3,6 +3,7 @@ set -ex -o pipefail # Log some general info about the environment +uname -a env | sort if [ "$JOB_NAME" = "" ]; then From abab7cabc661414905ff27bbd8103b8bfff4dc2e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Dec 2020 05:44:29 +0000 Subject: [PATCH 0506/1498] Bump certifi from 2020.11.8 to 2020.12.5 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.11.8 to 2020.12.5. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.11.08...2020.12.05) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 81d8fb9211..2bc2c23da9 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in attrs==20.3.0 # via -r docs-requirements.in, outcome babel==2.9.0 # via sphinx -certifi==2020.11.8 # via requests +certifi==2020.12.5 # via requests chardet==3.0.4 # via requests click==7.1.2 # via towncrier docutils==0.16 # via sphinx From 8f7d4af30ea1ef39d083cd5eecd987381d0a23ed Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 7 Dec 2020 05:45:36 +0000 Subject: [PATCH 0507/1498] Bump pygments from 2.7.2 to 2.7.3 Bumps [pygments](https://github.com/pygments/pygments) from 2.7.2 to 2.7.3. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.7.2...2.7.3) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 81d8fb9211..88319484c1 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -20,7 +20,7 @@ jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in packaging==20.4 # via sphinx -pygments==2.7.2 # via sphinx +pygments==2.7.3 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.4 # via babel requests==2.25.0 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index 898e9716fd..5d64b23295 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ py==1.9.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 -pygments==2.7.2 # via ipython +pygments==2.7.3 # via ipython pylint==2.6.0 # via -r test-requirements.in pyopenssl==20.0.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging From df7c829384ffc5a6718130949c1031ea7006dcb7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 8 Dec 2020 10:28:53 +0000 Subject: [PATCH 0508/1498] Bump packaging from 20.4 to 20.7 Bumps [packaging](https://github.com/pypa/packaging) from 20.4 to 20.7. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/20.4...20.7) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 3 +-- test-requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 5324c7fb6e..fee2256b6f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -19,12 +19,11 @@ incremental==17.5.0 # via towncrier jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in -packaging==20.4 # via sphinx +packaging==20.7 # via sphinx pygments==2.7.3 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.4 # via babel requests==2.25.0 # via sphinx -six==1.15.0 # via packaging sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.3.0 # via -r docs-requirements.in diff --git a/test-requirements.txt b/test-requirements.txt index 5d64b23295..0c9245e1cb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -29,7 +29,7 @@ mccabe==0.6.1 # via flake8, pylint mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy mypy==0.790 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in -packaging==20.4 # via pytest +packaging==20.7 # via pytest parso==0.7.1 # via jedi pathspec==0.8.1 # via black pexpect==4.8.0 # via ipython @@ -48,7 +48,7 @@ pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.2 # via -r test-requirements.in, pytest-cov regex==2020.11.13 # via black -six==1.15.0 # via astroid, cryptography, packaging, pyopenssl +six==1.15.0 # via astroid, cryptography, pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.3.0 # via -r test-requirements.in toml==0.10.2 # via black, pylint, pytest From b3d4f1d7ce71a013dcebe80afe53e51fb9dcda04 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 9 Dec 2020 05:32:24 +0000 Subject: [PATCH 0509/1498] Bump cryptography from 3.2.1 to 3.3 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.2.1 to 3.3. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.2.1...3.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 0c9245e1cb..cb2deed0cd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,7 +14,7 @@ black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.4 # via cryptography click==7.1.2 # via black coverage==5.3 # via pytest-cov -cryptography==3.2.1 # via pyopenssl, trustme +cryptography==3.3 # via pyopenssl, trustme decorator==4.4.2 # via ipython flake8==3.8.4 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme From 3078166644bf602251df4b703d2901f67bba1891 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 10 Dec 2020 05:23:10 +0000 Subject: [PATCH 0510/1498] Bump cryptography from 3.3 to 3.3.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.3 to 3.3.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.3...3.3.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index cb2deed0cd..8cc2125abe 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,7 +14,7 @@ black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.4 # via cryptography click==7.1.2 # via black coverage==5.3 # via pytest-cov -cryptography==3.3 # via pyopenssl, trustme +cryptography==3.3.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython flake8==3.8.4 # via -r test-requirements.in idna==2.10 # via -r test-requirements.in, trustme From 317e11bc958b636c60b7f85e6e894d48199614c1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Dec 2020 06:12:43 +0000 Subject: [PATCH 0511/1498] Bump py from 1.9.0 to 1.10.0 Bumps [py](https://github.com/pytest-dev/py) from 1.9.0 to 1.10.0. - [Release notes](https://github.com/pytest-dev/py/releases) - [Changelog](https://github.com/pytest-dev/py/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/py/compare/1.9.0...1.10.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8cc2125abe..9bd4f82238 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -37,7 +37,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest prompt-toolkit==3.0.8 # via ipython ptyprocess==0.6.0 # via pexpect -py==1.9.0 # via pytest +py==1.10.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 From af847749417aaa9d4614502b077195cc5108b54c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 16 Dec 2020 05:38:54 +0000 Subject: [PATCH 0512/1498] Bump pyopenssl from 20.0.0 to 20.0.1 Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 20.0.0 to 20.0.1. - [Release notes](https://github.com/pyca/pyopenssl/releases) - [Changelog](https://github.com/pyca/pyopenssl/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/pyopenssl/compare/20.0.0...20.0.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9bd4f82238..2f4363a942 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,7 +43,7 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 pygments==2.7.3 # via ipython pylint==2.6.0 # via -r test-requirements.in -pyopenssl==20.0.0 # via -r test-requirements.in +pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging pytest-cov==2.10.1 # via -r test-requirements.in pytest==6.1.2 # via -r test-requirements.in, pytest-cov From 8dcb3b4ba0650bfe80ec4bf75edc57aae0b308c8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 16 Dec 2020 05:59:32 +0000 Subject: [PATCH 0513/1498] Bump packaging from 20.7 to 20.8 Bumps [packaging](https://github.com/pypa/packaging) from 20.7 to 20.8. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/20.7...20.8) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index fee2256b6f..da896bd9d6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -19,7 +19,7 @@ incremental==17.5.0 # via towncrier jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in -packaging==20.7 # via sphinx +packaging==20.8 # via sphinx pygments==2.7.3 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.4 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index 2f4363a942..47dc51635f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -29,7 +29,7 @@ mccabe==0.6.1 # via flake8, pylint mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy mypy==0.790 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in -packaging==20.7 # via pytest +packaging==20.8 # via pytest parso==0.7.1 # via jedi pathspec==0.8.1 # via black pexpect==4.8.0 # via ipython From 560d1744b4cc8ba1ee3f8e3a18ecdb8e4cc12cf7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 17 Dec 2020 05:21:14 +0000 Subject: [PATCH 0514/1498] Bump requests from 2.25.0 to 2.25.1 Bumps [requests](https://github.com/psf/requests) from 2.25.0 to 2.25.1. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.25.0...v2.25.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index da896bd9d6..1a93b16c4e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -23,7 +23,7 @@ packaging==20.8 # via sphinx pygments==2.7.3 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.4 # via babel -requests==2.25.0 # via sphinx +requests==2.25.1 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.3.0 # via -r docs-requirements.in From 84c81eb53ce56ba9a7c2b42680a3f89594a5961a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 18 Dec 2020 05:28:30 +0000 Subject: [PATCH 0515/1498] Bump chardet from 3.0.4 to 4.0.0 Bumps [chardet](https://github.com/chardet/chardet) from 3.0.4 to 4.0.0. - [Release notes](https://github.com/chardet/chardet/releases) - [Commits](https://github.com/chardet/chardet/compare/3.0.4...4.0.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 1a93b16c4e..66d9f048e2 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -9,7 +9,7 @@ async-generator==1.10 # via -r docs-requirements.in attrs==20.3.0 # via -r docs-requirements.in, outcome babel==2.9.0 # via sphinx certifi==2020.12.5 # via requests -chardet==3.0.4 # via requests +chardet==4.0.0 # via requests click==7.1.2 # via towncrier docutils==0.16 # via sphinx idna==2.10 # via -r docs-requirements.in, requests From 1bdbfba0fa217d91b70cc722d1bda6138c358797 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 18 Dec 2020 09:32:04 +0400 Subject: [PATCH 0516/1498] Test the actual 3.9 now that it's out --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c1cd9d061..2f67c7e61b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9.0-rc.2'] + python: ['3.6', '3.7', '3.8', '3.9'] arch: ['x86', 'x64'] lsp: [''] lsp_extract_file: [''] @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9.0-rc.2'] + python: ['3.6', '3.7', '3.8', '3.9'] check_formatting: ['0'] extra_name: [''] include: @@ -87,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9.0-rc.2'] + python: ['3.6', '3.7', '3.8', '3.9'] steps: - name: Checkout uses: actions/checkout@v2 From e5d9c42fb8df8bcdfffa58da92be17b263f9b9c2 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 18 Dec 2020 09:34:18 +0400 Subject: [PATCH 0517/1498] Test dev Python in GitHub Actions We also restore 3.10-dev now that PEP 641 was rejected. --- .github/workflows/ci.yml | 8 +++++++- .travis.yml | 7 ------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f67c7e61b..0fde947ed0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9'] + python: ['3.6', '3.7', '3.8', '3.9', '3.6-dev', '3.7-dev', '3.8-dev', '3.9-dev'] check_formatting: ['0'] extra_name: [''] include: @@ -71,6 +71,12 @@ jobs: uses: actions/checkout@v2 - name: Setup python uses: actions/setup-python@v2 + if: "!endsWith(matrix.python, '-dev')" + with: + python-version: '${{ matrix.python }}' + - name: Setup python (dev) + uses: deadsnakes/action@v2.0.2 + if: endsWith(matrix.python, '-dev') with: python-version: '${{ matrix.python }}' - name: Run tests diff --git a/.travis.yml b/.travis.yml index 43e24c8b9e..d04994cf2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,13 +16,6 @@ jobs: # https://github.com/pypa/setuptools/issues/2350 env: SETUPTOOLS_USE_DISTUTILS=stdlib dist: bionic - - python: 3.6-dev - - python: 3.7-dev - - python: 3.8-dev - - python: 3.9-dev - # Temporarily disabled during the high-churn period of 3.10 - # E.g.: https://github.com/MagicStack/immutables/issues/46 - #- python: nightly script: - ./ci.sh From 71288a5058bcdba59c63be52a7c89e242fe8f12c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 21 Dec 2020 05:39:47 +0000 Subject: [PATCH 0518/1498] Bump coverage from 5.3 to 5.3.1 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.3 to 5.3.1. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.3...coverage-5.3.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 47dc51635f..afb423e2ea 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,7 +13,7 @@ backcall==0.2.0 # via ipython black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.4 # via cryptography click==7.1.2 # via black -coverage==5.3 # via pytest-cov +coverage==5.3.1 # via pytest-cov cryptography==3.3.1 # via pyopenssl, trustme decorator==4.4.2 # via ipython flake8==3.8.4 # via -r test-requirements.in From 3c4173010cf762fb49046b73e9414cf2ec9513a8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 25 Dec 2020 05:33:14 +0000 Subject: [PATCH 0519/1498] Bump pytz from 2020.4 to 2020.5 Bumps [pytz](https://github.com/stub42/pytz) from 2020.4 to 2020.5. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2020.4...release_2020.5) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 66d9f048e2..c50dbaf08a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -22,7 +22,7 @@ outcome==1.1.0 # via -r docs-requirements.in packaging==20.8 # via sphinx pygments==2.7.3 # via sphinx pyparsing==2.4.7 # via packaging -pytz==2020.4 # via babel +pytz==2020.5 # via babel requests==2.25.1 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx From 4827c52a4f382331bb2f25904d9ef0c8e78b9bda Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 27 Dec 2020 22:11:52 -0800 Subject: [PATCH 0520/1498] Bump freebsd CI to test on python 3.9 Apparently py38 isn't available in the package repos anymore on our test machines. --- .builds/freebsd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 34e95cb099..0459ed8f71 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -1,14 +1,14 @@ image: freebsd/latest packages: - curl - - python38 - - py38-sqlite3 + - python39 + - py39-sqlite3 sources: - https://github.com/python-trio/trio tasks: - setup: sudo ln -s /usr/local/bin/bash /bin/bash - test: | - python3.8 -m venv venv + python3.9 -m venv venv source venv/bin/activate cd trio CI_BUILD_ID=$JOB_ID CI_BUILD_URL=$JOB_URL ./ci.sh From 8bf1686b90dcfd8133d11b7c840e123f0d1547a7 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Wed, 23 Dec 2020 17:02:09 -0500 Subject: [PATCH 0521/1498] avoid cubic AFD cancellation scaling with AFDGroups --- notes-to-self/socket-scaling.py | 8 ---- trio/_core/_io_windows.py | 66 ++++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/notes-to-self/socket-scaling.py b/notes-to-self/socket-scaling.py index 1a18141e17..1571be4d17 100644 --- a/notes-to-self/socket-scaling.py +++ b/notes-to-self/socket-scaling.py @@ -11,14 +11,6 @@ # On Windows: with the old 'select'-based loop, the cost of scheduling grew # with the number of outstanding sockets, which was bad. # -# With the new IOCP-based loop, the cost of scheduling is constant, which is -# good. But, we find that the cost of cancelling a single wait_readable -# appears to grow like O(n**2) or so in the number of outstanding -# wait_readables. This is bad -- it means that cancelling all of the -# outstanding operations here is something like O(n**3)! To avoid this, we -# should consider creating multiple AFD helper handles and distributing the -# AFD_POLL operations across them. -# # To run this on Unix systems, you'll probably first have to run: # # ulimit -n 31000 diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index e7f2d08a4b..f82bed81c4 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import attr +from sortedcontainers import SortedList from .. import _core from ._run import _public @@ -343,6 +344,24 @@ class AFDPollOp: lpOverlapped = attr.ib() poll_info = attr.ib() waiters = attr.ib() + afd_group = attr.ib() + + +# With the new IOCP-based loop, the cost of scheduling is constant, which is +# good. But, we find that the cost of cancelling a single wait_readable +# appears to grow like O(n**2) or so in the number of outstanding +# wait_readables. This is bad -- it means that cancelling all of the +# outstanding operations here is something like O(n**3)! To avoid this, we +# multiple AFD helper handles for each set of MAX_AFD_GROUP_SIZE and distribute +# AFD_POLL operations across them. +MAX_AFD_GROUP_SIZE = 500 # at 1000, the cubic scaling is just starting to bite + + +@attr.s(slots=True) +class AFDGroup: + size = attr.ib() + afd_id = attr.ib() + afd_handle = attr.ib() @attr.s(slots=True, eq=False, frozen=True) @@ -374,15 +393,14 @@ def __init__(self): # touches to safe values up front, before we do anything that can # fail. self._iocp = None - self._afd = None + self._sorted_afd_groups = None self._iocp = _check( kernel32.CreateIoCompletionPort(INVALID_HANDLE_VALUE, ffi.NULL, 0, 0) ) self._events = ffi.new("OVERLAPPED_ENTRY[]", MAX_EVENTS) - self._afd = _afd_helper_handle() - self._register_with_iocp(self._afd, CKeys.AFD_POLL) + self._sorted_afd_groups = SortedList() # {lpOverlapped: AFDPollOp} self._afd_ops = {} # {socket handle: AFDWaiters} @@ -439,10 +457,11 @@ def close(self): self._iocp = None _check(kernel32.CloseHandle(iocp)) finally: - if self._afd is not None: - afd = self._afd - self._afd = None - _check(kernel32.CloseHandle(afd)) + if self._sorted_afd_groups is not None: + afd_groups = self._sorted_afd_groups + self._sorted_afd_groups = None + for afd_group in afd_groups: + _check(kernel32.CloseHandle(afd_group.afd_handle)) def __del__(self): self.close() @@ -581,14 +600,25 @@ def _register_with_iocp(self, handle, completion_key): def _refresh_afd(self, base_handle): waiters = self._afd_waiters[base_handle] if waiters.current_op is not None: + afd_group = waiters.current_op.afd_group try: - _check(kernel32.CancelIoEx(self._afd, waiters.current_op.lpOverlapped)) + _check( + kernel32.CancelIoEx( + afd_group.afd_handle, waiters.current_op.lpOverlapped + ) + ) except OSError as exc: if exc.winerror != ErrorCodes.ERROR_NOT_FOUND: # I don't think this is possible, so if it happens let's # crash noisily. raise # pragma: no cover waiters.current_op = None + self._sorted_afd_groups.discard(afd_group) + afd_group.size -= 1 + if MAX_AFD_GROUP_SIZE > afd_group.size > 0: + self._sorted_afd_groups.add(afd_group) + else: + _check(kernel32.CloseHandle(afd_group.afd_handle)) flags = 0 if waiters.read_task is not None: @@ -599,6 +629,17 @@ def _refresh_afd(self, base_handle): if not flags: del self._afd_waiters[base_handle] else: + + # get largest afd_group less than MAX_AFD_GROUP_SIZE + try: + afd_group = self._sorted_afd_groups.pop() + except IndexError: + afd = _afd_helper_handle() + self._register_with_iocp(afd, CKeys.AFD_POLL) + afd_group = AFDGroup(0, id(afd), afd) + else: + afd = afd_group.afd_handle + lpOverlapped = ffi.new("LPOVERLAPPED") poll_info = ffi.new("AFD_POLL_INFO *") @@ -612,7 +653,7 @@ def _refresh_afd(self, base_handle): try: _check( kernel32.DeviceIoControl( - self._afd, + afd, IoControlCodes.IOCTL_AFD_POLL, poll_info, ffi.sizeof("AFD_POLL_INFO"), @@ -629,12 +670,17 @@ def _refresh_afd(self, base_handle): # to re-issue the call. Clear our state and wake up any # pending calls. del self._afd_waiters[base_handle] + if afd_group.size == 0: + _check(kernel32.CloseHandle(afd)) # Do this last, because it could raise. wake_all(waiters, exc) return - op = AFDPollOp(lpOverlapped, poll_info, waiters) + op = AFDPollOp(lpOverlapped, poll_info, waiters, afd_group) waiters.current_op = op self._afd_ops[lpOverlapped] = op + afd_group.size += 1 + if MAX_AFD_GROUP_SIZE > afd_group.size: + self._sorted_afd_groups.add(afd_group) async def _afd_poll(self, sock, mode): base_handle = _get_base_socket(sock) From 34806a35e6812d25534911a01aeea5552bfee418 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Thu, 24 Dec 2020 08:36:17 -0500 Subject: [PATCH 0522/1498] Add newsfragment --- newsfragments/1832.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1832.bugfix.rst diff --git a/newsfragments/1832.bugfix.rst b/newsfragments/1832.bugfix.rst new file mode 100644 index 0000000000..ed4ba47b7b --- /dev/null +++ b/newsfragments/1832.bugfix.rst @@ -0,0 +1 @@ +Avoid O(n**3) scaling of windows socket cancellation \ No newline at end of file From 27a0efb927aa930680d54c4e40bd464fecbd7062 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Thu, 24 Dec 2020 09:00:31 -0500 Subject: [PATCH 0523/1498] test full afd groups --- trio/tests/test_socket.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 3124c20d4a..7c150a508e 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -990,3 +990,20 @@ async def receiver(): nursery.start_soon(receiver) await wait_all_tasks_blocked() a.close() + + +async def test_many_sockets(): + total = 5000 # Must be more than MAX_AFD_GROUP_SIZE + sockets = [] + for _ in range(total // 2): + a, b = stdlib_socket.socketpair() + sockets += [a, b] + async with _core.open_nursery() as nursery: + for s in sockets: + nursery.start_soon(_core.wait_readable, s) + await _core.wait_all_tasks_blocked() + for _ in range(1000): + await _core.cancel_shielded_checkpoint() + nursery.cancel_scope.cancel() + for sock in sockets: + sock.close() From 9bcbcda58fea77fa16e601fddbbaf2bb1f54ddc0 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Thu, 24 Dec 2020 09:01:57 -0500 Subject: [PATCH 0524/1498] improve size comparisons --- trio/_core/_io_windows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index f82bed81c4..e397f7de2f 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -615,7 +615,7 @@ def _refresh_afd(self, base_handle): waiters.current_op = None self._sorted_afd_groups.discard(afd_group) afd_group.size -= 1 - if MAX_AFD_GROUP_SIZE > afd_group.size > 0: + if afd_group.size > 0: self._sorted_afd_groups.add(afd_group) else: _check(kernel32.CloseHandle(afd_group.afd_handle)) @@ -679,7 +679,7 @@ def _refresh_afd(self, base_handle): waiters.current_op = op self._afd_ops[lpOverlapped] = op afd_group.size += 1 - if MAX_AFD_GROUP_SIZE > afd_group.size: + if afd_group.size < MAX_AFD_GROUP_SIZE: self._sorted_afd_groups.add(afd_group) async def _afd_poll(self, sock, mode): From 7f5a2256dfac76c4c2652622d83fb81fd7752291 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Thu, 24 Dec 2020 09:55:48 -0500 Subject: [PATCH 0525/1498] be less strict about afd handle handling trade SortedList for set track all afd handles only close handles in .close() --- trio/_core/_io_windows.py | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index e397f7de2f..f54776b492 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import attr -from sortedcontainers import SortedList from .. import _core from ._run import _public @@ -357,10 +356,9 @@ class AFDPollOp: MAX_AFD_GROUP_SIZE = 500 # at 1000, the cubic scaling is just starting to bite -@attr.s(slots=True) +@attr.s(slots=True, hash=True) class AFDGroup: size = attr.ib() - afd_id = attr.ib() afd_handle = attr.ib() @@ -393,14 +391,15 @@ def __init__(self): # touches to safe values up front, before we do anything that can # fail. self._iocp = None - self._sorted_afd_groups = None + self._all_afd_handles = None self._iocp = _check( kernel32.CreateIoCompletionPort(INVALID_HANDLE_VALUE, ffi.NULL, 0, 0) ) self._events = ffi.new("OVERLAPPED_ENTRY[]", MAX_EVENTS) - self._sorted_afd_groups = SortedList() + self._all_afd_handles = [] + self._vacant_afd_groups = set() # {lpOverlapped: AFDPollOp} self._afd_ops = {} # {socket handle: AFDWaiters} @@ -457,11 +456,11 @@ def close(self): self._iocp = None _check(kernel32.CloseHandle(iocp)) finally: - if self._sorted_afd_groups is not None: - afd_groups = self._sorted_afd_groups - self._sorted_afd_groups = None - for afd_group in afd_groups: - _check(kernel32.CloseHandle(afd_group.afd_handle)) + if self._all_afd_handles is not None: + afd_handles = self._all_afd_handles + self._all_afd_handles = None + for afd_handle in afd_handles: + _check(kernel32.CloseHandle(afd_handle)) def __del__(self): self.close() @@ -613,12 +612,8 @@ def _refresh_afd(self, base_handle): # crash noisily. raise # pragma: no cover waiters.current_op = None - self._sorted_afd_groups.discard(afd_group) afd_group.size -= 1 - if afd_group.size > 0: - self._sorted_afd_groups.add(afd_group) - else: - _check(kernel32.CloseHandle(afd_group.afd_handle)) + self._vacant_afd_groups.add(afd_group) flags = 0 if waiters.read_task is not None: @@ -632,11 +627,12 @@ def _refresh_afd(self, base_handle): # get largest afd_group less than MAX_AFD_GROUP_SIZE try: - afd_group = self._sorted_afd_groups.pop() - except IndexError: + afd_group = self._vacant_afd_groups.pop() + except KeyError: afd = _afd_helper_handle() self._register_with_iocp(afd, CKeys.AFD_POLL) - afd_group = AFDGroup(0, id(afd), afd) + afd_group = AFDGroup(0, afd) + self._all_afd_handles.append(afd) else: afd = afd_group.afd_handle @@ -670,8 +666,6 @@ def _refresh_afd(self, base_handle): # to re-issue the call. Clear our state and wake up any # pending calls. del self._afd_waiters[base_handle] - if afd_group.size == 0: - _check(kernel32.CloseHandle(afd)) # Do this last, because it could raise. wake_all(waiters, exc) return @@ -680,7 +674,7 @@ def _refresh_afd(self, base_handle): self._afd_ops[lpOverlapped] = op afd_group.size += 1 if afd_group.size < MAX_AFD_GROUP_SIZE: - self._sorted_afd_groups.add(afd_group) + self._vacant_afd_groups.add(afd_group) async def _afd_poll(self, sock, mode): base_handle = _get_base_socket(sock) From 8dbe743041ca148691f9cb0128c372ec935be21f Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Fri, 25 Dec 2020 15:42:43 -0500 Subject: [PATCH 0526/1498] attrs quirk: hash by ID with eq=False --- trio/_core/_io_windows.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index f54776b492..baa8ad5e85 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -356,7 +356,7 @@ class AFDPollOp: MAX_AFD_GROUP_SIZE = 500 # at 1000, the cubic scaling is just starting to bite -@attr.s(slots=True, hash=True) +@attr.s(slots=True, eq=False) class AFDGroup: size = attr.ib() afd_handle = attr.ib() @@ -624,8 +624,6 @@ def _refresh_afd(self, base_handle): if not flags: del self._afd_waiters[base_handle] else: - - # get largest afd_group less than MAX_AFD_GROUP_SIZE try: afd_group = self._vacant_afd_groups.pop() except KeyError: From 986f69b3865bfda676c266ea7300583c36027674 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 26 Dec 2020 10:55:13 -0500 Subject: [PATCH 0527/1498] avoid temp variable for afd handle --- trio/_core/_io_windows.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index baa8ad5e85..08aaa77656 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -359,7 +359,7 @@ class AFDPollOp: @attr.s(slots=True, eq=False) class AFDGroup: size = attr.ib() - afd_handle = attr.ib() + handle = attr.ib() @attr.s(slots=True, eq=False, frozen=True) @@ -603,7 +603,7 @@ def _refresh_afd(self, base_handle): try: _check( kernel32.CancelIoEx( - afd_group.afd_handle, waiters.current_op.lpOverlapped + afd_group.handle, waiters.current_op.lpOverlapped ) ) except OSError as exc: @@ -627,12 +627,9 @@ def _refresh_afd(self, base_handle): try: afd_group = self._vacant_afd_groups.pop() except KeyError: - afd = _afd_helper_handle() - self._register_with_iocp(afd, CKeys.AFD_POLL) - afd_group = AFDGroup(0, afd) - self._all_afd_handles.append(afd) - else: - afd = afd_group.afd_handle + afd_group = AFDGroup(0, _afd_helper_handle()) + self._register_with_iocp(afd_group.handle, CKeys.AFD_POLL) + self._all_afd_handles.append(afd_group.handle) lpOverlapped = ffi.new("LPOVERLAPPED") @@ -647,7 +644,7 @@ def _refresh_afd(self, base_handle): try: _check( kernel32.DeviceIoControl( - afd, + afd_group.handle, IoControlCodes.IOCTL_AFD_POLL, poll_info, ffi.sizeof("AFD_POLL_INFO"), From 23b563bc7c0f00b1b10377ef150562ae769dea0c Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 26 Dec 2020 11:22:39 -0500 Subject: [PATCH 0528/1498] Clarify AFDGroup comment --- trio/_core/_io_windows.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 08aaa77656..a41ea8b135 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -346,13 +346,14 @@ class AFDPollOp: afd_group = attr.ib() -# With the new IOCP-based loop, the cost of scheduling is constant, which is -# good. But, we find that the cost of cancelling a single wait_readable -# appears to grow like O(n**2) or so in the number of outstanding -# wait_readables. This is bad -- it means that cancelling all of the -# outstanding operations here is something like O(n**3)! To avoid this, we -# multiple AFD helper handles for each set of MAX_AFD_GROUP_SIZE and distribute -# AFD_POLL operations across them. +# The Windows kernel has a weird issue when using AFD handles. If you have N +# instances of wait_readable/wait_writable registered with a single AFD handle, +# then cancelling any one of them takes something like O(N**2) time. So if we +# used just a single AFD handle, then cancellation would quickly become very +# expensive, e.g. a program with N active sockets would take something like +# O(N**3) time to unwind after control-C. The solution is to spread our sockets +# out over multiple AFD handles, so that N doesn't grow too large for any +# individual handle. MAX_AFD_GROUP_SIZE = 500 # at 1000, the cubic scaling is just starting to bite From ca8dea93ac17eca2728b114bb8f16483871505e1 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 26 Dec 2020 14:38:19 -0500 Subject: [PATCH 0529/1498] make _all_afd_handles list always exist --- trio/_core/_io_windows.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index a41ea8b135..b2a605c75a 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -392,14 +392,13 @@ def __init__(self): # touches to safe values up front, before we do anything that can # fail. self._iocp = None - self._all_afd_handles = None + self._all_afd_handles = [] self._iocp = _check( kernel32.CreateIoCompletionPort(INVALID_HANDLE_VALUE, ffi.NULL, 0, 0) ) self._events = ffi.new("OVERLAPPED_ENTRY[]", MAX_EVENTS) - self._all_afd_handles = [] self._vacant_afd_groups = set() # {lpOverlapped: AFDPollOp} self._afd_ops = {} @@ -457,11 +456,9 @@ def close(self): self._iocp = None _check(kernel32.CloseHandle(iocp)) finally: - if self._all_afd_handles is not None: - afd_handles = self._all_afd_handles - self._all_afd_handles = None - for afd_handle in afd_handles: - _check(kernel32.CloseHandle(afd_handle)) + while self._all_afd_handles: + afd_handle = self._all_afd_handles.pop() + _check(kernel32.CloseHandle(afd_handle)) def __del__(self): self.close() From 8196d0be99e863ea23f190a4b1306238f8316d3a Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 26 Dec 2020 14:50:30 -0500 Subject: [PATCH 0530/1498] keep AFDGroups in vacant set as long as possible --- trio/_core/_io_windows.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index b2a605c75a..c318d59629 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -628,6 +628,7 @@ def _refresh_afd(self, base_handle): afd_group = AFDGroup(0, _afd_helper_handle()) self._register_with_iocp(afd_group.handle, CKeys.AFD_POLL) self._all_afd_handles.append(afd_group.handle) + self._vacant_afd_groups.add(afd_group) lpOverlapped = ffi.new("LPOVERLAPPED") @@ -666,8 +667,8 @@ def _refresh_afd(self, base_handle): waiters.current_op = op self._afd_ops[lpOverlapped] = op afd_group.size += 1 - if afd_group.size < MAX_AFD_GROUP_SIZE: - self._vacant_afd_groups.add(afd_group) + if afd_group.size >= MAX_AFD_GROUP_SIZE: + self._vacant_afd_groups.remove(afd_group) async def _afd_poll(self, sock, mode): base_handle = _get_base_socket(sock) From fbb0ad44bf5a4975e6a3110bc7f2bc9825e2a7a2 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 26 Dec 2020 15:00:17 -0500 Subject: [PATCH 0531/1498] drop apparently useless loop in test --- trio/tests/test_socket.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 7c150a508e..421c826d81 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -1002,8 +1002,6 @@ async def test_many_sockets(): for s in sockets: nursery.start_soon(_core.wait_readable, s) await _core.wait_all_tasks_blocked() - for _ in range(1000): - await _core.cancel_shielded_checkpoint() nursery.cancel_scope.cancel() for sock in sockets: sock.close() From 91a2899c9aad86086604f3bb8fcc24190009c858 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Mon, 28 Dec 2020 07:47:41 -0500 Subject: [PATCH 0532/1498] fix lack of file descriptors case --- trio/tests/test_socket.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 421c826d81..f8c061ffd3 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -1,3 +1,5 @@ +import errno + import pytest import attr @@ -995,8 +997,12 @@ async def receiver(): async def test_many_sockets(): total = 5000 # Must be more than MAX_AFD_GROUP_SIZE sockets = [] - for _ in range(total // 2): - a, b = stdlib_socket.socketpair() + for x in range(total // 2): + try: + a, b = stdlib_socket.socketpair() + except OSError as e: + assert e.errno in (errno.EMFILE, errno.ENFILE) + break sockets += [a, b] async with _core.open_nursery() as nursery: for s in sockets: @@ -1005,3 +1011,5 @@ async def test_many_sockets(): nursery.cancel_scope.cancel() for sock in sockets: sock.close() + if x != total // 2 - 1: + print(f"Unable to open more than {(x-1)*2} sockets.") From 944eea1cba1cf6ece9e28b3aa0474977d8c4ac37 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 27 Dec 2020 22:02:08 -0800 Subject: [PATCH 0533/1498] Update newsfragment I guess this is slightly better, but really I'm just doing this to force a CI re-run. --- newsfragments/1280.bugfix.rst | 1 + newsfragments/1832.bugfix.rst | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 newsfragments/1280.bugfix.rst delete mode 100644 newsfragments/1832.bugfix.rst diff --git a/newsfragments/1280.bugfix.rst b/newsfragments/1280.bugfix.rst new file mode 100644 index 0000000000..ca087ff8c7 --- /dev/null +++ b/newsfragments/1280.bugfix.rst @@ -0,0 +1 @@ +Previously, on Windows, Trio programs using thousands of sockets at the same time could trigger extreme slowdowns in the Windows kernel. Now, Trio works around this issue, so you should be able to use as many sockets as you want. diff --git a/newsfragments/1832.bugfix.rst b/newsfragments/1832.bugfix.rst deleted file mode 100644 index ed4ba47b7b..0000000000 --- a/newsfragments/1832.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid O(n**3) scaling of windows socket cancellation \ No newline at end of file From 1e5321ccc12529314e9fdda5f1a364bdeeed9e79 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 29 Dec 2020 05:33:33 +0000 Subject: [PATCH 0534/1498] Bump ptyprocess from 0.6.0 to 0.7.0 Bumps [ptyprocess](https://github.com/pexpect/ptyprocess) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/pexpect/ptyprocess/releases) - [Commits](https://github.com/pexpect/ptyprocess/compare/0.6.0...0.7.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index afb423e2ea..43f9c55215 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -36,7 +36,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest prompt-toolkit==3.0.8 # via ipython -ptyprocess==0.6.0 # via pexpect +ptyprocess==0.7.0 # via pexpect py==1.10.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi From 517a13d612dfdb591eaf8d34019802201a86e910 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 29 Dec 2020 06:47:52 +0000 Subject: [PATCH 0535/1498] Bump jedi from 0.17.2 to 0.18.0 Bumps [jedi](https://github.com/davidhalter/jedi) from 0.17.2 to 0.18.0. - [Release notes](https://github.com/davidhalter/jedi/releases) - [Changelog](https://github.com/davidhalter/jedi/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/jedi/compare/v0.17.2...v0.18.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 43f9c55215..11ca4f0ae3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,14 +23,14 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.19.0 # via -r test-requirements.in isort==5.6.4 # via pylint -jedi==0.17.2 # via -r test-requirements.in, ipython +jedi==0.18.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy mypy==0.790 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in packaging==20.8 # via pytest -parso==0.7.1 # via jedi +parso==0.8.1 # via jedi pathspec==0.8.1 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython From 69ddbd951c1805db7e9fd886f0333155c05a0128 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 28 Dec 2020 23:08:00 -0800 Subject: [PATCH 0536/1498] Test non-nightly pypy's on GHA --- .github/workflows/ci.yml | 2 +- .travis.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fde947ed0..59460ffd94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9', '3.6-dev', '3.7-dev', '3.8-dev', '3.9-dev'] + python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.6-dev', '3.7-dev', '3.8-dev', '3.9-dev'] check_formatting: ['0'] extra_name: [''] include: diff --git a/.travis.yml b/.travis.yml index d04994cf2b..38ec33d60f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ dist: focal jobs: include: # The pypy tests are slow, so we list them first - - python: pypy3.6-7.2.0 - dist: bionic - language: generic env: PYPY_NIGHTLY_BRANCH=py3.6 - language: generic From 3301d9cc643ed57125c82ec9f762fad706c1a349 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 28 Dec 2020 23:18:55 -0800 Subject: [PATCH 0537/1498] Try moving nightly pypy builds off travis I also dropped the pypy-36 nightly here, b/c I don't think it gives us much -- new features seem to be happening in py3.7. --- .github/workflows/ci.yml | 5 +++++ .travis.yml | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59460ffd94..e0959c1797 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,11 +61,15 @@ jobs: matrix: python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.6-dev', '3.7-dev', '3.8-dev', '3.9-dev'] check_formatting: ['0'] + pypy_nightly_branch: [''] extra_name: [''] include: - python: '3.8' check_formatting: '1' extra_name: ', check formatting' + - python: '3.7' # <- not actually used + pypy_nightly_branch: 'py3.7' + extra_name: ', pypy 3.7 nightly' steps: - name: Checkout uses: actions/checkout@v2 @@ -82,6 +86,7 @@ jobs: - name: Run tests run: ./ci.sh env: + PYPY_NIGHTLY_BRANCH: '${{ matrix.pypy_nightly_branch }}' CHECK_FORMATTING: '${{ matrix.check_formatting }}' # Should match 'name:' up above JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' diff --git a/.travis.yml b/.travis.yml index 38ec33d60f..1f397d2a40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,6 @@ dist: focal jobs: include: - # The pypy tests are slow, so we list them first - - language: generic - env: PYPY_NIGHTLY_BRANCH=py3.6 - - language: generic - env: PYPY_NIGHTLY_BRANCH=py3.7 - - python: 3.6.1 # earliest 3.6 version available on Travis # https://github.com/pypa/setuptools/issues/2350 env: SETUPTOOLS_USE_DISTUTILS=stdlib From f41a3e98530bd1ab11becc59543da4b52f1bb7d9 Mon Sep 17 00:00:00 2001 From: rednafi Date: Wed, 30 Dec 2020 04:17:43 +0600 Subject: [PATCH 0538/1498] Replaced .format() with f-strings in the tutorial examples --- docs/source/tutorial.rst | 14 +++++++------- docs/source/tutorial/echo-client.py | 6 +++--- docs/source/tutorial/echo-server.py | 9 ++++----- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 676ff91b23..82d4663a2d 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -273,7 +273,7 @@ this, that tries to call an async function but leaves out the trio.sleep(2 * x) sleep_time = time.perf_counter() - start_time - print("Woke up after {:.2f} seconds, feeling well rested!".format(sleep_time)) + print(f"Woke up after {sleep_time:.2f} seconds, feeling well rested!") trio.run(broken_double_sleep, 3) @@ -774,7 +774,7 @@ above, it's baked into Trio's design that when it has multiple tasks, they take turns, so at each moment only one of them is actively running. We're not so much overcoming the GIL as embracing it. But if you're willing to accept that, plus a bit of extra work to put these new -``async`` and ``await`` keywords in the right places, then in exchange +``async`` and ``await`` keywords in the right places, then in exchange you get: * Excellent scalability: Trio can run 10,000+ tasks simultaneously @@ -836,8 +836,8 @@ Networking with Trio Now let's take what we've learned and use it to do some I/O, which is where async/await really shines. -The traditional toy application for demonstrating network APIs is an -"echo server": a program that awaits arbitrary data from remote clients, +The traditional toy application for demonstrating network APIs is an +"echo server": a program that awaits arbitrary data from remote clients, and then sends that same data right back. (Probably a more relevant example these days would be an application that does lots of concurrent HTTP requests, but for that `you need an HTTP library @@ -845,9 +845,9 @@ requests, but for that `you need an HTTP library such as `asks `__, so we'll stick with the echo server tradition.) -In this tutorial, we present both ends of the pipe: the client, and the -server. The client periodically sends data to the server, and displays its -answers. The server awaits connections; when a client connects, it recopies +In this tutorial, we present both ends of the pipe: the client, and the +server. The client periodically sends data to the server, and displays its +answers. The server awaits connections; when a client connects, it recopies the received data back on the pipe. diff --git a/docs/source/tutorial/echo-client.py b/docs/source/tutorial/echo-client.py index 06f6a81e7e..8ee3e2a7ca 100644 --- a/docs/source/tutorial/echo-client.py +++ b/docs/source/tutorial/echo-client.py @@ -13,19 +13,19 @@ async def sender(client_stream): print("sender: started!") while True: data = b"async can sometimes be confusing, but I believe in you!" - print("sender: sending {!r}".format(data)) + print(f"sender: sending {data!r}") await client_stream.send_all(data) await trio.sleep(1) async def receiver(client_stream): print("receiver: started!") async for data in client_stream: - print("receiver: got data {!r}".format(data)) + print(f"receiver: got data {data!r}") print("receiver: connection closed") sys.exit() async def parent(): - print("parent: connecting to 127.0.0.1:{}".format(PORT)) + print(f"parent: connecting to 127.0.0.1:{PORT}") client_stream = await trio.open_tcp_stream("127.0.0.1", PORT) async with client_stream: async with trio.open_nursery() as nursery: diff --git a/docs/source/tutorial/echo-server.py b/docs/source/tutorial/echo-server.py index a184cdf46b..ffa63ab5f4 100644 --- a/docs/source/tutorial/echo-server.py +++ b/docs/source/tutorial/echo-server.py @@ -15,12 +15,12 @@ async def echo_server(server_stream): # Assign each connection a unique number to make our debug prints easier # to understand when there are multiple simultaneous connections. ident = next(CONNECTION_COUNTER) - print("echo_server {}: started".format(ident)) + print(f"echo_server {ident}: started") try: async for data in server_stream: - print("echo_server {}: received data {!r}".format(ident, data)) + print(f"echo_server {ident}: received data {data!r}") await server_stream.send_all(data) - print("echo_server {}: connection closed".format(ident)) + print(f"echo_server {ident}: connection closed") # FIXME: add discussion of MultiErrors to the tutorial, and use # MultiError.catch here. (Not important in this case, but important if the # server code uses nurseries internally.) @@ -28,7 +28,7 @@ async def echo_server(server_stream): # Unhandled exceptions will propagate into our parent and take # down the whole program. If the exception is KeyboardInterrupt, # that's what we want, but otherwise maybe not... - print("echo_server {}: crashed: {!r}".format(ident, exc)) + print(f"echo_server {ident}: crashed: {exc!r}") async def main(): await trio.serve_tcp(echo_server, PORT) @@ -38,4 +38,3 @@ async def main(): # back and factor it out into a separate function anyway. So it's simplest to # just make it a standalone function from the beginning. trio.run(main) - From 6bb95b54fa1b6cd3e837bf439d277e54e47b1bc1 Mon Sep 17 00:00:00 2001 From: rednafi Date: Wed, 30 Dec 2020 04:41:29 +0600 Subject: [PATCH 0539/1498] Replaced .format() with f-strings in the tutorial examples --- docs/source/tutorial.rst | 14 +++++++------- docs/source/tutorial/echo-client.py | 6 +++--- docs/source/tutorial/echo-server.py | 9 ++++----- docs/source/tutorial/tasks-with-trace.py | 6 +++--- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 676ff91b23..82d4663a2d 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -273,7 +273,7 @@ this, that tries to call an async function but leaves out the trio.sleep(2 * x) sleep_time = time.perf_counter() - start_time - print("Woke up after {:.2f} seconds, feeling well rested!".format(sleep_time)) + print(f"Woke up after {sleep_time:.2f} seconds, feeling well rested!") trio.run(broken_double_sleep, 3) @@ -774,7 +774,7 @@ above, it's baked into Trio's design that when it has multiple tasks, they take turns, so at each moment only one of them is actively running. We're not so much overcoming the GIL as embracing it. But if you're willing to accept that, plus a bit of extra work to put these new -``async`` and ``await`` keywords in the right places, then in exchange +``async`` and ``await`` keywords in the right places, then in exchange you get: * Excellent scalability: Trio can run 10,000+ tasks simultaneously @@ -836,8 +836,8 @@ Networking with Trio Now let's take what we've learned and use it to do some I/O, which is where async/await really shines. -The traditional toy application for demonstrating network APIs is an -"echo server": a program that awaits arbitrary data from remote clients, +The traditional toy application for demonstrating network APIs is an +"echo server": a program that awaits arbitrary data from remote clients, and then sends that same data right back. (Probably a more relevant example these days would be an application that does lots of concurrent HTTP requests, but for that `you need an HTTP library @@ -845,9 +845,9 @@ requests, but for that `you need an HTTP library such as `asks `__, so we'll stick with the echo server tradition.) -In this tutorial, we present both ends of the pipe: the client, and the -server. The client periodically sends data to the server, and displays its -answers. The server awaits connections; when a client connects, it recopies +In this tutorial, we present both ends of the pipe: the client, and the +server. The client periodically sends data to the server, and displays its +answers. The server awaits connections; when a client connects, it recopies the received data back on the pipe. diff --git a/docs/source/tutorial/echo-client.py b/docs/source/tutorial/echo-client.py index 06f6a81e7e..8ee3e2a7ca 100644 --- a/docs/source/tutorial/echo-client.py +++ b/docs/source/tutorial/echo-client.py @@ -13,19 +13,19 @@ async def sender(client_stream): print("sender: started!") while True: data = b"async can sometimes be confusing, but I believe in you!" - print("sender: sending {!r}".format(data)) + print(f"sender: sending {data!r}") await client_stream.send_all(data) await trio.sleep(1) async def receiver(client_stream): print("receiver: started!") async for data in client_stream: - print("receiver: got data {!r}".format(data)) + print(f"receiver: got data {data!r}") print("receiver: connection closed") sys.exit() async def parent(): - print("parent: connecting to 127.0.0.1:{}".format(PORT)) + print(f"parent: connecting to 127.0.0.1:{PORT}") client_stream = await trio.open_tcp_stream("127.0.0.1", PORT) async with client_stream: async with trio.open_nursery() as nursery: diff --git a/docs/source/tutorial/echo-server.py b/docs/source/tutorial/echo-server.py index a184cdf46b..ffa63ab5f4 100644 --- a/docs/source/tutorial/echo-server.py +++ b/docs/source/tutorial/echo-server.py @@ -15,12 +15,12 @@ async def echo_server(server_stream): # Assign each connection a unique number to make our debug prints easier # to understand when there are multiple simultaneous connections. ident = next(CONNECTION_COUNTER) - print("echo_server {}: started".format(ident)) + print(f"echo_server {ident}: started") try: async for data in server_stream: - print("echo_server {}: received data {!r}".format(ident, data)) + print(f"echo_server {ident}: received data {data!r}") await server_stream.send_all(data) - print("echo_server {}: connection closed".format(ident)) + print(f"echo_server {ident}: connection closed") # FIXME: add discussion of MultiErrors to the tutorial, and use # MultiError.catch here. (Not important in this case, but important if the # server code uses nurseries internally.) @@ -28,7 +28,7 @@ async def echo_server(server_stream): # Unhandled exceptions will propagate into our parent and take # down the whole program. If the exception is KeyboardInterrupt, # that's what we want, but otherwise maybe not... - print("echo_server {}: crashed: {!r}".format(ident, exc)) + print(f"echo_server {ident}: crashed: {exc!r}") async def main(): await trio.serve_tcp(echo_server, PORT) @@ -38,4 +38,3 @@ async def main(): # back and factor it out into a separate function anyway. So it's simplest to # just make it a standalone function from the beginning. trio.run(main) - diff --git a/docs/source/tutorial/tasks-with-trace.py b/docs/source/tutorial/tasks-with-trace.py index 38a1ffe862..26f8fb205d 100644 --- a/docs/source/tutorial/tasks-with-trace.py +++ b/docs/source/tutorial/tasks-with-trace.py @@ -32,7 +32,7 @@ def before_run(self): def _print_with_task(self, msg, task): # repr(task) is perhaps more useful than task.name in general, # but in context of a tutorial the extra noise is unhelpful. - print("{}: {}".format(msg, task.name)) + print(f"{msg}: {task.name}") def task_spawned(self, task): self._print_with_task("### new task spawned", task) @@ -51,14 +51,14 @@ def task_exited(self, task): def before_io_wait(self, timeout): if timeout: - print("### waiting for I/O for up to {} seconds".format(timeout)) + print(f"### waiting for I/O for up to {timeout} seconds") else: print("### doing a quick check for I/O") self._sleep_time = trio.current_time() def after_io_wait(self, timeout): duration = trio.current_time() - self._sleep_time - print("### finished I/O check (took {} seconds)".format(duration)) + print(f"### finished I/O check (took {duration} seconds)") def after_run(self): print("!!! run finished") From ccdaaa355f606214dd75ff9d2c278fa24692a957 Mon Sep 17 00:00:00 2001 From: rednafi Date: Thu, 31 Dec 2020 05:00:00 +0600 Subject: [PATCH 0540/1498] F-stringified reference-core examples --- docs/source/reference-core.rst | 2 +- docs/source/reference-core/channels-mpmc-broken.py | 4 ++-- docs/source/reference-core/channels-mpmc-fixed.py | 4 ++-- docs/source/reference-core/channels-shutdown.py | 4 ++-- docs/source/reference-core/channels-simple.py | 4 ++-- docs/source/reference-core/contextvar-example.py | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 7d2cf44a61..bba1b09142 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1086,7 +1086,7 @@ you'll see that the two tasks politely take turns:: async def loopy_child(number, lock): while True: async with lock: - print("Child {} has the lock!".format(number)) + print(f"Child {number} has the lock!") await trio.sleep(0.5) async def main(): diff --git a/docs/source/reference-core/channels-mpmc-broken.py b/docs/source/reference-core/channels-mpmc-broken.py index 2a755acba3..6b4adf298a 100644 --- a/docs/source/reference-core/channels-mpmc-broken.py +++ b/docs/source/reference-core/channels-mpmc-broken.py @@ -16,14 +16,14 @@ async def main(): async def producer(name, send_channel): async with send_channel: for i in range(3): - await send_channel.send("{} from producer {}".format(i, name)) + await send_channel.send(f"{i} from producer {name}") # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) async def consumer(name, receive_channel): async with receive_channel: async for value in receive_channel: - print("consumer {} got value {!r}".format(name, value)) + print(f"consumer {name} got value {value!r}") # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) diff --git a/docs/source/reference-core/channels-mpmc-fixed.py b/docs/source/reference-core/channels-mpmc-fixed.py index a3e7044fe7..0233938058 100644 --- a/docs/source/reference-core/channels-mpmc-fixed.py +++ b/docs/source/reference-core/channels-mpmc-fixed.py @@ -15,14 +15,14 @@ async def main(): async def producer(name, send_channel): async with send_channel: for i in range(3): - await send_channel.send("{} from producer {}".format(i, name)) + await send_channel.send(f"{i} from producer {name}") # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) async def consumer(name, receive_channel): async with receive_channel: async for value in receive_channel: - print("consumer {} got value {!r}".format(name, value)) + print(f"consumer {name} got value {value!r}") # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) diff --git a/docs/source/reference-core/channels-shutdown.py b/docs/source/reference-core/channels-shutdown.py index dcd35767ae..926eb0efc4 100644 --- a/docs/source/reference-core/channels-shutdown.py +++ b/docs/source/reference-core/channels-shutdown.py @@ -9,11 +9,11 @@ async def main(): async def producer(send_channel): async with send_channel: for i in range(3): - await send_channel.send("message {}".format(i)) + await send_channel.send(f"message {i}") async def consumer(receive_channel): async with receive_channel: async for value in receive_channel: - print("got value {!r}".format(value)) + print("got value {value!r}") trio.run(main) diff --git a/docs/source/reference-core/channels-simple.py b/docs/source/reference-core/channels-simple.py index d04ebd722c..c00fb145cf 100644 --- a/docs/source/reference-core/channels-simple.py +++ b/docs/source/reference-core/channels-simple.py @@ -13,11 +13,11 @@ async def producer(send_channel): # Producer sends 3 messages for i in range(3): # The producer sends using 'await send_channel.send(...)' - await send_channel.send("message {}".format(i)) + await send_channel.send(f"message {i}") async def consumer(receive_channel): # The consumer uses an 'async for' loop to receive the values: async for value in receive_channel: - print("got value {!r}".format(value)) + print(f"got value {value!r}") trio.run(main) diff --git a/docs/source/reference-core/contextvar-example.py b/docs/source/reference-core/contextvar-example.py index 7b98355f90..d2caa0d0dc 100644 --- a/docs/source/reference-core/contextvar-example.py +++ b/docs/source/reference-core/contextvar-example.py @@ -10,7 +10,7 @@ def log(msg): # Read from task-local storage: request_tag = request_info.get() - print("request {}: {}".format(request_tag, msg)) + print(f"request {request_tag}: {msg}") # An example "request handler" that does some work itself and also @@ -29,9 +29,9 @@ async def handle_request(tag): async def concurrent_helper(job): - log("Helper task {} started".format(job)) + log(f"Helper task {job} started") await trio.sleep(random.random()) - log("Helper task {} finished".format(job)) + log(f"Helper task {job} finished") # Spawn several "request handlers" simultaneously, to simulate a From 0a2585924d67b9e99fe0edc59ed239478460e18f Mon Sep 17 00:00:00 2001 From: rednafi Date: Thu, 31 Dec 2020 05:57:34 +0600 Subject: [PATCH 0541/1498] Fixed missing f --- docs/source/reference-core/channels-shutdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-core/channels-shutdown.py b/docs/source/reference-core/channels-shutdown.py index 926eb0efc4..8cb8e28225 100644 --- a/docs/source/reference-core/channels-shutdown.py +++ b/docs/source/reference-core/channels-shutdown.py @@ -14,6 +14,6 @@ async def producer(send_channel): async def consumer(receive_channel): async with receive_channel: async for value in receive_channel: - print("got value {value!r}") + print(f"got value {value!r}") trio.run(main) From 45732a7402bcd56ae59ed172209a6a42450a3120 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 31 Dec 2020 05:31:33 +0000 Subject: [PATCH 0542/1498] Bump typed-ast from 1.4.1 to 1.4.2 Bumps [typed-ast](https://github.com/python/typed_ast) from 1.4.1 to 1.4.2. - [Release notes](https://github.com/python/typed_ast/releases) - [Changelog](https://github.com/python/typed_ast/blob/master/release_process.md) - [Commits](https://github.com/python/typed_ast/compare/1.4.1...1.4.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 11ca4f0ae3..7b0d899473 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -54,7 +54,7 @@ sortedcontainers==2.3.0 # via -r test-requirements.in toml==0.10.2 # via black, pylint, pytest traitlets==5.0.5 # via ipython trustme==0.6.0 # via -r test-requirements.in -typed-ast==1.4.1 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy +typed-ast==1.4.2 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy wcwidth==0.2.5 # via prompt-toolkit wrapt==1.12.1 # via astroid From 1cd70d4f82a772b20b5caf59df8ad61ee598f795 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 31 Dec 2020 05:32:25 +0000 Subject: [PATCH 0543/1498] Bump isort from 5.6.4 to 5.7.0 Bumps [isort](https://github.com/pycqa/isort) from 5.6.4 to 5.7.0. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.6.4...5.7.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 11ca4f0ae3..a18827b82b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ immutables==0.14 # via -r test-requirements.in iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets ipython==7.19.0 # via -r test-requirements.in -isort==5.6.4 # via pylint +isort==5.7.0 # via pylint jedi==0.18.0 # via -r test-requirements.in, ipython lazy-object-proxy==1.4.3 # via astroid mccabe==0.6.1 # via flake8, pylint From 0ddd2d4064f77a3830805c65e2a2b362306f90b6 Mon Sep 17 00:00:00 2001 From: Alexey Shamrin Date: Mon, 4 Jan 2021 17:34:19 +0200 Subject: [PATCH 0544/1498] mention Nursery.cancel_scope in CancelScope docs fixes #1831 --- docs/source/reference-core.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 7d2cf44a61..d20f6fc811 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -524,6 +524,10 @@ objects. .. autoattribute:: cancel_called +Often there is no need to create :class:`CancelScope` object. Trio +already includes :attr:`~trio.Nursery.cancel_scope` attribute in a +task-related :class:`Nursery` object. We will cover nurseries later in +the manual. Trio also provides several convenience functions for the common situation of just wanting to impose a timeout on some code: From fe52822f00d87204ae0200671cb9961d05a3b905 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 5 Jan 2021 05:43:24 +0000 Subject: [PATCH 0545/1498] Bump sphinx-rtd-theme from 0.5.0 to 0.5.1 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 0.5.0 to 0.5.1. - [Release notes](https://github.com/readthedocs/sphinx_rtd_theme/releases) - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/0.5.0...0.5.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c50dbaf08a..ae655e95a4 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -27,8 +27,8 @@ requests==2.25.1 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.3.0 # via -r docs-requirements.in -sphinx-rtd-theme==0.5.0 # via -r docs-requirements.in sphinx==3.3.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio +sphinx_rtd_theme==0.5.1 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 9a4db8648d5542dea643711f680da6237593fb1c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 6 Jan 2021 05:55:12 +0000 Subject: [PATCH 0546/1498] Bump prompt-toolkit from 3.0.8 to 3.0.9 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.8 to 3.0.9. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/commits/3.0.9) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4312ef7c53..f627499c8b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -35,7 +35,7 @@ pathspec==0.8.1 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.8 # via ipython +prompt-toolkit==3.0.9 # via ipython ptyprocess==0.7.0 # via pexpect py==1.10.0 # via pytest pycodestyle==2.6.0 # via flake8 From 657803f7ec164a252f57f2bdf91e3cf3f7140bd9 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 8 Jan 2021 10:24:36 +0400 Subject: [PATCH 0547/1498] Remove deprecated SSLStream max_refill_bytes parameter --- newsfragments/959.deprecated.rst | 1 + trio/_ssl.py | 7 ------- trio/tests/test_ssl.py | 12 ------------ 3 files changed, 1 insertion(+), 19 deletions(-) create mode 100644 newsfragments/959.deprecated.rst diff --git a/newsfragments/959.deprecated.rst b/newsfragments/959.deprecated.rst new file mode 100644 index 0000000000..5d2083f34f --- /dev/null +++ b/newsfragments/959.deprecated.rst @@ -0,0 +1 @@ +Remove deprecated ``max_refill_bytes`` from ``SSLStream``. diff --git a/trio/_ssl.py b/trio/_ssl.py index 9c346d268b..c4ffa3ddbe 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -159,7 +159,6 @@ from ._highlevel_generic import aclose_forcefully from . import _sync from ._util import ConflictDetector, Final -from ._deprecate import warn_deprecated ################################################################ # SSLStream @@ -328,12 +327,9 @@ def __init__( server_hostname=None, server_side=False, https_compatible=False, - max_refill_bytes="unused and deprecated", ): self.transport_stream = transport_stream self._state = _State.OK - if max_refill_bytes != "unused and deprecated": - warn_deprecated("max_refill_bytes=...", "0.12.0", issue=959, instead=None) self._https_compatible = https_compatible self._outgoing = _stdlib_ssl.MemoryBIO() self._delayed_outgoing = None @@ -898,10 +894,7 @@ def __init__( ssl_context, *, https_compatible=False, - max_refill_bytes="unused and deprecated", ): - if max_refill_bytes != "unused and deprecated": - warn_deprecated("max_refill_bytes=...", "0.12.0", issue=959, instead=None) self.transport_listener = transport_listener self._ssl_context = ssl_context self._https_compatible = https_compatible diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index a4e22716ae..f160af4999 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1278,15 +1278,3 @@ async def setup(**kwargs): await aclose_forcefully(ssl_listener) await aclose_forcefully(ssl_client) await aclose_forcefully(ssl_server) - - -async def test_deprecated_max_refill_bytes(client_ctx): - stream1, stream2 = memory_stream_pair() - with pytest.warns(trio.TrioDeprecationWarning): - SSLStream(stream1, client_ctx, max_refill_bytes=100) - with pytest.warns(trio.TrioDeprecationWarning): - # passing None is wrong here, but I'm too lazy to make a fake Listener - # and we get away with it for now. And this test will be deleted in a - # release or two anyway, so hopefully we'll keep getting away with it - # for long enough. - SSLListener(None, client_ctx, max_refill_bytes=100) From 4e93f86c5f37438ed6b931e1de2722b02e186158 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 8 Jan 2021 10:39:32 +0400 Subject: [PATCH 0548/1498] Remove the deprecated `tiebreaker` argument --- newsfragments/1558.deprecated.rst | 1 + trio/_core/_generated_run.py | 4 ++-- trio/_core/_run.py | 18 ++++-------------- trio/tests/test_testing.py | 27 --------------------------- 4 files changed, 7 insertions(+), 43 deletions(-) create mode 100644 newsfragments/1558.deprecated.rst diff --git a/newsfragments/1558.deprecated.rst b/newsfragments/1558.deprecated.rst new file mode 100644 index 0000000000..aff1acb943 --- /dev/null +++ b/newsfragments/1558.deprecated.rst @@ -0,0 +1 @@ +Remove the deprecated ``tiebreaker`` argument to `trio.testing.wait_all_tasks_blocked`. diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index 0da2c4f48e..1272b4c73c 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -169,7 +169,7 @@ def current_trio_token(): raise RuntimeError("must be called from async context") -async def wait_all_tasks_blocked(cushion=0.0, tiebreaker='deprecated'): +async def wait_all_tasks_blocked(cushion=0.0): """Block until there are no runnable tasks. This is useful in testing code when you want to give other tasks a @@ -229,7 +229,7 @@ async def test_lock_fairness(): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return await GLOBAL_RUN_CONTEXT.runner.wait_all_tasks_blocked(cushion, tiebreaker) + return await GLOBAL_RUN_CONTEXT.runner.wait_all_tasks_blocked(cushion) except AttributeError: raise RuntimeError("must be called from async context") diff --git a/trio/_core/_run.py b/trio/_core/_run.py index fe51f7bfc8..154b9bb590 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1661,7 +1661,7 @@ def _deliver_ki_cb(self): waiting_for_idle = attr.ib(factory=SortedDict) @_public - async def wait_all_tasks_blocked(self, cushion=0.0, tiebreaker="deprecated"): + async def wait_all_tasks_blocked(self, cushion=0.0): """Block until there are no runnable tasks. This is useful in testing code when you want to give other tasks a @@ -1719,18 +1719,8 @@ async def test_lock_fairness(): print("FAIL") """ - if tiebreaker == "deprecated": - tiebreaker = 0 - else: - warn_deprecated( - "the 'tiebreaker' argument to wait_all_tasks_blocked", - "v0.16.0", - issue=1558, - instead=None, - ) - task = current_task() - key = (cushion, tiebreaker, id(task)) + key = (cushion, id(task)) self.waiting_for_idle[key] = task def abort(_): @@ -2057,7 +2047,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): idle_primed = None if runner.waiting_for_idle: - cushion, tiebreaker, _ = runner.waiting_for_idle.keys()[0] + cushion, _ = runner.waiting_for_idle.keys()[0] if cushion < timeout: timeout = cushion idle_primed = IdlePrimedTypes.WAITING_FOR_IDLE @@ -2109,7 +2099,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): if idle_primed is IdlePrimedTypes.WAITING_FOR_IDLE: while runner.waiting_for_idle: key, task = runner.waiting_for_idle.peekitem(0) - if key[:2] == (cushion, tiebreaker): + if key[0] == cushion: del runner.waiting_for_idle[key] runner.reschedule(task) else: diff --git a/trio/tests/test_testing.py b/trio/tests/test_testing.py index b9fb8ac2bf..0b10ae71e1 100644 --- a/trio/tests/test_testing.py +++ b/trio/tests/test_testing.py @@ -103,33 +103,6 @@ async def wait_big_cushion(): ] -# This test can be deleted after tiebreaker= is removed -async def test_wait_all_tasks_blocked_with_tiebreaker(recwarn): - record = [] - - async def do_wait(cushion, tiebreaker): - await wait_all_tasks_blocked(cushion=cushion, tiebreaker=tiebreaker) - record.append((cushion, tiebreaker)) - - async with _core.open_nursery() as nursery: - nursery.start_soon(do_wait, 0, 0) - nursery.start_soon(do_wait, 0, -1) - nursery.start_soon(do_wait, 0, 1) - nursery.start_soon(do_wait, 0, -1) - nursery.start_soon(do_wait, 0.0001, 10) - nursery.start_soon(do_wait, 0.0001, -10) - - assert record == sorted(record) - assert record == [ - (0, -1), - (0, -1), - (0, 0), - (0, 1), - (0.0001, -10), - (0.0001, 10), - ] - - ################################################################ From 4a7dce0c0c515ac7243ac6c4e714780ee633c4f6 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 8 Jan 2021 13:09:03 +0400 Subject: [PATCH 0549/1498] Fix newfragments to link to SSLStream Co-authored-by: Alexey Shamrin --- newsfragments/959.deprecated.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/959.deprecated.rst b/newsfragments/959.deprecated.rst index 5d2083f34f..4999cbf79a 100644 --- a/newsfragments/959.deprecated.rst +++ b/newsfragments/959.deprecated.rst @@ -1 +1 @@ -Remove deprecated ``max_refill_bytes`` from ``SSLStream``. +Remove deprecated ``max_refill_bytes`` from :class:`SSLStream`. From 257e5ba7dcc2fdd3dfe7df15855b95bb112b108a Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 15 Nov 2020 19:24:30 -0800 Subject: [PATCH 0550/1498] Avoid creating reference cycles in MultiError.filter When put together, this change + https://github.com/python-trio/outcome/pull/29 should fix #1770 --- trio/_core/_multierror.py | 4 ++++ trio/_core/tests/test_run.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 9e0de7cf9f..08cc22cc63 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -114,6 +114,9 @@ def push_tb_down(tb, exc, preserved): preserved = set() new_root_exc = filter_tree(root_exc, preserved) push_tb_down(None, root_exc, preserved) + # Delete the local functions avoid a reference cycle (see + # test_simple_cancel_scope_usage_doesnt_create_cyclic_garbage) + del filter_tree, push_tb_down return new_root_exc @@ -132,6 +135,7 @@ def __enter__(self): def __exit__(self, etype, exc, tb): if exc is not None: filtered_exc = MultiError.filter(self._handler, exc) + if filtered_exc is exc: # Let the interpreter re-raise it return False diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index bc1f7867a0..060b0b0c93 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -9,6 +9,7 @@ from contextlib import contextmanager, ExitStack from math import inf from textwrap import dedent +import gc import attr import outcome @@ -2195,3 +2196,33 @@ async def test_cancel_scope_deadline_duplicates(): cscope.deadline = now + 9998 cscope.deadline = now + 9999 await sleep(0.01) + + +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" +) +async def test_simple_cancel_scope_usage_doesnt_create_cyclic_garbage(): + # https://github.com/python-trio/trio/issues/1770 + gc.collect() + + async def do_a_cancel(): + with _core.CancelScope() as cscope: + cscope.cancel() + await sleep_forever() + + old_flags = gc.get_debug() + try: + gc.collect() + gc.set_debug(gc.DEBUG_SAVEALL) + + await do_a_cancel() + await do_a_cancel() + + async with _core.open_nursery() as nursery: + nursery.start_soon(do_a_cancel) + + gc.collect() + assert not gc.garbage + finally: + gc.set_debug(old_flags) + gc.garbage.clear() From bbd6b43de745459b11785985323baaf677c65478 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 15 Nov 2020 19:37:02 -0800 Subject: [PATCH 0551/1498] Add newsfragment --- newsfragments/1770.bugfix.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 newsfragments/1770.bugfix.rst diff --git a/newsfragments/1770.bugfix.rst b/newsfragments/1770.bugfix.rst new file mode 100644 index 0000000000..4ef2303d9d --- /dev/null +++ b/newsfragments/1770.bugfix.rst @@ -0,0 +1,3 @@ +Trio now avoids creating cyclic garbage as often. This should have a +minimal impact on most programs, but can slightly reduce how often the +cycle collector GC runs on CPython, which can reduce latency spikes. From de07659cc728c6893e5f2af638098087945d835c Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Fri, 8 Jan 2021 21:54:09 +0900 Subject: [PATCH 0552/1498] Avoid creating reference cycles in case of nursery cancel --- trio/_core/_multierror.py | 2 +- trio/_core/_run.py | 7 ++++++- trio/_core/tests/test_run.py | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 08cc22cc63..13fc3f3d0f 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -114,7 +114,7 @@ def push_tb_down(tb, exc, preserved): preserved = set() new_root_exc = filter_tree(root_exc, preserved) push_tb_down(None, root_exc, preserved) - # Delete the local functions avoid a reference cycle (see + # Delete the local functions to avoid a reference cycle (see # test_simple_cancel_scope_usage_doesnt_create_cyclic_garbage) del filter_tree, push_tb_down return new_root_exc diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 154b9bb590..e56977f386 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -937,7 +937,12 @@ def aborted(raise_cancel): popped = self._parent_task._child_nurseries.pop() assert popped is self if self._pending_excs: - return MultiError(self._pending_excs) + try: + return MultiError(self._pending_excs) + finally: + # avoid a garbage cycle + # (see test_nursery_cancel_doesnt_create_cyclic_garbage) + del self._pending_excs def start_soon(self, async_fn, *args, name=None): """Creates a child task, scheduling ``await async_fn(*args)``. diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 060b0b0c93..4c4e12b5df 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2226,3 +2226,26 @@ async def do_a_cancel(): finally: gc.set_debug(old_flags) gc.garbage.clear() + + +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" +) +async def test_nursery_cancel_doesnt_create_cyclic_garbage(): + # https://github.com/python-trio/trio/issues/1770#issuecomment-730229423 + gc.collect() + + old_flags = gc.get_debug() + try: + for i in range(3): + async with _core.open_nursery() as nursery: + gc.collect() + gc.set_debug(gc.DEBUG_LEAK) + nursery.cancel_scope.cancel() + + gc.collect() + gc.set_debug(0) + assert not gc.garbage + finally: + gc.set_debug(old_flags) + gc.garbage.clear() From 06f63640bac675877be213bcb6d6f467050e5257 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 05:55:24 +0000 Subject: [PATCH 0553/1498] Bump prompt-toolkit from 3.0.9 to 3.0.10 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.9 to 3.0.10. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.9...3.0.10) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f627499c8b..e6e68d9e8d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -35,7 +35,7 @@ pathspec==0.8.1 # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.9 # via ipython +prompt-toolkit==3.0.10 # via ipython ptyprocess==0.7.0 # via pexpect py==1.10.0 # via pytest pycodestyle==2.6.0 # via flake8 From 069bdd03cafdf3824c81ddf8fddc8c1359bf1ee1 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 11 Jan 2021 10:30:59 +0400 Subject: [PATCH 0554/1498] Bump version to 0.18.0 --- docs/source/history.rst | 35 +++++++++++++++++++++++++++++++ newsfragments/1280.bugfix.rst | 1 - newsfragments/1558.deprecated.rst | 1 - newsfragments/1722.deprecated.rst | 1 - newsfragments/1726.deprecated.rst | 1 - newsfragments/1738.bugfix.rst | 6 ------ newsfragments/1765.bugfix.rst | 1 - newsfragments/1770.bugfix.rst | 3 --- newsfragments/1797.feature.rst | 2 -- newsfragments/959.deprecated.rst | 1 - trio/_version.py | 2 +- 11 files changed, 36 insertions(+), 18 deletions(-) delete mode 100644 newsfragments/1280.bugfix.rst delete mode 100644 newsfragments/1558.deprecated.rst delete mode 100644 newsfragments/1722.deprecated.rst delete mode 100644 newsfragments/1726.deprecated.rst delete mode 100644 newsfragments/1738.bugfix.rst delete mode 100644 newsfragments/1765.bugfix.rst delete mode 100644 newsfragments/1770.bugfix.rst delete mode 100644 newsfragments/1797.feature.rst delete mode 100644 newsfragments/959.deprecated.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 830e3acafc..7c63b54c88 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,41 @@ Release history .. towncrier release notes start +Trio 0.18.0 (2021-01-11) +------------------------ + +Features +~~~~~~~~ + +- Add synchronous ``.close()`` methods and context manager (``with x``) support + for `.MemorySendChannel` and `.MemoryReceiveChannel`. (`#1797 `__) + + +Bugfixes +~~~~~~~~ + +- Previously, on Windows, Trio programs using thousands of sockets at the same time could trigger extreme slowdowns in the Windows kernel. Now, Trio works around this issue, so you should be able to use as many sockets as you want. (`#1280 `__) +- :func:`trio.from_thread.run` no longer crashes the Trio run if it is + executed after the system nursery has been closed but before the run + has finished. Calls made at this time will now raise + `trio.RunFinishedError`. This fixes a regression introduced in + Trio 0.17.0. The window in question is only one scheduler tick long in + most cases, but may be longer if async generators need to be cleaned up. (`#1738 `__) +- Fix a crash in pypy-3.7 (`#1765 `__) +- Trio now avoids creating cyclic garbage as often. This should have a + minimal impact on most programs, but can slightly reduce how often the + cycle collector GC runs on CPython, which can reduce latency spikes. (`#1770 `__) + + +Deprecations and removals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Remove deprecated ``max_refill_bytes`` from :class:`SSLStream`. (`#959 `__) +- Remove the deprecated ``tiebreaker`` argument to `trio.testing.wait_all_tasks_blocked`. (`#1558 `__) +- Remove the deprecated ``trio.hazmat`` module. (`#1722 `__) +- Stop allowing subclassing public classes. This behavior was deprecated in 0.15.0. (`#1726 `__) + + Trio 0.17.0 (2020-09-15) ------------------------ diff --git a/newsfragments/1280.bugfix.rst b/newsfragments/1280.bugfix.rst deleted file mode 100644 index ca087ff8c7..0000000000 --- a/newsfragments/1280.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Previously, on Windows, Trio programs using thousands of sockets at the same time could trigger extreme slowdowns in the Windows kernel. Now, Trio works around this issue, so you should be able to use as many sockets as you want. diff --git a/newsfragments/1558.deprecated.rst b/newsfragments/1558.deprecated.rst deleted file mode 100644 index aff1acb943..0000000000 --- a/newsfragments/1558.deprecated.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the deprecated ``tiebreaker`` argument to `trio.testing.wait_all_tasks_blocked`. diff --git a/newsfragments/1722.deprecated.rst b/newsfragments/1722.deprecated.rst deleted file mode 100644 index 0c963d5a4a..0000000000 --- a/newsfragments/1722.deprecated.rst +++ /dev/null @@ -1 +0,0 @@ -Remove the deprecated ``trio.hazmat`` module. diff --git a/newsfragments/1726.deprecated.rst b/newsfragments/1726.deprecated.rst deleted file mode 100644 index 14cb51b9a7..0000000000 --- a/newsfragments/1726.deprecated.rst +++ /dev/null @@ -1 +0,0 @@ -Stop allowing subclassing public classes. This behavior was deprecated in 0.15.0. diff --git a/newsfragments/1738.bugfix.rst b/newsfragments/1738.bugfix.rst deleted file mode 100644 index ddb42050b4..0000000000 --- a/newsfragments/1738.bugfix.rst +++ /dev/null @@ -1,6 +0,0 @@ -:func:`trio.from_thread.run` no longer crashes the Trio run if it is -executed after the system nursery has been closed but before the run -has finished. Calls made at this time will now raise -`trio.RunFinishedError`. This fixes a regression introduced in -Trio 0.17.0. The window in question is only one scheduler tick long in -most cases, but may be longer if async generators need to be cleaned up. diff --git a/newsfragments/1765.bugfix.rst b/newsfragments/1765.bugfix.rst deleted file mode 100644 index 7d771c13e5..0000000000 --- a/newsfragments/1765.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash in pypy-3.7 diff --git a/newsfragments/1770.bugfix.rst b/newsfragments/1770.bugfix.rst deleted file mode 100644 index 4ef2303d9d..0000000000 --- a/newsfragments/1770.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Trio now avoids creating cyclic garbage as often. This should have a -minimal impact on most programs, but can slightly reduce how often the -cycle collector GC runs on CPython, which can reduce latency spikes. diff --git a/newsfragments/1797.feature.rst b/newsfragments/1797.feature.rst deleted file mode 100644 index f71c36ae75..0000000000 --- a/newsfragments/1797.feature.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add synchronous ``.close()`` methods and context manager (``with x``) support -for `.MemorySendChannel` and `.MemoryReceiveChannel`. diff --git a/newsfragments/959.deprecated.rst b/newsfragments/959.deprecated.rst deleted file mode 100644 index 4999cbf79a..0000000000 --- a/newsfragments/959.deprecated.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated ``max_refill_bytes`` from :class:`SSLStream`. diff --git a/trio/_version.py b/trio/_version.py index cd6e4e1649..c76ca48b2b 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.17.0+dev" +__version__ = "0.18.0" From b9f9d1d218cc485f6610c4290ba9ab50544d15f7 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 11 Jan 2021 10:44:25 +0400 Subject: [PATCH 0555/1498] Bump version to 0.18.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index c76ca48b2b..48369a7a2b 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.18.0" +__version__ = "0.18.0+dev" From a835badac061fc850796c456da80ec528d2b3cd1 Mon Sep 17 00:00:00 2001 From: Davor Cubranic Date: Mon, 11 Jan 2021 15:14:55 -0800 Subject: [PATCH 0556/1498] Clarify that deadline in `sleep_until()` uses internal clock units Fixes: #1860 --- trio/_timeouts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_timeouts.py b/trio/_timeouts.py index 517f344bf3..1f7878f89e 100644 --- a/trio/_timeouts.py +++ b/trio/_timeouts.py @@ -44,7 +44,8 @@ async def sleep_until(deadline): """Pause execution of the current task until the given time. The difference between :func:`sleep` and :func:`sleep_until` is that the - former takes a relative time and the latter takes an absolute time. + former takes a relative time and the latter takes an absolute time + according to Trio's internal clock (as returned by :func:`current_time`). Args: deadline (float): The time at which we should wake up again. May be in From 8ee6b9644238731c4b29c32b5cf331e450552a5d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 13 Jan 2021 06:49:06 +0000 Subject: [PATCH 0557/1498] Bump pygments from 2.7.3 to 2.7.4 Bumps [pygments](https://github.com/pygments/pygments) from 2.7.3 to 2.7.4. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.7.3...2.7.4) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 4 ++-- test-requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index ae655e95a4..8c2cc9c673 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -20,15 +20,15 @@ jinja2==2.11.2 # via sphinx, towncrier markupsafe==1.1.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in packaging==20.8 # via sphinx -pygments==2.7.3 # via sphinx +pygments==2.7.4 # via sphinx pyparsing==2.4.7 # via packaging pytz==2020.5 # via babel requests==2.25.1 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.0.0 # via sphinx sortedcontainers==2.3.0 # via -r docs-requirements.in +sphinx-rtd-theme==0.5.1 # via -r docs-requirements.in sphinx==3.3.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio -sphinx_rtd_theme==0.5.1 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index e6e68d9e8d..2988b07808 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ py==1.10.0 # via pytest pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 -pygments==2.7.3 # via ipython +pygments==2.7.4 # via ipython pylint==2.6.0 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging From a11196c2102e81b5e6c0706c208cd9dedcc364d5 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 13 Jan 2021 11:01:41 +0400 Subject: [PATCH 0558/1498] Add three random fixes noticed when releasing --- ci.sh | 2 +- docs/source/releasing.rst | 2 +- trio/_util.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ci.sh b/ci.sh index 430bfcff00..2601b1dcdd 100755 --- a/ci.sh +++ b/ci.sh @@ -30,7 +30,7 @@ function curl-harder() { # Bootstrap python environment, if necessary ################################################################ -### PyPy nightly (currently on Travis) ### +### PyPy nightly ### if [ "$PYPY_NIGHTLY_BRANCH" != "" ]; then JOB_NAME="pypy_nightly_${PYPY_NIGHTLY_BRANCH}" diff --git a/docs/source/releasing.rst b/docs/source/releasing.rst index f39aaf274f..27cee864c0 100644 --- a/docs/source/releasing.rst +++ b/docs/source/releasing.rst @@ -39,7 +39,7 @@ Things to do for releasing: * verify that all checks succeeded -* tag with vVERSION, push tag +* tag with vVERSION, push tag on ``python-trio/trio`` (not on your personal repository) * push to PyPI:: diff --git a/trio/_util.py b/trio/_util.py index 0775a107fd..ec0350b305 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -14,8 +14,6 @@ from async_generator import isasyncgen -from ._deprecate import warn_deprecated - import trio # Equivalent to the C function raise(), which Python doesn't wrap From 1c7b8691b1f0a2f4ac89b01b1bc4115ed36165f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Fri, 22 Jan 2021 01:11:28 +0700 Subject: [PATCH 0559/1498] Update dependencies for Fedora CI I guess previously python3-pip depended on something that depends on python3-devel (which contains the C headers), but apparently relying on implicit dependencies is never a good thing. --- .builds/fedora.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.builds/fedora.yml b/.builds/fedora.yml index 133cf3d40f..eddb2368a0 100644 --- a/.builds/fedora.yml +++ b/.builds/fedora.yml @@ -1,5 +1,6 @@ image: fedora/rawhide packages: + - python3-devel - python3-pip sources: - https://github.com/python-trio/trio From be6b91579d3302513cc865e837ca32a64ec65841 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 22 Jan 2021 05:35:38 +0000 Subject: [PATCH 0560/1498] Bump snowballstemmer from 2.0.0 to 2.1.0 Bumps [snowballstemmer](https://github.com/snowballstem/snowball) from 2.0.0 to 2.1.0. - [Release notes](https://github.com/snowballstem/snowball/releases) - [Changelog](https://github.com/snowballstem/snowball/blob/master/NEWS) - [Commits](https://github.com/snowballstem/snowball/compare/v2.0.0...v2.1.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 114 +++++++++++++++++++++++++++++------------- 1 file changed, 79 insertions(+), 35 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 8c2cc9c673..c5e7b66ab9 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -4,38 +4,82 @@ # # pip-compile --output-file docs-requirements.txt docs-requirements.in # -alabaster==0.7.12 # via sphinx -async-generator==1.10 # via -r docs-requirements.in -attrs==20.3.0 # via -r docs-requirements.in, outcome -babel==2.9.0 # via sphinx -certifi==2020.12.5 # via requests -chardet==4.0.0 # via requests -click==7.1.2 # via towncrier -docutils==0.16 # via sphinx -idna==2.10 # via -r docs-requirements.in, requests -imagesize==1.2.0 # via sphinx -immutables==0.14 # via -r docs-requirements.in -incremental==17.5.0 # via towncrier -jinja2==2.11.2 # via sphinx, towncrier -markupsafe==1.1.1 # via jinja2 -outcome==1.1.0 # via -r docs-requirements.in -packaging==20.8 # via sphinx -pygments==2.7.4 # via sphinx -pyparsing==2.4.7 # via packaging -pytz==2020.5 # via babel -requests==2.25.1 # via sphinx -sniffio==1.2.0 # via -r docs-requirements.in -snowballstemmer==2.0.0 # via sphinx -sortedcontainers==2.3.0 # via -r docs-requirements.in -sphinx-rtd-theme==0.5.1 # via -r docs-requirements.in -sphinx==3.3.1 # via -r docs-requirements.in, sphinx-rtd-theme, sphinxcontrib-trio -sphinxcontrib-applehelp==1.0.2 # via sphinx -sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 # via sphinx -sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -toml==0.10.2 # via towncrier -towncrier==19.2.0 # via -r docs-requirements.in -urllib3==1.26.2 # via requests +alabaster==0.7.12 + # via sphinx +async-generator==1.10 + # via -r docs-requirements.in +attrs==20.3.0 + # via + # -r docs-requirements.in + # outcome +babel==2.9.0 + # via sphinx +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +click==7.1.2 + # via towncrier +docutils==0.16 + # via sphinx +idna==2.10 + # via + # -r docs-requirements.in + # requests +imagesize==1.2.0 + # via sphinx +immutables==0.14 + # via -r docs-requirements.in +incremental==17.5.0 + # via towncrier +jinja2==2.11.2 + # via + # sphinx + # towncrier +markupsafe==1.1.1 + # via jinja2 +outcome==1.1.0 + # via -r docs-requirements.in +packaging==20.8 + # via sphinx +pygments==2.7.4 + # via sphinx +pyparsing==2.4.7 + # via packaging +pytz==2020.5 + # via babel +requests==2.25.1 + # via sphinx +sniffio==1.2.0 + # via -r docs-requirements.in +snowballstemmer==2.1.0 + # via sphinx +sortedcontainers==2.3.0 + # via -r docs-requirements.in +sphinx-rtd-theme==0.5.1 + # via -r docs-requirements.in +sphinx==3.3.1 + # via + # -r docs-requirements.in + # sphinx-rtd-theme + # sphinxcontrib-trio +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==1.0.3 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.4 + # via sphinx +sphinxcontrib-trio==1.1.2 + # via -r docs-requirements.in +toml==0.10.2 + # via towncrier +towncrier==19.2.0 + # via -r docs-requirements.in +urllib3==1.26.2 + # via requests From 4eac18ecbedfb7f887dea65b221d0b4334ae6d1f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 22 Jan 2021 06:40:37 +0000 Subject: [PATCH 0561/1498] Bump prompt-toolkit from 3.0.10 to 3.0.11 (#1870) --- test-requirements.txt | 190 ++++++++++++++++++++++++++++++------------ 1 file changed, 136 insertions(+), 54 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2988b07808..d9f7cbf1f5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,57 +4,139 @@ # # pip-compile --output-file test-requirements.txt test-requirements.in # -appdirs==1.4.4 # via black -astor==0.8.1 # via -r test-requirements.in -astroid==2.4.2 # via pylint -async-generator==1.10 # via -r test-requirements.in -attrs==20.3.0 # via -r test-requirements.in, outcome, pytest -backcall==0.2.0 # via ipython -black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.4 # via cryptography -click==7.1.2 # via black -coverage==5.3.1 # via pytest-cov -cryptography==3.3.1 # via pyopenssl, trustme -decorator==4.4.2 # via ipython -flake8==3.8.4 # via -r test-requirements.in -idna==2.10 # via -r test-requirements.in, trustme -immutables==0.14 # via -r test-requirements.in -iniconfig==1.1.1 # via pytest -ipython-genutils==0.2.0 # via traitlets -ipython==7.19.0 # via -r test-requirements.in -isort==5.7.0 # via pylint -jedi==0.18.0 # via -r test-requirements.in, ipython -lazy-object-proxy==1.4.3 # via astroid -mccabe==0.6.1 # via flake8, pylint -mypy-extensions==0.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy -mypy==0.790 ; implementation_name == "cpython" # via -r test-requirements.in -outcome==1.1.0 # via -r test-requirements.in -packaging==20.8 # via pytest -parso==0.8.1 # via jedi -pathspec==0.8.1 # via black -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.10 # via ipython -ptyprocess==0.7.0 # via pexpect -py==1.10.0 # via pytest -pycodestyle==2.6.0 # via flake8 -pycparser==2.20 # via cffi -pyflakes==2.2.0 # via flake8 -pygments==2.7.4 # via ipython -pylint==2.6.0 # via -r test-requirements.in -pyopenssl==20.0.1 # via -r test-requirements.in -pyparsing==2.4.7 # via packaging -pytest-cov==2.10.1 # via -r test-requirements.in -pytest==6.1.2 # via -r test-requirements.in, pytest-cov -regex==2020.11.13 # via black -six==1.15.0 # via astroid, cryptography, pyopenssl -sniffio==1.2.0 # via -r test-requirements.in -sortedcontainers==2.3.0 # via -r test-requirements.in -toml==0.10.2 # via black, pylint, pytest -traitlets==5.0.5 # via ipython -trustme==0.6.0 # via -r test-requirements.in -typed-ast==1.4.2 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy -typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via -r test-requirements.in, black, mypy -wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.12.1 # via astroid +appdirs==1.4.4 + # via black +astor==0.8.1 + # via -r test-requirements.in +astroid==2.4.2 + # via pylint +async-generator==1.10 + # via -r test-requirements.in +attrs==20.3.0 + # via + # -r test-requirements.in + # outcome + # pytest +backcall==0.2.0 + # via ipython +black==20.8b1 ; implementation_name == "cpython" + # via -r test-requirements.in +cffi==1.14.4 + # via cryptography +click==7.1.2 + # via black +coverage==5.3.1 + # via pytest-cov +cryptography==3.3.1 + # via + # pyopenssl + # trustme +decorator==4.4.2 + # via ipython +flake8==3.8.4 + # via -r test-requirements.in +idna==2.10 + # via + # -r test-requirements.in + # trustme +immutables==0.14 + # via -r test-requirements.in +iniconfig==1.1.1 + # via pytest +ipython-genutils==0.2.0 + # via traitlets +ipython==7.19.0 + # via -r test-requirements.in +isort==5.7.0 + # via pylint +jedi==0.18.0 + # via + # -r test-requirements.in + # ipython +lazy-object-proxy==1.4.3 + # via astroid +mccabe==0.6.1 + # via + # flake8 + # pylint +mypy-extensions==0.4.3 ; implementation_name == "cpython" + # via + # -r test-requirements.in + # black + # mypy +mypy==0.790 ; implementation_name == "cpython" + # via -r test-requirements.in +outcome==1.1.0 + # via -r test-requirements.in +packaging==20.8 + # via pytest +parso==0.8.1 + # via jedi +pathspec==0.8.1 + # via black +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pluggy==0.13.1 + # via pytest +prompt-toolkit==3.0.11 + # via ipython +ptyprocess==0.7.0 + # via pexpect +py==1.10.0 + # via pytest +pycodestyle==2.6.0 + # via flake8 +pycparser==2.20 + # via cffi +pyflakes==2.2.0 + # via flake8 +pygments==2.7.4 + # via ipython +pylint==2.6.0 + # via -r test-requirements.in +pyopenssl==20.0.1 + # via -r test-requirements.in +pyparsing==2.4.7 + # via packaging +pytest-cov==2.10.1 + # via -r test-requirements.in +pytest==6.1.2 + # via + # -r test-requirements.in + # pytest-cov +regex==2020.11.13 + # via black +six==1.15.0 + # via + # astroid + # cryptography + # pyopenssl +sniffio==1.2.0 + # via -r test-requirements.in +sortedcontainers==2.3.0 + # via -r test-requirements.in +toml==0.10.2 + # via + # black + # pylint + # pytest +traitlets==5.0.5 + # via ipython +trustme==0.6.0 + # via -r test-requirements.in +typed-ast==1.4.2 ; implementation_name == "cpython" + # via + # -r test-requirements.in + # black + # mypy +typing-extensions==3.7.4.3 ; implementation_name == "cpython" + # via + # -r test-requirements.in + # black + # mypy +wcwidth==0.2.5 + # via prompt-toolkit +wrapt==1.12.1 + # via astroid From d890affa8fed3a49b616a8fd218f4fb928bd3002 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 22 Jan 2021 06:50:12 +0000 Subject: [PATCH 0562/1498] Bump pytest-cov from 2.10.1 to 2.11.1 (#1869) --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d9f7cbf1f5..00994509a4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -100,7 +100,7 @@ pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.10.1 +pytest-cov==2.11.1 # via -r test-requirements.in pytest==6.1.2 # via From 070df40f874617574692549bb207d10e0243e843 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:37:41 +0000 Subject: [PATCH 0563/1498] Bump mypy from 0.790 to 0.800 Bumps [mypy](https://github.com/python/mypy) from 0.790 to 0.800. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.790...v0.800) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 00994509a4..9c2546c6ce 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -64,7 +64,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # -r test-requirements.in # black # mypy -mypy==0.790 ; implementation_name == "cpython" +mypy==0.800 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in From 510465744ad47082145b4e0b83837bd92a949954 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:38:31 +0000 Subject: [PATCH 0564/1498] Bump prompt-toolkit from 3.0.11 to 3.0.14 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.11 to 3.0.14. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.11...3.0.14) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 00994509a4..cdbccb3fb7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -80,7 +80,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.11 +prompt-toolkit==3.0.14 # via ipython ptyprocess==0.7.0 # via pexpect From 7708b9cf4eef0f04c1857b4f27e85af42a960537 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 25 Jan 2021 15:43:01 -0500 Subject: [PATCH 0565/1498] Add 3.9 classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 726da99365..11eda8e96f 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,7 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Topic :: System :: Networking", "Framework :: Trio", ], From faf2b031b2536359245e54339bd9301aa0165440 Mon Sep 17 00:00:00 2001 From: vlad doster Date: Mon, 25 Jan 2021 21:54:57 -0600 Subject: [PATCH 0566/1498] (docs) update README.rst - reduce verbiage - correct punctuation - strengthen intent --- README.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cc200bbcca..c4b84c1711 100644 --- a/README.rst +++ b/README.rst @@ -29,14 +29,14 @@ Trio – a friendly Python library for async concurrency and I/O :width: 200px :align: right -The Trio project's goal is to produce a production-quality, +The Trio project aims to produce a production-quality, `permissively licensed `__, async/await-native I/O library for Python. Like all async libraries, its main purpose is to help you write programs that do **multiple things at the same time** with **parallelized I/O**. A web spider that wants to fetch lots of pages in parallel, a web server that needs to -juggle lots of downloads and websocket connections at the same time, a +juggle lots of downloads and websocket connections simultaneously, a process supervisor monitoring multiple subprocesses... that sort of thing. Compared to other libraries, Trio attempts to distinguish itself with an obsessive focus on **usability** and @@ -57,11 +57,11 @@ fun. `Perhaps you'll find the same `__. This project is young and still somewhat experimental: the overall -design is solid and the existing features are fully tested and +design is solid, and the existing features are fully tested and documented, but you may encounter missing functionality or rough edges. We *do* encourage you to use it, but you should `read and subscribe to issue #1 -`__ to get warning and a +`__ to get a warning and a chance to give feedback about any compatibility-breaking changes. @@ -94,12 +94,12 @@ older library versus Trio. **Cool, but will it work on my system?** Probably! As long as you have some kind of Python 3.6-or-better (CPython or the latest PyPy3 are both fine), and are using Linux, macOS, Windows, or FreeBSD, then Trio -should definitely work. Other environments might work too, but those +will work. Other environments might work too, but those are the ones we test on. And all of our dependencies are pure Python, -except for CFFI on Windows, and that has wheels available, so +except for CFFI on Windows, which has wheels available, so installation should be easy (no C compiler needed). -**I tried it but it's not working.** Sorry to hear that! You can try +**I tried it, but it's not working.** Sorry to hear that! You can try asking for help in our `chat room `__ or `forum `__, `filing a bug @@ -108,7 +108,7 @@ question on StackOverflow `__, and we'll do our best to help you out. -**Trio is awesome and I want to help make it more awesome!** You're +**Trio is awesome, and I want to help make it more awesome!** You're the best! There's tons of work to do – filling in missing functionality, building up an ecosystem of Trio-using libraries, usability testing (e.g., maybe try teaching yourself or a friend to From df5c6d79d00f4b6455296681a81e63e429753e66 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 27 Jan 2021 05:34:53 +0000 Subject: [PATCH 0567/1498] Bump urllib3 from 1.26.2 to 1.26.3 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.2 to 1.26.3. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.3/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.2...1.26.3) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c5e7b66ab9..2c8d8448b8 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -81,5 +81,5 @@ toml==0.10.2 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in -urllib3==1.26.2 +urllib3==1.26.3 # via requests From 1495f3f1d439318928991bd31b21c3e012ce9cae Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 27 Jan 2021 05:36:16 +0000 Subject: [PATCH 0568/1498] Bump coverage from 5.3.1 to 5.4 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.3.1 to 5.4. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.3.1...coverage-5.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index eeaf445d2c..9236f547a1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ cffi==1.14.4 # via cryptography click==7.1.2 # via black -coverage==5.3.1 +coverage==5.4 # via pytest-cov cryptography==3.3.1 # via From 49fa90c95ba62e529b665d34793e22d51cc59ab6 Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Fri, 29 Jan 2021 16:57:29 -0700 Subject: [PATCH 0569/1498] WindowsIOManager.wait_overlapped returns CompletionKeyEventInfo This give access to the number of bytes transferred the I/O operation. --- trio/_core/_io_windows.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index c318d59629..1605a17c88 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -407,6 +407,7 @@ def __init__(self): # {lpOverlapped: task} self._overlapped_waiters = {} + self._overlapped_entries = {} self._posted_too_late_to_cancel = set() self._completion_key_queues = {} @@ -530,6 +531,12 @@ def process_events(self, received): elif entry.lpCompletionKey == CKeys.WAIT_OVERLAPPED: # Regular I/O event, dispatch on lpOverlapped waiter = self._overlapped_waiters.pop(entry.lpOverlapped) + overlapped = entry.lpOverlapped + transferred = entry.dwNumberOfBytesTransferred + info = CompletionKeyEventInfo( + lpOverlapped=overlapped, dwNumberOfBytesTransferred=transferred + ) + self._overlapped_entries[overlapped] = info _core.reschedule(waiter) elif entry.lpCompletionKey == CKeys.LATE_CANCEL: # Post made by a regular I/O event's abort_fn @@ -763,6 +770,7 @@ def abort(raise_cancel_): return _core.Abort.FAILED await _core.wait_task_rescheduled(abort) + info = self._overlapped_entries.pop(lpOverlapped) if lpOverlapped.Internal != 0: # the lpOverlapped reports the error as an NT status code, # which we must convert back to a Win32 error code before @@ -778,6 +786,7 @@ def abort(raise_cancel_): raise _core.ClosedResourceError("another task closed this resource") else: raise_winerror(code) + return info async def _perform_overlapped(self, handle, submit_fn): # submit_fn(lpOverlapped) submits some I/O From 5e0af362d87e769d485faae826536ffabc743872 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Tue, 12 Jan 2021 18:28:04 -0500 Subject: [PATCH 0570/1498] make sure coroutine locals are promptly destroyed --- trio/_core/_run.py | 8 ++++++++ trio/_core/tests/test_run.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index e56977f386..83b7984c8d 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2176,6 +2176,10 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): for _ in range(CONTEXT_RUN_TB_FRAMES): tb = tb.tb_next final_outcome = Error(task_exc.with_traceback(tb)) + # Remove local refs so that e.g. cancelled coroutine locals + # are not kept alive by this frame until another exception + # comes along. + # del tb, task_exc if final_outcome is not None: # We can't call this directly inside the except: blocks @@ -2183,6 +2187,10 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # themselves to other exceptions as __context__ in # unwanted ways. runner.task_exited(task, final_outcome) + # final_outcome may contain a traceback ref. It's not as + # crucial compared to the above, but will still allow more + # prompt release of resources. + final_outcome = None else: task._schedule_points += 1 if msg is CancelShieldedCheckpoint: diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 4c4e12b5df..d2bcdfd740 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -6,6 +6,7 @@ import time import types import warnings +import weakref from contextlib import contextmanager, ExitStack from math import inf from textwrap import dedent @@ -2249,3 +2250,27 @@ async def test_nursery_cancel_doesnt_create_cyclic_garbage(): finally: gc.set_debug(old_flags) gc.garbage.clear() + + +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" +) +async def test_locals_destroyed_promptly_on_cancel(): + destroyed = False + + def finalizer(): + nonlocal destroyed + destroyed = True + + class A: + pass + + async def task(): + a = A() + weakref.finalize(a, finalizer) + await _core.checkpoint() + + async with _core.open_nursery() as nursery: + nursery.start_soon(task) + nursery.cancel_scope.cancel() + assert destroyed From d04a20445ba5a5e0f536682fa1522c5aa221dbb5 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Tue, 12 Jan 2021 19:01:33 -0500 Subject: [PATCH 0571/1498] uncomment the fix --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 83b7984c8d..5dc981a0ef 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2179,7 +2179,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # Remove local refs so that e.g. cancelled coroutine locals # are not kept alive by this frame until another exception # comes along. - # del tb, task_exc + del tb, task_exc if final_outcome is not None: # We can't call this directly inside the except: blocks From 9502152c8e3e64d32da911fc36446f08008f5d7a Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 16 Jan 2021 09:55:28 -0500 Subject: [PATCH 0572/1498] sprinkle a few more dels to free as as many objects as possible before sleeping --- trio/_core/_run.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 5dc981a0ef..234db2b803 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2167,6 +2167,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): msg = task.context.run(next_send_fn, next_send) except StopIteration as stop_iteration: final_outcome = Value(stop_iteration.value) + # prevent long-lived traceback reference + del stop_iteration except BaseException as task_exc: # Store for later, removing uninteresting top frames: 1 # frame we always remove, because it's this function @@ -2188,8 +2190,8 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # unwanted ways. runner.task_exited(task, final_outcome) # final_outcome may contain a traceback ref. It's not as - # crucial compared to the above, but will still allow more - # prompt release of resources. + # crucial compared to the above, but this will allow more + # prompt release of resources in coroutine locals. final_outcome = None else: task._schedule_points += 1 @@ -2219,10 +2221,13 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # which works for at least asyncio and curio. runner.reschedule(task, exc) task._next_send_fn = task.coro.throw + del msg if "after_task_step" in runner.instruments: runner.instruments.call("after_task_step", task) del GLOBAL_RUN_CONTEXT.task + # prevent long-lived task reference + del task, next_send, next_send_fn except GeneratorExit: # The run-loop generator has been garbage collected without finishing From 69f2de14bb1417690fceacd05d030d3e640fb320 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 16 Jan 2021 10:06:50 -0500 Subject: [PATCH 0573/1498] Add newsfragment --- newsfragments/1864.bugfix.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/1864.bugfix.rst diff --git a/newsfragments/1864.bugfix.rst b/newsfragments/1864.bugfix.rst new file mode 100644 index 0000000000..440984a091 --- /dev/null +++ b/newsfragments/1864.bugfix.rst @@ -0,0 +1,2 @@ +The event loop now holds on to references of coroutine frames for only +the minimum necessary period of time. \ No newline at end of file From daf2b36cb53fb7ae8a835da7937bb11a8809a9bb Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Fri, 29 Jan 2021 11:02:06 -0500 Subject: [PATCH 0574/1498] exception variables assigned in except block are auto-deleted at the end of the block --- trio/_core/_run.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 234db2b803..14a61933db 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2167,8 +2167,6 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): msg = task.context.run(next_send_fn, next_send) except StopIteration as stop_iteration: final_outcome = Value(stop_iteration.value) - # prevent long-lived traceback reference - del stop_iteration except BaseException as task_exc: # Store for later, removing uninteresting top frames: 1 # frame we always remove, because it's this function @@ -2181,7 +2179,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # Remove local refs so that e.g. cancelled coroutine locals # are not kept alive by this frame until another exception # comes along. - del tb, task_exc + del tb if final_outcome is not None: # We can't call this directly inside the except: blocks From 8170b42faa9b45c415428a7ce5a6c449c9042df4 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 30 Jan 2021 22:09:41 -0500 Subject: [PATCH 0575/1498] Document missing deletion tests --- trio/_core/_run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 14a61933db..807e330c1d 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2219,12 +2219,15 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): # which works for at least asyncio and curio. runner.reschedule(task, exc) task._next_send_fn = task.coro.throw + # prevent long-lived reference + # TODO: develop test for this deletion del msg if "after_task_step" in runner.instruments: runner.instruments.call("after_task_step", task) del GLOBAL_RUN_CONTEXT.task - # prevent long-lived task reference + # prevent long-lived references + # TODO: develop test for these deletions del task, next_send, next_send_fn except GeneratorExit: From 551d1b723f2b12d598cd94ebc43cdb4b15329d9d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 05:54:26 +0000 Subject: [PATCH 0576/1498] Bump packaging from 20.8 to 20.9 Bumps [packaging](https://github.com/pypa/packaging) from 20.8 to 20.9. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/20.8...20.9) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 2c8d8448b8..bbc6566b7f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -40,7 +40,7 @@ markupsafe==1.1.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in -packaging==20.8 +packaging==20.9 # via sphinx pygments==2.7.4 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index 9236f547a1..d2e199f2e2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -68,7 +68,7 @@ mypy==0.800 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in -packaging==20.8 +packaging==20.9 # via pytest parso==0.8.1 # via jedi From fedc4c4f769be29a65cc214447a88ff0552d270f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 06:03:24 +0000 Subject: [PATCH 0577/1498] Bump jinja2 from 2.11.2 to 2.11.3 Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.2 to 2.11.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.11.2...2.11.3) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index bbc6566b7f..efd8a53e60 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -32,7 +32,7 @@ immutables==0.14 # via -r docs-requirements.in incremental==17.5.0 # via towncrier -jinja2==2.11.2 +jinja2==2.11.3 # via # sphinx # towncrier From 502e20cea74214c0e72d4e95bb66026095f38d8c Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Mon, 1 Feb 2021 15:25:48 -0700 Subject: [PATCH 0578/1498] use changes recommended by oremanj --- trio/_core/_io_windows.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 1605a17c88..6d3994499f 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import attr +from outcome import Value from .. import _core from ._run import _public @@ -407,7 +408,6 @@ def __init__(self): # {lpOverlapped: task} self._overlapped_waiters = {} - self._overlapped_entries = {} self._posted_too_late_to_cancel = set() self._completion_key_queues = {} @@ -536,8 +536,7 @@ def process_events(self, received): info = CompletionKeyEventInfo( lpOverlapped=overlapped, dwNumberOfBytesTransferred=transferred ) - self._overlapped_entries[overlapped] = info - _core.reschedule(waiter) + _core.reschedule(waiter, Value(info)) elif entry.lpCompletionKey == CKeys.LATE_CANCEL: # Post made by a regular I/O event's abort_fn # after it failed to cancel the I/O. If we still @@ -769,8 +768,7 @@ def abort(raise_cancel_): ) from exc return _core.Abort.FAILED - await _core.wait_task_rescheduled(abort) - info = self._overlapped_entries.pop(lpOverlapped) + info = await _core.wait_task_rescheduled(abort) if lpOverlapped.Internal != 0: # the lpOverlapped reports the error as an NT status code, # which we must convert back to a Win32 error code before From 1e7c3deaff13d6d8875f3940bf8cf757683fd375 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Feb 2021 05:22:00 +0000 Subject: [PATCH 0579/1498] Bump pytz from 2020.5 to 2021.1 Bumps [pytz](https://github.com/stub42/pytz) from 2020.5 to 2021.1. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2020.5...release_2021.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index efd8a53e60..df20e7de60 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -46,7 +46,7 @@ pygments==2.7.4 # via sphinx pyparsing==2.4.7 # via packaging -pytz==2020.5 +pytz==2021.1 # via babel requests==2.25.1 # via sphinx From 12b6c800150483f299cbee5d538a160beaa4fe1b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Feb 2021 05:22:53 +0000 Subject: [PATCH 0580/1498] Bump ipython from 7.19.0 to 7.20.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.19.0 to 7.20.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.19.0...7.20.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d2e199f2e2..4c495569e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.19.0 +ipython==7.20.0 # via -r test-requirements.in isort==5.7.0 # via pylint From 5181d38a6d484ace1cafff4ee1426f7f108ccb39 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 4 Feb 2021 10:43:50 -0500 Subject: [PATCH 0581/1498] Add QTrio to the awesome list --- docs/source/awesome-trio-libraries.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index e31ba608b5..09ecb3e222 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -61,6 +61,11 @@ Building Command Line Apps * `urwid `__ - Urwid is a console user interface library for Python. +Building GUI Apps +----------------- +* `QTrio `__ - Integration between Trio and either the PyQt or PySide Qt wrapper libraries. Uses Trio's :ref:`guest mode `. + + Multi-Core/Multiprocessing -------------------------- * `tractor `__ - An experimental, trionic (aka structured concurrent) "actor model" for distributed multi-core Python. From f9d91f0f0983a218c19dba7be3a4fdcdab928f05 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 4 Feb 2021 11:07:29 -0500 Subject: [PATCH 0582/1498] Reference QTrio's docs instead of repo --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 09ecb3e222..e9228d4cd9 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -63,7 +63,7 @@ Building Command Line Apps Building GUI Apps ----------------- -* `QTrio `__ - Integration between Trio and either the PyQt or PySide Qt wrapper libraries. Uses Trio's :ref:`guest mode `. +* `QTrio `__ - Integration between Trio and either the PyQt or PySide Qt wrapper libraries. Uses Trio's :ref:`guest mode `. Multi-Core/Multiprocessing From fd2224193ebae779a5a6375878644f4f7c8244e3 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 6 Feb 2021 12:45:01 -0500 Subject: [PATCH 0583/1498] Add trio-parallel to the awesome list --- docs/source/awesome-trio-libraries.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index e9228d4cd9..88db0d93b9 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -70,6 +70,7 @@ Multi-Core/Multiprocessing -------------------------- * `tractor `__ - An experimental, trionic (aka structured concurrent) "actor model" for distributed multi-core Python. * `Trio run_in_process `__ - Trio based API for running code in a separate process. +* `trio-parallel `__ - CPU parallelism for Trio RPC From 2b1e34174907dc7217c1289dc31529fb18997f1f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Feb 2021 05:59:13 +0000 Subject: [PATCH 0584/1498] Bump cryptography from 3.3.1 to 3.4.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.3.1 to 3.4.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.3.1...3.4.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4c495569e6..5b146099d8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -27,7 +27,7 @@ click==7.1.2 # via black coverage==5.4 # via pytest-cov -cryptography==3.3.1 +cryptography==3.4.1 # via # pyopenssl # trustme @@ -108,10 +108,13 @@ pytest==6.1.2 # pytest-cov regex==2020.11.13 # via black +semantic-version==2.8.5 + # via setuptools-rust +setuptools-rust==0.11.6 + # via cryptography six==1.15.0 # via # astroid - # cryptography # pyopenssl sniffio==1.2.0 # via -r test-requirements.in @@ -122,6 +125,7 @@ toml==0.10.2 # black # pylint # pytest + # setuptools-rust traitlets==5.0.5 # via ipython trustme==0.6.0 From 087faa249cfcd4b5e4435e87c91d090b775ae5cf Mon Sep 17 00:00:00 2001 From: Jay Satiro Date: Mon, 8 Feb 2021 04:13:48 -0500 Subject: [PATCH 0585/1498] notes-to-self: fix table in how-does-windows-so-reuseaddr-work.py - Generate the table headers dynamically. Prior to this change the table headers were static and incorrect, 'specific' and 'wildcard' labels were reversed. Example output from before the change (incorrect): ~~~ | default | | specific| wildcard| --------------------- default | wildcard | INUSE | Success | default | wildcard | Success | INUSE | ~~~ Example output from after the change (correct): ~~~ | default | | wildcard | specific | ----------------------- default | wildcard | INUSE | Success | default | wildcard | Success | INUSE | ~~~ Bug: https://github.com/python-trio/trio/issues/928#issuecomment-774630606 Closes #xxxx --- .../how-does-windows-so-reuseaddr-work.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/notes-to-self/how-does-windows-so-reuseaddr-work.py b/notes-to-self/how-does-windows-so-reuseaddr-work.py index 64430a0f92..4865ea17b3 100644 --- a/notes-to-self/how-does-windows-so-reuseaddr-work.py +++ b/notes-to-self/how-does-windows-so-reuseaddr-work.py @@ -43,10 +43,17 @@ def table_entry(mode1, bind_type1, mode2, bind_type2): print(""" second bind - | default | SO_REUSEADDR | SO_EXCLUSIVEADDRUSE - | specific| wildcard| specific| wildcard| specific| wildcard -first bind ------------------------------------------------------------""" -# default | wildcard | INUSE | Success | ACCESS | Success | INUSE | Success + | """ ++ " | ".join(["%-19s" % mode for mode in modes]) +) + +print(""" """, end='') +for mode in modes: + print(" | " + " | ".join(["%8s" % bind_type for bind_type in bind_types]), end='') + +print(""" +first bind -----------------------------------------------------------------""" +# default | wildcard | INUSE | Success | ACCESS | Success | INUSE | Success ) for i, mode1 in enumerate(modes): @@ -58,4 +65,4 @@ def table_entry(mode1, bind_type1, mode2, bind_type2): row.append(entry) #print(mode1, bind_type1, mode2, bind_type2, entry) print("{:>19} | {:>8} | ".format(mode1, bind_type1) - + " | ".join(["%7s" % entry for entry in row])) + + " | ".join(["%8s" % entry for entry in row])) From e4f470ce45ac3fad062f730b23a1a95cd9e4e4bc Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 8 Feb 2021 13:45:50 +0400 Subject: [PATCH 0586/1498] Install rust in Alpine/FreeBSD to build cryptography --- .builds/alpine.yml | 1 + .builds/freebsd.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.builds/alpine.yml b/.builds/alpine.yml index f69280d852..7c67a4d7f5 100644 --- a/.builds/alpine.yml +++ b/.builds/alpine.yml @@ -6,6 +6,7 @@ packages: - musl-dev - openssl-dev - python3-dev + - rust # required to build cryptography sources: - https://github.com/python-trio/trio tasks: diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 0459ed8f71..2f55cac0e9 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -3,6 +3,7 @@ packages: - curl - python39 - py39-sqlite3 + - rust # required to build cryptography sources: - https://github.com/python-trio/trio tasks: From b228d9f72d3e604ecb0a96605a14c37049642647 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 8 Feb 2021 13:51:06 +0400 Subject: [PATCH 0587/1498] Install cargo in Alpine too --- .builds/alpine.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.builds/alpine.yml b/.builds/alpine.yml index 7c67a4d7f5..d4c60f6d7d 100644 --- a/.builds/alpine.yml +++ b/.builds/alpine.yml @@ -6,7 +6,9 @@ packages: - musl-dev - openssl-dev - python3-dev - - rust # required to build cryptography + # required to build cryptography + - rust + - cargo sources: - https://github.com/python-trio/trio tasks: From 8f9048d51dd37df71e5dad4fba6d71beff4b8684 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 9 Feb 2021 05:23:04 +0000 Subject: [PATCH 0588/1498] Bump cryptography from 3.4.1 to 3.4.3 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.1 to 3.4.3. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.1...3.4.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5b146099d8..954cd03f4b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -27,7 +27,7 @@ click==7.1.2 # via black coverage==5.4 # via pytest-cov -cryptography==3.4.1 +cryptography==3.4.3 # via # pyopenssl # trustme @@ -108,10 +108,6 @@ pytest==6.1.2 # pytest-cov regex==2020.11.13 # via black -semantic-version==2.8.5 - # via setuptools-rust -setuptools-rust==0.11.6 - # via cryptography six==1.15.0 # via # astroid @@ -125,7 +121,6 @@ toml==0.10.2 # black # pylint # pytest - # setuptools-rust traitlets==5.0.5 # via ipython trustme==0.6.0 From 9451776ab3d5a55c3a3372a82c7f8ed593ef40a9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 9 Feb 2021 05:31:02 +0000 Subject: [PATCH 0589/1498] Bump pytest from 6.1.2 to 6.2.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.1.2 to 6.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.1.2...6.2.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 954cd03f4b..fcd313825e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -102,7 +102,7 @@ pyparsing==2.4.7 # via packaging pytest-cov==2.11.1 # via -r test-requirements.in -pytest==6.1.2 +pytest==6.2.2 # via # -r test-requirements.in # pytest-cov From 5f151d13a12545d730fe6dece72d31c5ddc318e9 Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Tue, 9 Feb 2021 07:31:09 +0100 Subject: [PATCH 0590/1498] Additions to the Trio-Awesome list * asyncakumuli * asyncowfs * DistKV (already mentioned in DistMQTT) Also updated the URL for DistKV to point to the MoaT project on github. --- docs/source/awesome-trio-libraries.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 88db0d93b9..f09e8663e2 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -45,14 +45,15 @@ Database * `sqlalchemy_aio `__ - Add asyncio and Trio support to SQLAlchemy core, derived from alchimia. * `redio `__ - Redis client, pure Python and Trio. * `trio_redis `__ - A Redis client for Trio. Depends on hiredis-py. +* `asyncakumuli `__ - Client for the `Akumuli `__ time series database. IOT --- -* `DistMQTT `__ - DistMQTT is an open source MQTT client and broker implementation. It is a fork of hbmqtt with support for anyio and DistKV. +* `DistMQTT `__ - DistMQTT is an open source MQTT client and broker implementation. It is a fork of hbmqtt with support for anyio and DistKV. * `asyncgpio `__ - Allows easy access to the GPIO pins on your Raspberry Pi or similar embedded computer. - - +* `asyncowfs `__ - High-level, object-oriented access to 1wire sensors and actors. +* `DistKV `__ - a persistent, distributed, master-less key/value storage with async notification and some IoT-related plug-ins. Building Command Line Apps From 05d15113043a5a22e45fe94e0349556462638975 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 9 Feb 2021 10:26:44 -0500 Subject: [PATCH 0591/1498] Just ignore the new pytest unraisable exception checks for now --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 90b7a189bb..c41a227af9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,5 @@ faulthandler_timeout=60 markers = redistributors_should_skip: tests that should be skipped by downstream redistributors junit_family = xunit2 +addopts = + -p no:unraisableexception From 5eb44343d3e281dba43a9bdb9cb3c596f5b5b4ce Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 9 Feb 2021 10:31:30 -0500 Subject: [PATCH 0592/1498] also -p no:threadexception --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index c41a227af9..6f79c6eab9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,3 +6,4 @@ markers = junit_family = xunit2 addopts = -p no:unraisableexception + -p no:threadexception From 251d2c1d7255df89ebd7d0f1c304a7a1d039e619 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 10 Feb 2021 05:19:57 +0000 Subject: [PATCH 0593/1498] Bump immutables from 0.14 to 0.15 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.14 to 0.15. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.14...v0.15) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index df20e7de60..7d6220f8c0 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -28,7 +28,7 @@ idna==2.10 # requests imagesize==1.2.0 # via sphinx -immutables==0.14 +immutables==0.15 # via -r docs-requirements.in incremental==17.5.0 # via towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 954cd03f4b..4f1ca7ac2b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,7 +39,7 @@ idna==2.10 # via # -r test-requirements.in # trustme -immutables==0.14 +immutables==0.15 # via -r test-requirements.in iniconfig==1.1.1 # via pytest From 38cbfd91760eef50ad638a2ef3c8f7087438241b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 10 Feb 2021 05:20:42 +0000 Subject: [PATCH 0594/1498] Bump cryptography from 3.4.3 to 3.4.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.3 to 3.4.4. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.3...3.4.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 954cd03f4b..543eed169b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -27,7 +27,7 @@ click==7.1.2 # via black coverage==5.4 # via pytest-cov -cryptography==3.4.3 +cryptography==3.4.4 # via # pyopenssl # trustme From 633830718fcb8003d5a53c2fdd0f34ff86c3aa3b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 11 Feb 2021 05:22:54 +0000 Subject: [PATCH 0595/1498] Bump trustme from 0.6.0 to 0.7.0 Bumps [trustme](https://github.com/python-trio/trustme) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/python-trio/trustme/releases) - [Commits](https://github.com/python-trio/trustme/compare/v0.6.0...v0.7.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 14132459c5..189b5e1c83 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -123,7 +123,7 @@ toml==0.10.2 # pytest traitlets==5.0.5 # via ipython -trustme==0.6.0 +trustme==0.7.0 # via -r test-requirements.in typed-ast==1.4.2 ; implementation_name == "cpython" # via From 7162a5fa45d4b663ac7122ec487b0b1bef2d1231 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 11 Feb 2021 05:24:02 +0000 Subject: [PATCH 0596/1498] Bump prompt-toolkit from 3.0.14 to 3.0.15 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.14 to 3.0.15. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.14...3.0.15) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 14132459c5..b5a6c24b19 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -80,7 +80,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.14 +prompt-toolkit==3.0.15 # via ipython ptyprocess==0.7.0 # via pexpect From d7a889d0861e0d072a69c1c0b873c9b9b5c2dc2e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 12 Feb 2021 05:35:01 +0000 Subject: [PATCH 0597/1498] Bump cffi from 1.14.4 to 1.14.5 Bumps [cffi](https://github.com/python-cffi/release-doc) from 1.14.4 to 1.14.5. - [Release notes](https://github.com/python-cffi/release-doc/releases) - [Commits](https://github.com/python-cffi/release-doc/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 942eb56607..97f7cfdf05 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ backcall==0.2.0 # via ipython black==20.8b1 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.4 +cffi==1.14.5 # via cryptography click==7.1.2 # via black From 2a60d421202707ccdcebbcb343740ea9768ddd50 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 12 Feb 2021 05:36:26 +0000 Subject: [PATCH 0598/1498] Bump prompt-toolkit from 3.0.15 to 3.0.16 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.15 to 3.0.16. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.15...3.0.16) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 942eb56607..e0f442de61 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -80,7 +80,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.15 +prompt-toolkit==3.0.16 # via ipython ptyprocess==0.7.0 # via pexpect From 17d784c05d20e0c2ef45c39c52d4f9966ab32b0b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 16 Feb 2021 05:26:34 +0000 Subject: [PATCH 0599/1498] Bump astroid from 2.4.2 to 2.5 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.4.2 to 2.5. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/astroid-2.4.2...astroid-2.5) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1a974170a6..2eb7b9b12b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.4.2 +astroid==2.5 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -109,9 +109,7 @@ pytest==6.2.2 regex==2020.11.13 # via black six==1.15.0 - # via - # astroid - # pyopenssl + # via pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.3.0 From 6a7300bac562f22479593b201b3554b7ce2342d6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 17 Feb 2021 05:33:13 +0000 Subject: [PATCH 0600/1498] Bump cryptography from 3.4.4 to 3.4.6 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.4 to 3.4.6. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.4...3.4.6) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1a974170a6..1b7e14935c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -27,7 +27,7 @@ click==7.1.2 # via black coverage==5.4 # via pytest-cov -cryptography==3.4.4 +cryptography==3.4.6 # via # pyopenssl # trustme From 75b5053b7729fd24870eec80fcab0031307e28d2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 17 Feb 2021 05:33:47 +0000 Subject: [PATCH 0601/1498] Bump pylint from 2.6.0 to 2.6.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.6.0 to 2.6.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.6.0...pylint-2.6.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1a974170a6..8bd6d45fbd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -94,7 +94,7 @@ pyflakes==2.2.0 # via flake8 pygments==2.7.4 # via ipython -pylint==2.6.0 +pylint==2.6.2 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From fd45d4d8b7694715962b26db948417c02c6cab0b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 11 Feb 2021 11:20:57 +0000 Subject: [PATCH 0602/1498] reenable pytest -p unraisableexception and -p threadexception --- pyproject.toml | 7 +++++ setup.cfg | 9 ------ trio/_core/tests/test_asyncgen.py | 42 +++++++++++++++------------ trio/_core/tests/test_guest_mode.py | 4 ++- trio/_core/tests/test_run.py | 4 +++ trio/_core/tests/test_thread_cache.py | 36 ++++++++++++++++------- trio/_core/tests/tutil.py | 35 ++++++++++++++++++++++ 7 files changed, 97 insertions(+), 40 deletions(-) delete mode 100644 setup.cfg diff --git a/pyproject.toml b/pyproject.toml index 8597a4f545..1378e5df7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,10 @@ directory = "misc" name = "Miscellaneous internal changes" showcontent = true +[tool.pytest.ini_options] +addopts = ["--strict-markers", "--strict-config"] +xfail_strict = true +faulthandler_timeout = 60 +markers = ["redistributors_should_skip: tests that should be skipped by downstream redistributors"] +junit_family = "xunit2" +filterwarnings = ["error"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6f79c6eab9..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[tool:pytest] -xfail_strict = true -faulthandler_timeout=60 -markers = - redistributors_should_skip: tests that should be skipped by downstream redistributors -junit_family = xunit2 -addopts = - -p no:unraisableexception - -p no:threadexception diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index 1f886e11ab..6ce0af366f 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -5,7 +5,7 @@ from functools import partial from async_generator import aclosing from ... import _core -from .tutil import gc_collect_harder, buggy_pypy_asyncgens +from .tutil import gc_collect_harder, buggy_pypy_asyncgens, restore_unraisablehook def test_asyncgen_basics(): @@ -94,8 +94,9 @@ async def agen(): record.append("crashing") raise ValueError("oops") - await agen().asend(None) - gc_collect_harder() + with restore_unraisablehook(): + await agen().asend(None) + gc_collect_harder() await _core.wait_all_tasks_blocked() assert record == ["crashing"] exc_type, exc_value, exc_traceback = caplog.records[0].exc_info @@ -170,6 +171,7 @@ async def async_main(): assert record == ["innermost"] + list(range(100)) +@restore_unraisablehook() def test_last_minute_gc_edge_case(): saved = [] record = [] @@ -267,17 +269,18 @@ async def awaits_after_yield(): yield 42 await _core.cancel_shielded_checkpoint() - await step_outside_async_context(well_behaved()) - gc_collect_harder() - assert capsys.readouterr().err == "" + with restore_unraisablehook(): + await step_outside_async_context(well_behaved()) + gc_collect_harder() + assert capsys.readouterr().err == "" - await step_outside_async_context(yields_after_yield()) - gc_collect_harder() - assert "ignored GeneratorExit" in capsys.readouterr().err + await step_outside_async_context(yields_after_yield()) + gc_collect_harder() + assert "ignored GeneratorExit" in capsys.readouterr().err - await step_outside_async_context(awaits_after_yield()) - gc_collect_harder() - assert "awaited something during finalization" in capsys.readouterr().err + await step_outside_async_context(awaits_after_yield()) + gc_collect_harder() + assert "awaited something during finalization" in capsys.readouterr().err @pytest.mark.skipif(buggy_pypy_asyncgens, reason="pypy 7.2.0 is buggy") @@ -307,10 +310,11 @@ async def async_main(): await _core.wait_all_tasks_blocked() assert record == ["trio collected ours"] - old_hooks = sys.get_asyncgen_hooks() - sys.set_asyncgen_hooks(my_firstiter, my_finalizer) - try: - _core.run(async_main) - finally: - assert sys.get_asyncgen_hooks() == (my_firstiter, my_finalizer) - sys.set_asyncgen_hooks(*old_hooks) + with restore_unraisablehook(): + old_hooks = sys.get_asyncgen_hooks() + sys.set_asyncgen_hooks(my_firstiter, my_finalizer) + try: + _core.run(async_main) + finally: + assert sys.get_asyncgen_hooks() == (my_firstiter, my_finalizer) + sys.set_asyncgen_hooks(*old_hooks) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index c9701e7cdd..0184ff3103 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -13,7 +13,7 @@ import trio import trio.testing -from .tutil import gc_collect_harder, buggy_pypy_asyncgens +from .tutil import gc_collect_harder, buggy_pypy_asyncgens, restore_unraisablehook from ..._util import signal_raise # The simplest possible "host" loop. @@ -277,6 +277,7 @@ def after_io_wait(self, timeout): assert trivial_guest_run(trio_main) == "ok" +@restore_unraisablehook() def test_guest_warns_if_abandoned(): # This warning is emitted from the garbage collector. So we have to make # sure that our abandoned run is garbage. The easiest way to do this is to @@ -506,6 +507,7 @@ async def trio_main(in_host): sys.implementation.name == "pypy" and sys.version_info >= (3, 7), reason="async generator issue under investigation", ) +@restore_unraisablehook() def test_guest_mode_asyncgens(): import sniffio diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index d2bcdfd740..8d92e6cec4 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -23,6 +23,7 @@ gc_collect_harder, ignore_coroutine_never_awaited_warnings, buggy_pypy_asyncgens, + restore_unraisablehook, ) from ... import _core @@ -844,6 +845,7 @@ async def stubborn_sleeper(): assert record == ["sleep", "woke", "cancelled"] +@restore_unraisablehook() def test_broken_abort(): async def main(): # These yields are here to work around an annoying warning -- we're @@ -870,6 +872,7 @@ async def main(): gc_collect_harder() +@restore_unraisablehook() def test_error_in_run_loop(): # Blow stuff up real good to check we at least get a TrioInternalError async def main(): @@ -2154,6 +2157,7 @@ def abort_fn(_): assert abort_fn_called +@restore_unraisablehook() def test_async_function_implemented_in_C(): # These used to crash because we'd try to mutate the coroutine object's # cr_frame, but C functions don't have Python frames. diff --git a/trio/_core/tests/test_thread_cache.py b/trio/_core/tests/test_thread_cache.py index 0f6e0a0715..d60288b3c1 100644 --- a/trio/_core/tests/test_thread_cache.py +++ b/trio/_core/tests/test_thread_cache.py @@ -3,8 +3,9 @@ from queue import Queue import time import sys +from contextlib import contextmanager -from .tutil import slow, gc_collect_harder +from .tutil import slow, gc_collect_harder, disable_threading_excepthook from .. import _thread_cache from .._thread_cache import start_thread_soon, ThreadCache @@ -99,6 +100,17 @@ def test_idle_threads_exit(monkeypatch): assert not seen_thread.is_alive() +@contextmanager +def _join_started_threads(): + before = frozenset(threading.enumerate()) + try: + yield + finally: + for thread in threading.enumerate(): + if thread not in before: + thread.join() + + def test_race_between_idle_exit_and_job_assignment(monkeypatch): # This is a lock where the first few times you try to acquire it with a # timeout, it waits until the lock is available and then pretends to time @@ -138,13 +150,15 @@ def release(self): monkeypatch.setattr(_thread_cache, "Lock", JankyLock) - tc = ThreadCache() - done = threading.Event() - tc.start_thread_soon(lambda: None, lambda _: done.set()) - done.wait() - # Let's kill the thread we started, so it doesn't hang around until the - # test suite finishes. Doesn't really do any harm, but it can be confusing - # to see it in debug output. This is hacky, and leaves our ThreadCache - # object in an inconsistent state... but it doesn't matter, because we're - # not going to use it again anyway. - tc.start_thread_soon(lambda: None, lambda _: sys.exit()) + with disable_threading_excepthook(), _join_started_threads(): + tc = ThreadCache() + done = threading.Event() + tc.start_thread_soon(lambda: None, lambda _: done.set()) + done.wait() + # Let's kill the thread we started, so it doesn't hang around until the + # test suite finishes. Doesn't really do any harm, but it can be confusing + # to see it in debug output. This is hacky, and leaves our ThreadCache + # object in an inconsistent state... but it doesn't matter, because we're + # not going to use it again anyway. + + tc.start_thread_soon(lambda: None, lambda _: sys.exit()) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 00669e883e..7ab7aadfe5 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -1,5 +1,6 @@ # Utilities for testing import socket as stdlib_socket +import threading import os import sys from typing import TYPE_CHECKING @@ -80,6 +81,40 @@ def ignore_coroutine_never_awaited_warnings(): gc_collect_harder() +def _noop(*args, **kwargs): + pass + + +if sys.version_info >= (3, 8): + + @contextmanager + def restore_unraisablehook(): + sys.unraisablehook, prev = sys.__unraisablehook__, sys.unraisablehook + try: + yield + finally: + sys.unraisablehook = prev + + @contextmanager + def disable_threading_excepthook(): + threading.excepthook, prev = _noop, threading.excepthook + try: + yield + finally: + threading.excephtook = prev + + +else: + + @contextmanager + def restore_unraisablehook(): + yield + + @contextmanager + def disable_threading_excepthook(): + yield + + # template is like: # [1, {2.1, 2.2}, 3] -> matches [1, 2.1, 2.2, 3] or [1, 2.2, 2.1, 3] def check_sequence_matches(seq, template): From 77d11d995bfb859fa9e122564081daed0d5ae329 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 11 Feb 2021 11:28:25 +0000 Subject: [PATCH 0603/1498] pytest config now in pyproject.toml --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 2601b1dcdd..499868ba45 100755 --- a/ci.sh +++ b/ci.sh @@ -143,7 +143,7 @@ else cd empty INSTALLDIR=$(python -c "import os, trio; print(os.path.dirname(trio.__file__))") - cp ../setup.cfg $INSTALLDIR + cp ../pyproject.toml $INSTALLDIR # We have to copy .coveragerc into this directory, rather than passing # --cov-config=../.coveragerc to pytest, because codecov.sh will run # 'coverage xml' to generate the report that it uses, and that will only From ef3e52fdeaab47eb6cf7e0f4e461937421da51dd Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 11 Feb 2021 11:41:19 +0000 Subject: [PATCH 0604/1498] fix windows tests that trigger unraisablehook --- trio/_core/tests/test_windows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/test_windows.py b/trio/_core/tests/test_windows.py index e6bab82204..bd81ef0f33 100644 --- a/trio/_core/tests/test_windows.py +++ b/trio/_core/tests/test_windows.py @@ -8,7 +8,7 @@ # Mark all the tests in this file as being windows-only pytestmark = pytest.mark.skipif(not on_windows, reason="windows only") -from .tutil import slow, gc_collect_harder +from .tutil import slow, gc_collect_harder, restore_unraisablehook from ... import _core, sleep, move_on_after from ...testing import wait_all_tasks_blocked @@ -111,6 +111,7 @@ def pipe_with_overlapped_read(): kernel32.CloseHandle(ffi.cast("HANDLE", write_handle)) +@restore_unraisablehook() def test_forgot_to_register_with_iocp(): with pipe_with_overlapped_read() as (write_fp, read_handle): with write_fp: From 6273b1ec6cf21db6c676d1ef23c5c25616444c0a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 17 Feb 2021 11:51:19 +0000 Subject: [PATCH 0605/1498] pragma: no cover lines erroneously considered not coverred --- trio/_core/tests/tutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 7ab7aadfe5..8ed4380842 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -107,11 +107,11 @@ def disable_threading_excepthook(): else: @contextmanager - def restore_unraisablehook(): + def restore_unraisablehook(): # pragma: no cover yield @contextmanager - def disable_threading_excepthook(): + def disable_threading_excepthook(): # pragma: no cover yield From cb75b22ab01c629cedacb637ecc713a18334a570 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 21 Feb 2021 00:10:34 +0000 Subject: [PATCH 0606/1498] Update trio/_core/tests/tutil.py Co-authored-by: Kyle Altendorf --- trio/_core/tests/tutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 8ed4380842..7f51750869 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -101,7 +101,7 @@ def disable_threading_excepthook(): try: yield finally: - threading.excephtook = prev + threading.excepthook = prev else: From 9a95a5b2e86214b71428afd1198ecf6cbdf0674f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Feb 2021 05:20:25 +0000 Subject: [PATCH 0607/1498] Bump pylint from 2.6.2 to 2.7.0 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.6.2 to 2.7.0. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.6.2...pylint-2.7.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index cd74623138..3587743055 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5 +astroid==2.5.0 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -94,7 +94,7 @@ pyflakes==2.2.0 # via flake8 pygments==2.7.4 # via ipython -pylint==2.6.2 +pylint==2.7.0 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 12cac74ce9f64b7224ba0abc57e110e30f9227e7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 22 Feb 2021 05:28:29 +0000 Subject: [PATCH 0608/1498] Bump mypy from 0.800 to 0.812 Bumps [mypy](https://github.com/python/mypy) from 0.800 to 0.812. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.800...v0.812) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3587743055..b5e858e1c2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -64,7 +64,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # -r test-requirements.in # black # mypy -mypy==0.800 ; implementation_name == "cpython" +mypy==0.812 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in From 67ac323464be4ba2b4021f3dfc62f00583644bca Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 23 Feb 2021 05:21:31 +0000 Subject: [PATCH 0609/1498] Bump lazy-object-proxy from 1.4.3 to 1.5.2 Bumps [lazy-object-proxy](https://github.com/ionelmc/python-lazy-object-proxy) from 1.4.3 to 1.5.2. - [Release notes](https://github.com/ionelmc/python-lazy-object-proxy/releases) - [Changelog](https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-lazy-object-proxy/compare/v1.4.3...v1.5.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b5e858e1c2..2df523de07 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,7 +53,7 @@ jedi==0.18.0 # via # -r test-requirements.in # ipython -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.5.2 # via astroid mccabe==0.6.1 # via From 0e08888ded40af70395989535e28f73710cf0621 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 24 Feb 2021 05:28:34 +0000 Subject: [PATCH 0610/1498] Bump pylint from 2.7.0 to 2.7.1 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.7.0 to 2.7.1. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.7.0...pylint-2.7.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2df523de07..5c7dc6497e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -94,7 +94,7 @@ pyflakes==2.2.0 # via flake8 pygments==2.7.4 # via ipython -pylint==2.7.0 +pylint==2.7.1 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From ed869accdd1428c9cca452fdbe64b643d5662874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Thu, 25 Feb 2021 17:49:45 +0200 Subject: [PATCH 0611/1498] Added the "compact" keyword argument to TracebackException.__init__() This is a requirement to achieve Python 3.10 compatibility. --- trio/_core/_multierror.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 13fc3f3d0f..54295e71d5 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -381,11 +381,17 @@ def traceback_exception_init( limit=None, lookup_lines=True, capture_locals=False, + compact=False, _seen=None, ): if _seen is None: _seen = set() + if sys.version_info >= (3, 10): + kwargs = {"compact": compact} + else: + kwargs = {} + # Capture the original exception and its cause and context as TracebackExceptions traceback_exception_original_init( self, @@ -396,6 +402,7 @@ def traceback_exception_init( lookup_lines=lookup_lines, capture_locals=capture_locals, _seen=_seen, + **kwargs, ) # Capture each of the exceptions in the MultiError along with each of their causes and contexts From a907959c8fedabfe9e5b2c0ce782e871af58c982 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Mar 2021 07:11:33 +0000 Subject: [PATCH 0612/1498] Bump ipython from 7.20.0 to 7.21.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.20.0 to 7.21.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.20.0...7.21.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5c7dc6497e..ee9919884b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.20.0 +ipython==7.21.0 # via -r test-requirements.in isort==5.7.0 # via pylint From 782ce3ba2de16b10345232897cf9505b43f08a21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Mar 2021 07:21:16 +0000 Subject: [PATCH 0613/1498] Bump pylint from 2.7.1 to 2.7.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.7.1 to 2.7.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.7.1...pylint-2.7.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index ee9919884b..c06e95d326 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5.0 +astroid==2.5.1 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -94,7 +94,7 @@ pyflakes==2.2.0 # via flake8 pygments==2.7.4 # via ipython -pylint==2.7.1 +pylint==2.7.2 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 34c36111e0d7905a9accddee429316be51a0fed4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 2 Mar 2021 05:20:52 +0000 Subject: [PATCH 0614/1498] Bump incremental from 17.5.0 to 21.3.0 Bumps [incremental](https://github.com/twisted/incremental) from 17.5.0 to 21.3.0. - [Release notes](https://github.com/twisted/incremental/releases) - [Changelog](https://github.com/twisted/incremental/blob/master/NEWS.rst) - [Commits](https://github.com/twisted/incremental/compare/incremental-17.5.0...incremental-21.3.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 7d6220f8c0..84b09b6941 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -30,7 +30,7 @@ imagesize==1.2.0 # via sphinx immutables==0.15 # via -r docs-requirements.in -incremental==17.5.0 +incremental==21.3.0 # via towncrier jinja2==2.11.3 # via From 4f8c7f97c89d85cf8cb46e4f130374b0215c1fdc Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 4 Mar 2021 16:48:16 +0000 Subject: [PATCH 0615/1498] 3.6 and 3.7 are no longer available on deadsnakes/nightly --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0959c1797..2e5290e623 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.6-dev', '3.7-dev', '3.8-dev', '3.9-dev'] + python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.8-dev', '3.9-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] From 8c62409483b6be9c585e12929525b95a32c1f708 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 6 Mar 2021 12:40:39 -0500 Subject: [PATCH 0616/1498] Test against 3.10 alpha --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e5290e623..a4483fa2ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9'] + python: ['3.6', '3.7', '3.8', '3.9', '3.10'] arch: ['x86', 'x64'] lsp: [''] lsp_extract_file: [''] @@ -41,7 +41,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: '${{ matrix.python }}' + python-version: '${{ matrix.python.action }}.0-alpha - ${{ matrix.python.action }}.X' architecture: '${{ matrix.arch }}' - name: Run tests run: ./ci.sh @@ -77,7 +77,7 @@ jobs: uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" with: - python-version: '${{ matrix.python }}' + python-version: '${{ matrix.python.action }}.0-alpha - ${{ matrix.python.action }}.X' - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -105,7 +105,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: '${{ matrix.python }}' + python-version: '${{ matrix.python.action }}.0-alpha - ${{ matrix.python.action }}.X' - name: Run tests run: ./ci.sh env: From 2a946a4c0152a01a9502bed248c8a09b4c997b5c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 6 Mar 2021 12:41:50 -0500 Subject: [PATCH 0617/1498] Correct matrix variable references --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4483fa2ff..4bae0fba2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: '${{ matrix.python.action }}.0-alpha - ${{ matrix.python.action }}.X' + python-version: '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X' architecture: '${{ matrix.arch }}' - name: Run tests run: ./ci.sh @@ -77,7 +77,7 @@ jobs: uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" with: - python-version: '${{ matrix.python.action }}.0-alpha - ${{ matrix.python.action }}.X' + python-version: '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X' - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -105,7 +105,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: '${{ matrix.python.action }}.0-alpha - ${{ matrix.python.action }}.X' + python-version: '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X' - name: Run tests run: ./ci.sh env: From 9e8692f1ddf38851f7ae8288fba6403ef5824df0 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 6 Mar 2021 13:42:26 -0500 Subject: [PATCH 0618/1498] Add 3.10 to Linux and macOS as well --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bae0fba2c..81a7c676f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.8-dev', '3.9-dev'] + python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.8-dev', '3.9-dev', '3.10'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] @@ -98,7 +98,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9'] + python: ['3.6', '3.7', '3.8', '3.9', '3.10'] steps: - name: Checkout uses: actions/checkout@v2 From 5f841df700a15256ab44933e6f2d2f8cca374426 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 6 Mar 2021 23:41:05 +0000 Subject: [PATCH 0619/1498] remove redundant -W error pytest flag It's now configured in pyproject.toml --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 499868ba45..b4f56f7114 100755 --- a/ci.sh +++ b/ci.sh @@ -149,7 +149,7 @@ else # 'coverage xml' to generate the report that it uses, and that will only # apply the ignore patterns in the current directory's .coveragerc. cp ../.coveragerc . - if pytest -W error -r a --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --cov="$INSTALLDIR" --verbose; then + if pytest -r a --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --cov="$INSTALLDIR" --verbose; then PASSED=true else PASSED=false From d0255b2c9812a7483d585b4d876326bd5116cd78 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 6 Mar 2021 23:48:40 +0000 Subject: [PATCH 0620/1498] Bump coverage from 5.4 to 5.5 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.4 to 5.5. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.4...coverage-5.5) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c06e95d326..abfb0be12a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ cffi==1.14.5 # via cryptography click==7.1.2 # via black -coverage==5.4 +coverage==5.5 # via pytest-cov cryptography==3.4.6 # via From d6c1f1d500ce4d81f15442acedd7eebbdbe75e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 7 Mar 2021 19:56:08 +0200 Subject: [PATCH 0621/1498] Made TrioToken weak referable --- newsfragments/1924.bugfix.rst | 1 + trio/_core/_entry_queue.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/1924.bugfix.rst diff --git a/newsfragments/1924.bugfix.rst b/newsfragments/1924.bugfix.rst new file mode 100644 index 0000000000..2e3ee6a418 --- /dev/null +++ b/newsfragments/1924.bugfix.rst @@ -0,0 +1 @@ +The ``TrioToken`` class can now be used as a target of a weak reference. \ No newline at end of file diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index a1587a18cd..cd675352eb 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -145,7 +145,7 @@ class TrioToken(metaclass=NoPublicConstructor): """ - __slots__ = ("_reentry_queue",) + __slots__ = ("_reentry_queue", "__weakref__") def __init__(self, reentry_queue): self._reentry_queue = reentry_queue From a871d844fdf0b135976b5a8691d4fdcbac1f57ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 7 Mar 2021 23:59:09 +0200 Subject: [PATCH 0622/1498] Added test --- trio/tests/test_threads.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 9da2838cbd..dc10ca4935 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -576,3 +576,10 @@ async def main(): _core.run(main) assert record == ["ok"] + + +async def test_trio_token_weak_referenceable(): + token = current_trio_token() + assert isinstance(token, TrioToken) + weak_reference = weakref.ref(token) + assert token is weak_reference() From 8f67d5de51aeaf6b8dd25b2b74f2df1985cb8ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 8 Mar 2021 00:07:46 +0200 Subject: [PATCH 0623/1498] Fixed imports --- trio/tests/test_threads.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index dc10ca4935..47727817e6 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -1,8 +1,10 @@ import threading import queue as stdlib_queue import time +import weakref import pytest +from trio._core import TrioToken, current_trio_token from .. import _core from .. import Event, CapacityLimiter, sleep From b17a11c2a2a32c524ad9a940ca68b59ee7164fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 8 Mar 2021 00:27:49 +0200 Subject: [PATCH 0624/1498] Update newsfragments/1924.bugfix.rst Co-authored-by: Kyle Altendorf --- newsfragments/1924.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/1924.bugfix.rst b/newsfragments/1924.bugfix.rst index 2e3ee6a418..58768be1f3 100644 --- a/newsfragments/1924.bugfix.rst +++ b/newsfragments/1924.bugfix.rst @@ -1 +1 @@ -The ``TrioToken`` class can now be used as a target of a weak reference. \ No newline at end of file +The :class:`TrioToken` class can now be used as a target of a weak reference. From 4fdb36c3950c38362194b2eab232b19ec073f671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 8 Mar 2021 00:31:18 +0200 Subject: [PATCH 0625/1498] Hopefully fixed the link this time --- newsfragments/1924.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/1924.bugfix.rst b/newsfragments/1924.bugfix.rst index 58768be1f3..351d424865 100644 --- a/newsfragments/1924.bugfix.rst +++ b/newsfragments/1924.bugfix.rst @@ -1 +1 @@ -The :class:`TrioToken` class can now be used as a target of a weak reference. +The :class:`~.lowlevel.TrioToken` class can now be used as a target of a weak reference. From e42c8875d9e5887b25a8b9cc572b1aa7d5630d8f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 18:32:48 -0500 Subject: [PATCH 0626/1498] reorder ubunty python versions --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81a7c676f5..09dd689da3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.8-dev', '3.9-dev', '3.10'] + python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] From 0af8e6c1cfbcb2d9837505057a974890c99f39d9 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 21:49:53 -0500 Subject: [PATCH 0627/1498] Pass _seen=None through to TracebackException.__init__() monkey patch In CPython 3.10.0a5 TracebackException.__init__() started to care about the difference between _seen=set() and _seen=None. As such we need to avoid turning None into set() prior to passing it to the real .__init__(). https://github.com/python/cpython/commit/6dfd1734f5b230bb8fbd2a9df806c1333b6652a8 --- trio/_core/_multierror.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 54295e71d5..9d0eb22f2d 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -384,9 +384,6 @@ def traceback_exception_init( compact=False, _seen=None, ): - if _seen is None: - _seen = set() - if sys.version_info >= (3, 10): kwargs = {"compact": compact} else: @@ -405,6 +402,9 @@ def traceback_exception_init( **kwargs, ) + if _seen is None: + _seen = set() + # Capture each of the exceptions in the MultiError along with each of their causes and contexts if isinstance(exc_value, MultiError): embedded = [] From 896395738249d34496b02b62546bafdedbed41e1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 22:14:07 -0500 Subject: [PATCH 0628/1498] Handle _seen=None properly for TracebackException.from_exception() --- trio/_core/_multierror.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 9d0eb22f2d..ed926c029f 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -402,6 +402,8 @@ def traceback_exception_init( **kwargs, ) + seen_was_none = _seen is None + if _seen is None: _seen = set() @@ -418,7 +420,7 @@ def traceback_exception_init( capture_locals=capture_locals, # copy the set of _seen exceptions so that duplicates # shared between sub-exceptions are not omitted - _seen=set(_seen), + _seen=None if seen_was_none else set(_seen), ) ) self.embedded = embedded From 702fbb98f4efb54b42392ec34a52f501c5dddeb4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 22:25:39 -0500 Subject: [PATCH 0629/1498] Try indexing to pick fancy alpha-latest for cpython and straight for pypy --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09dd689da3..b90d0d8b85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X' + python-version: ${{ ['${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', ${{ matrix.python }}][startsWith(${{ matrix.python }}, 'pypy')] }} architecture: '${{ matrix.arch }}' - name: Run tests run: ./ci.sh @@ -77,7 +77,7 @@ jobs: uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" with: - python-version: '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X' + python-version: ${{ ['${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', ${{ matrix.python }}][startsWith(${{ matrix.python }}, 'pypy')] }} - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -105,7 +105,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X' + python-version: ${{ ['${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', ${{ matrix.python }}][startsWith(${{ matrix.python }}, 'pypy')] }} - name: Run tests run: ./ci.sh env: From 5558db5f88db734a7b0192b5999a1fb3283ea015 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 22:30:30 -0500 Subject: [PATCH 0630/1498] Try fromJSON to make an array in GitHub Actions --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b90d0d8b85..8ed586a1c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ ['${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', ${{ matrix.python }}][startsWith(${{ matrix.python }}, 'pypy')] }} + python-version: ${{ fromJSON('["${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X", "${{ matrix.python }}"]')[startsWith(matrix.python, 'pypy')] }} architecture: '${{ matrix.arch }}' - name: Run tests run: ./ci.sh @@ -77,7 +77,7 @@ jobs: uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" with: - python-version: ${{ ['${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', ${{ matrix.python }}][startsWith(${{ matrix.python }}, 'pypy')] }} + python-version: ${{ fromJSON('["${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X", "${{ matrix.python }}"]')[startsWith(matrix.python, 'pypy')] }} - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -105,7 +105,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ ['${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', ${{ matrix.python }}][startsWith(${{ matrix.python }}, 'pypy')] }} + python-version: ${{ fromJSON('["${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X", "${{ matrix.python }}"]')[startsWith(matrix.python, 'pypy')] }} - name: Run tests run: ./ci.sh env: From 183b8c8e9c98276f2a401cdea051769cccf4431b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 22:35:16 -0500 Subject: [PATCH 0631/1498] Try fromJSON with format to make an array in GitHub Actions --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ed586a1c6..0f02585a12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ fromJSON('["${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X", "${{ matrix.python }}"]')[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', '${{ matrix.python }}'))[startsWith(matrix.python, 'pypy')] }} architecture: '${{ matrix.arch }}' - name: Run tests run: ./ci.sh @@ -77,7 +77,7 @@ jobs: uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" with: - python-version: ${{ fromJSON('["${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X", "${{ matrix.python }}"]')[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', '${{ matrix.python }}'))[startsWith(matrix.python, 'pypy')] }} - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -105,7 +105,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ fromJSON('["${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X", "${{ matrix.python }}"]')[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', '${{ matrix.python }}'))[startsWith(matrix.python, 'pypy')] }} - name: Run tests run: ./ci.sh env: From 423d4976555d8e43802d563d8cf130eb336dcf6a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 22:38:36 -0500 Subject: [PATCH 0632/1498] More formatting for GHA --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f02585a12..51a5e07704 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ fromJSON(format('["{0}", "{1}"]', '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', '${{ matrix.python }}'))[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), ${{ matrix.python }}))[startsWith(matrix.python, 'pypy')] }} architecture: '${{ matrix.arch }}' - name: Run tests run: ./ci.sh @@ -77,7 +77,7 @@ jobs: uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" with: - python-version: ${{ fromJSON(format('["{0}", "{1}"]', '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', '${{ matrix.python }}'))[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), ${{ matrix.python }}))[startsWith(matrix.python, 'pypy')] }} - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -105,7 +105,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ fromJSON(format('["{0}", "{1}"]', '${{ matrix.python }}.0-alpha - ${{ matrix.python }}.X', '${{ matrix.python }}'))[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), ${{ matrix.python }}))[startsWith(matrix.python, 'pypy')] }} - name: Run tests run: ./ci.sh env: From 78a497166b4521ab8eabe2338271a4655f6bf915 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 22:42:59 -0500 Subject: [PATCH 0633/1498] Less ${{ }} for GHA --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51a5e07704..503f33d7dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), ${{ matrix.python }}))[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} architecture: '${{ matrix.arch }}' - name: Run tests run: ./ci.sh @@ -77,7 +77,7 @@ jobs: uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" with: - python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), ${{ matrix.python }}))[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') @@ -105,7 +105,7 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: - python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), ${{ matrix.python }}))[startsWith(matrix.python, 'pypy')] }} + python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} - name: Run tests run: ./ci.sh env: From dfcdf5002cc9be39c4485c5673fcb3b761cee54f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 7 Mar 2021 22:48:07 -0500 Subject: [PATCH 0634/1498] Add 3.10-dev as well --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 503f33d7dc..4e71fabc6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev'] + python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] From 46fd668ddd4d330eb810a9c224fa24fd32bc94fe Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 8 Mar 2021 15:15:49 -0500 Subject: [PATCH 0635/1498] Use typed-ast from git for 3.10 for now https://github.com/python/typed_ast/issues/156 --- ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci.sh b/ci.sh index b4f56f7114..0d5e77e6b4 100755 --- a/ci.sh +++ b/ci.sh @@ -69,6 +69,8 @@ python -c "import sys, struct, ssl; print('#' * 70); print('python:', sys.versio python -m pip install -U pip setuptools wheel python -m pip --version +python -m pip install 'typed-ast @ git+https://github.com/python/typed_ast; python_version >= "3.10"' + python setup.py sdist --formats=zip python -m pip install dist/*.zip From 37daed79b5d347d5c6e176a326086b969398a0e4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 8 Mar 2021 15:27:03 -0500 Subject: [PATCH 0636/1498] Use threading.__excepthook__ in disable_threading_excepthook() when available --- trio/_core/tests/tutil.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 7f51750869..84a5a9a9fb 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -97,7 +97,11 @@ def restore_unraisablehook(): @contextmanager def disable_threading_excepthook(): - threading.excepthook, prev = _noop, threading.excepthook + if sys.version_info >= (3, 10): + threading.excepthook, prev = threading.__excepthook__, threading.excepthook + else: + threading.excepthook, prev = _noop, threading.excepthook + try: yield finally: From 3fb82d1d7c93ceda782f3ef778656d3ba9989c30 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 8 Mar 2021 15:42:43 -0500 Subject: [PATCH 0637/1498] Document python-version: craziness for handling alpha->release easily --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e71fabc6d..9726053192 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,14 @@ jobs: - name: Setup python uses: actions/setup-python@v2 with: + # This allows the matrix to specify just the major.minor version while still + # expanding it to get the latest patch version including alpha releases. + # This avoids the need to update for each new alpha, beta, release candidate, + # and then finally an actual release version. actions/setup-python doesn't + # support this for PyPy presently so we get no help there. + # + # CPython -> 3.9.0-alpha - 3.9.X + # PyPy -> pypy-3.7 python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} architecture: '${{ matrix.arch }}' - name: Run tests From 8beb8722528e854af94c71346079eda7324a0c67 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 9 Mar 2021 10:19:31 +0000 Subject: [PATCH 0638/1498] use attr.s for slotted classes to avoid accidentally forgetting the __weakref__ slot --- trio/_core/_entry_queue.py | 6 ++---- trio/_core/_local.py | 22 ++++++++++------------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index cd675352eb..8f6eb05e3b 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -126,6 +126,7 @@ def run_sync_soon(self, sync_fn, *args, idempotent=False): self.wakeup.wakeup_thread_and_signal_safe() +@attr.s(eq=False, hash=False, slots=True) class TrioToken(metaclass=NoPublicConstructor): """An opaque object representing a single call to :func:`trio.run`. @@ -145,10 +146,7 @@ class TrioToken(metaclass=NoPublicConstructor): """ - __slots__ = ("_reentry_queue", "__weakref__") - - def __init__(self, reentry_queue): - self._reentry_queue = reentry_queue + _reentry_queue = attr.ib() def run_sync_soon(self, sync_fn, *args, idempotent=False): """Schedule a call to ``sync_fn(*args)`` to occur in the context of a diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 1f64d4ce85..139fb7830a 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -1,24 +1,25 @@ # Runvar implementations +import attr + from . import _run from .._util import Final +@attr.s(eq=False, hash=False, slots=True) class _RunVarToken: _no_value = object() - __slots__ = ("_var", "previous_value", "redeemed") + _var = attr.ib() + previous_value = attr.ib(default=_no_value) + redeemed = attr.ib(default=False, init=False) @classmethod def empty(cls, var): - return cls(var, value=cls._no_value) - - def __init__(self, var, value): - self._var = var - self.previous_value = value - self.redeemed = False + return cls(var) +@attr.s(eq=False, hash=False, slots=True) class RunVar(metaclass=Final): """The run-local variant of a context variable. @@ -29,11 +30,8 @@ class RunVar(metaclass=Final): """ _NO_DEFAULT = object() - __slots__ = ("_name", "_default") - - def __init__(self, name, default=_NO_DEFAULT): - self._name = name - self._default = default + _name = attr.ib() + _default = attr.ib(default=_NO_DEFAULT) def get(self, default=_NO_DEFAULT): """Gets the value of this :class:`RunVar` for the current run call.""" From b68a3eb0efe43635df0c94c533fd1595b4b1b139 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 9 Mar 2021 10:21:44 +0000 Subject: [PATCH 0639/1498] make trio._core._run.Task slotted --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 807e330c1d..ae05d81a4e 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1060,7 +1060,7 @@ def __del__(self): ################################################################ -@attr.s(eq=False, hash=False, repr=False) +@attr.s(eq=False, hash=False, repr=False, slots=True) class Task(metaclass=NoPublicConstructor): _parent_nursery = attr.ib() coro = attr.ib() From f0fbd2e6e5c6107557e48ef50d20a2d89929c5ac Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 9 Mar 2021 12:00:47 +0000 Subject: [PATCH 0640/1498] add newsfragment --- newsfragments/1927.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1927.feature.rst diff --git a/newsfragments/1927.feature.rst b/newsfragments/1927.feature.rst new file mode 100644 index 0000000000..cd5fc4e9f9 --- /dev/null +++ b/newsfragments/1927.feature.rst @@ -0,0 +1 @@ +Use slots for :class:`~.lowlevel.Task` which should make them slightly smaller and faster. From 8034ff5a8875218f960fafa14a460ebbf02e265f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 12 Mar 2021 05:29:39 +0000 Subject: [PATCH 0641/1498] Bump prompt-toolkit from 3.0.16 to 3.0.17 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.16 to 3.0.17. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index abfb0be12a..f3f7bb0cee 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -80,7 +80,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.16 +prompt-toolkit==3.0.17 # via ipython ptyprocess==0.7.0 # via pexpect From 508cbdfff6b1f5c4fecc67bbb6fd7d5c647ddd90 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 15 Mar 2021 05:52:58 +0000 Subject: [PATCH 0642/1498] Bump flake8 from 3.8.4 to 3.9.0 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.8.4 to 3.9.0. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.8.4...3.9.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index f3f7bb0cee..ca183b2377 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -33,7 +33,7 @@ cryptography==3.4.6 # trustme decorator==4.4.2 # via ipython -flake8==3.8.4 +flake8==3.9.0 # via -r test-requirements.in idna==2.10 # via @@ -86,11 +86,11 @@ ptyprocess==0.7.0 # via pexpect py==1.10.0 # via pytest -pycodestyle==2.6.0 +pycodestyle==2.7.0 # via flake8 pycparser==2.20 # via cffi -pyflakes==2.2.0 +pyflakes==2.3.0 # via flake8 pygments==2.7.4 # via ipython From b8b22740af33ed79a889ff189a608570bc7cc5d8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 16 Mar 2021 05:26:45 +0000 Subject: [PATCH 0643/1498] Bump urllib3 from 1.26.3 to 1.26.4 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.3 to 1.26.4. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.4/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.3...1.26.4) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 84b09b6941..226c2c74b2 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -81,5 +81,5 @@ toml==0.10.2 # via towncrier towncrier==19.2.0 # via -r docs-requirements.in -urllib3==1.26.3 +urllib3==1.26.4 # via requests From 3b135fa0887af9e17db9521a81d2c8aec461d839 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 23 Mar 2021 06:34:46 +0000 Subject: [PATCH 0644/1498] Bump lazy-object-proxy from 1.5.2 to 1.6.0 Bumps [lazy-object-proxy](https://github.com/ionelmc/python-lazy-object-proxy) from 1.5.2 to 1.6.0. - [Release notes](https://github.com/ionelmc/python-lazy-object-proxy/releases) - [Changelog](https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-lazy-object-proxy/compare/v1.5.2...v1.6.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ca183b2377..6b3dee3a8a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -53,7 +53,7 @@ jedi==0.18.0 # via # -r test-requirements.in # ipython -lazy-object-proxy==1.5.2 +lazy-object-proxy==1.6.0 # via astroid mccabe==0.6.1 # via From 9a018b05d097c9eb2b79c6e4340886d5b3ff9858 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 23 Mar 2021 06:36:44 +0000 Subject: [PATCH 0645/1498] Bump prompt-toolkit from 3.0.17 to 3.0.18 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.17 to 3.0.18. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ca183b2377..49c90c883f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -80,7 +80,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.17 +prompt-toolkit==3.0.18 # via ipython ptyprocess==0.7.0 # via pexpect From 4d94ee5f43314813018f850eded11cda2658a905 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 5 Apr 2021 06:07:19 +0000 Subject: [PATCH 0646/1498] Bump regex from 2020.11.13 to 2021.4.4 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2020.11.13 to 2021.4.4. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ca183b2377..92bb470f11 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -106,7 +106,7 @@ pytest==6.2.2 # via # -r test-requirements.in # pytest-cov -regex==2020.11.13 +regex==2021.4.4 # via black six==1.15.0 # via pyopenssl From a2d60c803554f560a93df90f6c46acf6bb41e289 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Mon, 5 Apr 2021 22:28:28 +0900 Subject: [PATCH 0647/1498] Disable Windows "with IFS LSP" test based on Spyware Doctor AV, since we lost the installer binary. Closes #1938 --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e5290e623..969e460054 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,11 @@ jobs: lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe' lsp_extract_file: '' extra_name: ', with IFS LSP' - - python: '3.8' - arch: 'x64' - lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' - lsp_extract_file: '' - extra_name: ', with non-IFS LSP' + #- python: '3.8' + # arch: 'x64' + # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' + # lsp_extract_file: '' + # extra_name: ', with non-IFS LSP' steps: - name: Checkout uses: actions/checkout@v2 From b765726aae14bb042ef9a836ef020740f2e86f11 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 5 Apr 2021 17:52:39 -0700 Subject: [PATCH 0648/1498] DOC: Demphasize `asks` from awesome libraries (broken) The project seem to have no activity since October, and does not work out of the box when installing: >>> ImportError: cannot import name 'aopen' from 'anyio' It requires anyio ~2.x but is actually failing with it. As it is the first of the list; it is likely to be tried first and users wondering if they are doing something wrong (guilty of this myself), or if trio is broken. --- docs/source/awesome-trio-libraries.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index f09e8663e2..e1966d1c38 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -27,7 +27,6 @@ Getting Started Web and HTML ------------ -* `asks `__ - asks is an async requests-like http library. * `trio-websocket `__ - This library implements the WebSocket protocol, striving for safety, correctness, and ergonomics. * `quart-trio `__ - Like Flask, but for Trio. A simple and powerful framework for building async web applications and REST APIs. Tip: this is an ASGI-based framework, so you'll also need an HTTP server with ASGI support. * `hypercorn `__ - An HTTP server for hosting your ASGI apps. Supports HTTP/1.1, HTTP/2, HTTP/3, and Websockets. Can be run as a standalone server, or embedded in a larger Trio app. Use it with ``quart-trio``, or any other Trio-compatible ASGI framework. @@ -37,6 +36,13 @@ Web and HTML * `pyscalpel `__ - A fast and powerful webscraping library. +The following libraries may not be maintained, or do not work with the latests version of trio. + +* `asks `__ - asks is an async requests-like http library. As if April 2020, this + project seem dormant, and does not work with the latests version of `anyio `__ (2.2.0) + despite advertising compatibility with anyio 2.x + + Database -------- From be31faabc121a3487b174179f6d0b90126d790ee Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 5 Apr 2021 19:09:06 -0700 Subject: [PATCH 0649/1498] From review: - drop `asks`. - move `httpx` first. --- docs/source/awesome-trio-libraries.rst | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index e1966d1c38..19e1d24179 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -27,22 +27,15 @@ Getting Started Web and HTML ------------ +* `httpx `__ - HTTPX is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. * `trio-websocket `__ - This library implements the WebSocket protocol, striving for safety, correctness, and ergonomics. * `quart-trio `__ - Like Flask, but for Trio. A simple and powerful framework for building async web applications and REST APIs. Tip: this is an ASGI-based framework, so you'll also need an HTTP server with ASGI support. * `hypercorn `__ - An HTTP server for hosting your ASGI apps. Supports HTTP/1.1, HTTP/2, HTTP/3, and Websockets. Can be run as a standalone server, or embedded in a larger Trio app. Use it with ``quart-trio``, or any other Trio-compatible ASGI framework. -* `httpx `__ - HTTPX is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. * `DeFramed `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi `__ GUI library. * `pura `__ - A simple web framework for embedding realtime graphical visualization into Trio apps, enabling inspection and manipulation of program state during development. * `pyscalpel `__ - A fast and powerful webscraping library. -The following libraries may not be maintained, or do not work with the latests version of trio. - -* `asks `__ - asks is an async requests-like http library. As if April 2020, this - project seem dormant, and does not work with the latests version of `anyio `__ (2.2.0) - despite advertising compatibility with anyio 2.x - - Database -------- From 7a35ff7076eae5c7ebd344a4a9c05b3a4aa4edae Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Wed, 7 Apr 2021 06:56:06 +0900 Subject: [PATCH 0650/1498] awesome libs: clarify that trio-websocket has client and server (#1951) --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 19e1d24179..7317643bf9 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -28,7 +28,7 @@ Getting Started Web and HTML ------------ * `httpx `__ - HTTPX is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. -* `trio-websocket `__ - This library implements the WebSocket protocol, striving for safety, correctness, and ergonomics. +* `trio-websocket `__ - A WebSocket client and server implementation striving for safety, correctness, and ergonomics. * `quart-trio `__ - Like Flask, but for Trio. A simple and powerful framework for building async web applications and REST APIs. Tip: this is an ASGI-based framework, so you'll also need an HTTP server with ASGI support. * `hypercorn `__ - An HTTP server for hosting your ASGI apps. Supports HTTP/1.1, HTTP/2, HTTP/3, and Websockets. Can be run as a standalone server, or embedded in a larger Trio app. Use it with ``quart-trio``, or any other Trio-compatible ASGI framework. * `DeFramed `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi `__ GUI library. From 6b313de9edf3212c6e5ccfa69c68ef8a47bafc2f Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Wed, 7 Apr 2021 12:29:50 +0900 Subject: [PATCH 0651/1498] implement Event directly with wait_task_rescheduled rather than ParkingLot (#1948) * implement Event directly with wait_task_rescheduled rather than ParkingLot raised in #1944 * clear listening tasks set after rescheduling Co-authored-by: Nathaniel J. Smith * eliminate use of custom_sleep_data * newsfragment Co-authored-by: Nathaniel J. Smith --- newsfragments/1948.feature.rst | 5 +++++ trio/_sync.py | 30 ++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 newsfragments/1948.feature.rst diff --git a/newsfragments/1948.feature.rst b/newsfragments/1948.feature.rst new file mode 100644 index 0000000000..6e8b7616b9 --- /dev/null +++ b/newsfragments/1948.feature.rst @@ -0,0 +1,5 @@ +Make :class:`~.Event` more lightweight by using less objects (about 2 rather +than 5, including a nested ParkingLot and attribute dicts) and simpler +structures (set rather than OrderedDict). This may benefit applications that +create a large number of event instances, such as with the "replace event +object on every set()" idiom. diff --git a/trio/_sync.py b/trio/_sync.py index bed339ef6b..bf5f5d5e0b 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -1,16 +1,20 @@ import math import attr -import outcome import trio +from . import _core from ._core import enable_ki_protection, ParkingLot -from ._deprecate import deprecated from ._util import Final -@attr.s(repr=False, eq=False, hash=False) +@attr.s(frozen=True) +class _EventStatistics: + tasks_waiting = attr.ib() + + +@attr.s(repr=False, eq=False, hash=False, slots=True) class Event(metaclass=Final): """A waitable boolean value useful for inter-task synchronization, inspired by :class:`threading.Event`. @@ -37,7 +41,7 @@ class Event(metaclass=Final): """ - _lot = attr.ib(factory=ParkingLot, init=False) + _tasks = attr.ib(factory=set, init=False) _flag = attr.ib(default=False, init=False) def is_set(self): @@ -47,8 +51,11 @@ def is_set(self): @enable_ki_protection def set(self): """Set the internal flag value to True, and wake any waiting tasks.""" - self._flag = True - self._lot.unpark_all() + if not self._flag: + self._flag = True + for task in self._tasks: + _core.reschedule(task) + self._tasks.clear() async def wait(self): """Block until the internal flag value becomes True. @@ -59,7 +66,14 @@ async def wait(self): if self._flag: await trio.lowlevel.checkpoint() else: - await self._lot.park() + task = _core.current_task() + self._tasks.add(task) + + def abort_fn(_): + self._tasks.remove(task) + return _core.Abort.SUCCEEDED + + await _core.wait_task_rescheduled(abort_fn) def statistics(self): """Return an object containing debugging information. @@ -70,7 +84,7 @@ def statistics(self): :meth:`wait` method. """ - return self._lot.statistics() + return _EventStatistics(tasks_waiting=len(self._tasks)) def async_cm(cls): From e28c664fc88d674a22e834f3bf0257f51bd5d1b3 Mon Sep 17 00:00:00 2001 From: Kirill Klenov Date: Thu, 8 Apr 2021 22:40:20 +0300 Subject: [PATCH 0652/1498] Update awesome-trio-libraries.rst I've built some libraries with trio support. The PR contains the information. --- docs/source/awesome-trio-libraries.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 7317643bf9..fd203bf3f7 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -34,6 +34,8 @@ Web and HTML * `DeFramed `__ - DeFramed is a Web non-framework that supports a 99%-server-centric approach to Web coding, including support for the `Remi `__ GUI library. * `pura `__ - A simple web framework for embedding realtime graphical visualization into Trio apps, enabling inspection and manipulation of program state during development. * `pyscalpel `__ - A fast and powerful webscraping library. +* `muffin `_ - Muffin is a fast, simple ASGI web-framework +* `asgi-tools `_ - Tools to quickly build lightest ASGI apps (also contains a test client with lifespan, websocket support) Database @@ -84,6 +86,7 @@ Testing * `pytest-trio `__ - Pytest plugin for trio. * `hypothesis-trio `__ - Hypothesis plugin for trio. * `trustme `__ - #1 quality TLS certs while you wait, for the discerning tester. +* `pytest-aio `_ - Pytest plugin with support for trio, curio, asyncio Tools and Utilities From 9ce93b64b357a2d3c2454cd8bf8c7c46d3b5a337 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 12 Apr 2021 06:16:13 +0000 Subject: [PATCH 0653/1498] Bump astroid from 2.5.1 to 2.5.3 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.5.1 to 2.5.3. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/astroid-2.5.1...astroid-2.5.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ca183b2377..8ce6a753fd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5.1 +astroid==2.5.3 # via pylint async-generator==1.10 # via -r test-requirements.in From 9d323786853a8ef3864c0a27b689d5d61aa73132 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 12 Apr 2021 19:52:34 -0400 Subject: [PATCH 0654/1498] update typed-ast to 1.4.3 for testing --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8ce6a753fd..e52434c252 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -123,7 +123,7 @@ traitlets==5.0.5 # via ipython trustme==0.7.0 # via -r test-requirements.in -typed-ast==1.4.2 ; implementation_name == "cpython" +typed-ast==1.4.3 ; implementation_name == "cpython" # via # -r test-requirements.in # black From 199d1c3636063f32f06733f44d090d4d21591dbb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 13 Apr 2021 05:48:02 +0000 Subject: [PATCH 0655/1498] Bump typed-ast from 1.4.2 to 1.4.3 Bumps [typed-ast](https://github.com/python/typed_ast) from 1.4.2 to 1.4.3. - [Release notes](https://github.com/python/typed_ast/releases) - [Changelog](https://github.com/python/typed_ast/blob/master/release_process.md) - [Commits](https://github.com/python/typed_ast/compare/1.4.2...1.4.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8ce6a753fd..e52434c252 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -123,7 +123,7 @@ traitlets==5.0.5 # via ipython trustme==0.7.0 # via -r test-requirements.in -typed-ast==1.4.2 ; implementation_name == "cpython" +typed-ast==1.4.3 ; implementation_name == "cpython" # via # -r test-requirements.in # black From 1ff8c7923fbb046ea2aab3c626b71e7492259c24 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Apr 2021 05:52:24 +0000 Subject: [PATCH 0656/1498] Bump towncrier from 19.2.0 to 21.3.0 Bumps [towncrier](https://github.com/hawkowl/towncrier) from 19.2.0 to 21.3.0. - [Release notes](https://github.com/hawkowl/towncrier/releases) - [Changelog](https://github.com/twisted/towncrier/blob/master/NEWS.rst) - [Commits](https://github.com/hawkowl/towncrier/compare/19.2.0...21.3.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 226c2c74b2..ba9346ea6c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,8 +18,12 @@ certifi==2020.12.5 # via requests chardet==4.0.0 # via requests -click==7.1.2 +click-default-group==1.2.2 # via towncrier +click==7.1.2 + # via + # click-default-group + # towncrier docutils==0.16 # via sphinx idna==2.10 @@ -79,7 +83,7 @@ sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in toml==0.10.2 # via towncrier -towncrier==19.2.0 +towncrier==21.3.0 # via -r docs-requirements.in urllib3==1.26.4 # via requests From 241ff9fa5900a53af944431388f2c56f06882f19 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 15 Apr 2021 06:39:15 +0000 Subject: [PATCH 0657/1498] Bump docutils from 0.16 to 0.17 Bumps [docutils](http://docutils.sourceforge.net/) from 0.16 to 0.17. Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index ba9346ea6c..12c3c0d62b 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -24,7 +24,7 @@ click==7.1.2 # via # click-default-group # towncrier -docutils==0.16 +docutils==0.17 # via sphinx idna==2.10 # via From f3c806820600c38595eeb44494f0528a1bb423c7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 16 Apr 2021 05:35:42 +0000 Subject: [PATCH 0658/1498] Bump parso from 0.8.1 to 0.8.2 Bumps [parso](https://github.com/davidhalter/parso) from 0.8.1 to 0.8.2. - [Release notes](https://github.com/davidhalter/parso/releases) - [Changelog](https://github.com/davidhalter/parso/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/parso/compare/v0.8.1...v0.8.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e52434c252..350790f11f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -70,7 +70,7 @@ outcome==1.1.0 # via -r test-requirements.in packaging==20.9 # via pytest -parso==0.8.1 +parso==0.8.2 # via jedi pathspec==0.8.1 # via black From 7e81863d63f94cbba56003265472d7f445cd5bf8 Mon Sep 17 00:00:00 2001 From: Jason Heiss Date: Fri, 16 Apr 2021 09:17:25 -0700 Subject: [PATCH 0659/1498] Fix doc refs to move_on_after missing trio module qualifier --- docs/source/reference-core.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index ad2a571f73..d52247c4ea 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -667,7 +667,7 @@ In Trio, child tasks inherit the parent nursery's cancel scopes. So in this example, both the child tasks will be cancelled when the timeout expires:: - with move_on_after(TIMEOUT): + with trio.move_on_after(TIMEOUT): async with trio.open_nursery() as nursery: nursery.start_soon(child1) nursery.start_soon(child2) @@ -678,12 +678,12 @@ Note that what matters here is the scopes that were active when nothing at all:: async with trio.open_nursery() as nursery: - with move_on_after(TIMEOUT): # don't do this! + with trio.move_on_after(TIMEOUT): # don't do this! nursery.start_soon(child) -Why is this so? Well, ``start_soon()`` returns as soon as it has scheduled the new task to start running. The flow of execution in the parent then continues on to exit the ``with move_on_after(TIMEOUT):`` block, at which point Trio forgets about the timeout entirely. In order for the timeout to apply to the child task, Trio must be able to tell that its associated cancel scope will stay open for at least as long as the child task is executing. And Trio can only know that for sure if the cancel scope block is outside the nursery block. +Why is this so? Well, ``start_soon()`` returns as soon as it has scheduled the new task to start running. The flow of execution in the parent then continues on to exit the ``with trio.move_on_after(TIMEOUT):`` block, at which point Trio forgets about the timeout entirely. In order for the timeout to apply to the child task, Trio must be able to tell that its associated cancel scope will stay open for at least as long as the child task is executing. And Trio can only know that for sure if the cancel scope block is outside the nursery block. -You might wonder why Trio can't just remember "this task should be cancelled in ``TIMEOUT`` seconds", even after the ``with move_on_after(TIMEOUT):`` block is gone. The reason has to do with :ref:`how cancellation is implemented `. Recall that cancellation is represented by a `Cancelled` exception, which eventually needs to be caught by the cancel scope that caused it. (Otherwise, the exception would take down your whole program!) In order to be able to cancel the child tasks, the cancel scope has to be able to "see" the `Cancelled` exceptions that they raise -- and those exceptions come out of the ``async with open_nursery()`` block, not out of the call to ``start_soon()``. +You might wonder why Trio can't just remember "this task should be cancelled in ``TIMEOUT`` seconds", even after the ``with trio.move_on_after(TIMEOUT):`` block is gone. The reason has to do with :ref:`how cancellation is implemented `. Recall that cancellation is represented by a `Cancelled` exception, which eventually needs to be caught by the cancel scope that caused it. (Otherwise, the exception would take down your whole program!) In order to be able to cancel the child tasks, the cancel scope has to be able to "see" the `Cancelled` exceptions that they raise -- and those exceptions come out of the ``async with open_nursery()`` block, not out of the call to ``start_soon()``. If you want a timeout to apply to one task but not another, then you need to put the cancel scope in that individual task's function -- ``child()``, in this example. @@ -763,7 +763,7 @@ example, the timeout does *not* apply to ``child`` (or to anything else):: async def do_spawn(nursery): - with move_on_after(TIMEOUT): # don't do this, it has no effect + with trio.move_on_after(TIMEOUT): # don't do this, it has no effect nursery.start_soon(child) async with trio.open_nursery() as nursery: From 3d2a6049638aef9d3a066e18d73db7c2e56e34c0 Mon Sep 17 00:00:00 2001 From: Cameron LeCrone Date: Fri, 16 Apr 2021 17:03:13 -0400 Subject: [PATCH 0660/1498] Addresses #1958: Move code that handles job into its own method --- trio/_core/_thread_cache.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index ae5e8450b9..ffea5a0ec8 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -56,21 +56,22 @@ def __init__(self, thread_cache): thread.name = f"Trio worker thread {next(name_counter)}" thread.start() + def _handle_job(self): + fn, deliver = self._job + self._job = None + result = outcome.capture(fn) + # Tell the cache that we're available to be assigned a new + # job. We do this *before* calling 'deliver', so that if + # 'deliver' triggers a new job, it can be assigned to us + # instead of spawning a new thread. + self._thread_cache._idle_workers[self] = None + deliver(result) + def _work(self): while True: if self._worker_lock.acquire(timeout=IDLE_TIMEOUT): # We got a job - fn, deliver = self._job - self._job = None - result = outcome.capture(fn) - # Tell the cache that we're available to be assigned a new - # job. We do this *before* calling 'deliver', so that if - # 'deliver' triggers a new job, it can be assigned to us - # instead of spawning a new thread. - self._thread_cache._idle_workers[self] = None - deliver(result) - del fn - del deliver + self._handle_job() else: # Timeout acquiring lock, so we can probably exit. But, # there's a race condition: we might be assigned a job *just* From ebea9283ac57272be2d6f9f05ac9e72d7e0cdb39 Mon Sep 17 00:00:00 2001 From: Cameron LeCrone Date: Sat, 17 Apr 2021 12:19:52 -0400 Subject: [PATCH 0661/1498] Add a comment explaining why _handle_job() is a separate method --- trio/_core/_thread_cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index ffea5a0ec8..8ce37fff7e 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -57,6 +57,8 @@ def __init__(self, thread_cache): thread.start() def _handle_job(self): + # Handle job in a separate method to ensure user-created + # objects are cleaned up in a consistent manner. fn, deliver = self._job self._job = None result = outcome.capture(fn) From b1d558b805acbfbd9609eaf9b092e593a4e14eba Mon Sep 17 00:00:00 2001 From: Cameron LeCrone Date: Sat, 17 Apr 2021 12:33:54 -0400 Subject: [PATCH 0662/1498] Fix formatting via Black --- trio/_core/_thread_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index 8ce37fff7e..fc3b8a9508 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -58,7 +58,7 @@ def __init__(self, thread_cache): def _handle_job(self): # Handle job in a separate method to ensure user-created - # objects are cleaned up in a consistent manner. + # objects are cleaned up in a consistent manner. fn, deliver = self._job self._job = None result = outcome.capture(fn) From a79d37b35db4311523281f0ce050f7b3e8bb8901 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Apr 2021 05:56:21 +0000 Subject: [PATCH 0663/1498] Bump flake8 from 3.9.0 to 3.9.1 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.9.0 to 3.9.1. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.9.0...3.9.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 350790f11f..e177afc952 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -33,7 +33,7 @@ cryptography==3.4.6 # trustme decorator==4.4.2 # via ipython -flake8==3.9.0 +flake8==3.9.1 # via -r test-requirements.in idna==2.10 # via From f51814fb4bff0109de17eaa1fe49fffd1be3bce2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Apr 2021 06:05:17 +0000 Subject: [PATCH 0664/1498] Bump cryptography from 3.4.6 to 3.4.7 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.6 to 3.4.7. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.6...3.4.7) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e177afc952..a65a768e92 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -27,7 +27,7 @@ click==7.1.2 # via black coverage==5.5 # via pytest-cov -cryptography==3.4.6 +cryptography==3.4.7 # via # pyopenssl # trustme From 189e834e683038835ba8df70a94dc0d7073ada61 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Apr 2021 06:05:20 +0000 Subject: [PATCH 0665/1498] Bump ipython from 7.21.0 to 7.22.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.21.0 to 7.22.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.21.0...7.22.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e177afc952..2f987db0fd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.21.0 +ipython==7.22.0 # via -r test-requirements.in isort==5.7.0 # via pylint From 089545c6c14bfafda9bafc89ac75bd96c8e5447b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Apr 2021 06:20:12 +0000 Subject: [PATCH 0666/1498] Bump isort from 5.7.0 to 5.8.0 Bumps [isort](https://github.com/pycqa/isort) from 5.7.0 to 5.8.0. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.7.0...5.8.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2f83431807..5f995b33b5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ ipython-genutils==0.2.0 # via traitlets ipython==7.22.0 # via -r test-requirements.in -isort==5.7.0 +isort==5.8.0 # via pylint jedi==0.18.0 # via From bfddec2ca4b31503ee0cd7e7994f0c800ae102e9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Apr 2021 05:39:42 +0000 Subject: [PATCH 0667/1498] Bump sphinx-rtd-theme from 0.5.1 to 0.5.2 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 0.5.1 to 0.5.2. - [Release notes](https://github.com/readthedocs/sphinx_rtd_theme/releases) - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/0.5.1...0.5.2) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 12c3c0d62b..d3ff6d487a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -24,8 +24,10 @@ click==7.1.2 # via # click-default-group # towncrier -docutils==0.17 - # via sphinx +docutils==0.16 + # via + # sphinx + # sphinx-rtd-theme idna==2.10 # via # -r docs-requirements.in @@ -60,13 +62,13 @@ snowballstemmer==2.1.0 # via sphinx sortedcontainers==2.3.0 # via -r docs-requirements.in -sphinx-rtd-theme==0.5.1 - # via -r docs-requirements.in sphinx==3.3.1 # via # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio +sphinx_rtd_theme==0.5.2 + # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 From 04db98abb3b1440f37c51e7c937af4bd463160ca Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Apr 2021 05:40:26 +0000 Subject: [PATCH 0668/1498] Bump decorator from 4.4.2 to 5.0.7 Bumps [decorator](https://github.com/micheles/decorator) from 4.4.2 to 5.0.7. - [Release notes](https://github.com/micheles/decorator/releases) - [Changelog](https://github.com/micheles/decorator/blob/master/CHANGES.md) - [Commits](https://github.com/micheles/decorator/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5f995b33b5..61db46d642 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -31,7 +31,7 @@ cryptography==3.4.7 # via # pyopenssl # trustme -decorator==4.4.2 +decorator==5.0.7 # via ipython flake8==3.9.1 # via -r test-requirements.in From bb4a3d890d2435d5a30751c9a634e3d29bd96721 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Apr 2021 05:41:33 +0000 Subject: [PATCH 0669/1498] Bump pytest from 6.2.2 to 6.2.3 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.2 to 6.2.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.2...6.2.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5f995b33b5..3e58ef2731 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -102,7 +102,7 @@ pyparsing==2.4.7 # via packaging pytest-cov==2.11.1 # via -r test-requirements.in -pytest==6.2.2 +pytest==6.2.3 # via # -r test-requirements.in # pytest-cov From df9f635a7cc140b48166c158d8acec7400203f51 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Apr 2021 05:42:07 +0000 Subject: [PATCH 0670/1498] Bump pylint from 2.7.2 to 2.7.4 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.7.2 to 2.7.4. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.7.2...pylint-2.7.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5f995b33b5..b23c8678b0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -94,7 +94,7 @@ pyflakes==2.3.0 # via flake8 pygments==2.7.4 # via ipython -pylint==2.7.2 +pylint==2.7.4 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 0a7ae94faa97958c3cfbd19108ff5e024233bc2d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Apr 2021 05:58:23 +0000 Subject: [PATCH 0671/1498] Bump pyflakes from 2.3.0 to 2.3.1 Bumps [pyflakes](https://github.com/PyCQA/pyflakes) from 2.3.0 to 2.3.1. - [Release notes](https://github.com/PyCQA/pyflakes/releases) - [Changelog](https://github.com/PyCQA/pyflakes/blob/master/NEWS.rst) - [Commits](https://github.com/PyCQA/pyflakes/compare/2.3.0...2.3.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e273bdd8a5..a6a8e085eb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -90,7 +90,7 @@ pycodestyle==2.7.0 # via flake8 pycparser==2.20 # via cffi -pyflakes==2.3.0 +pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython From 66786bcc033c7fb7d7cdc3929542b407e8cdcc29 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 20 Apr 2021 21:35:24 +0100 Subject: [PATCH 0672/1498] make ParkingLot slotted --- trio/_core/_parking_lot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_parking_lot.py b/trio/_core/_parking_lot.py index 8b114b5230..e0f57ff8fa 100644 --- a/trio/_core/_parking_lot.py +++ b/trio/_core/_parking_lot.py @@ -80,12 +80,12 @@ _counter = count() -@attr.s(frozen=True) +@attr.s(frozen=True, slots=True) class _ParkingLotStatistics: tasks_waiting = attr.ib() -@attr.s(eq=False, hash=False) +@attr.s(eq=False, hash=False, slots=True) class ParkingLot(metaclass=Final): """A fair wait queue with cancellation and requeueing. From 06e812abe5b7c63032cd7595315cb0a7a68d4af3 Mon Sep 17 00:00:00 2001 From: sadahalu Date: Thu, 22 Apr 2021 20:27:32 -0400 Subject: [PATCH 0673/1498] Add .gitattributes and mark generated files --- .gitattributes | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..991065e069 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# For files generated by trio/_tools/gen_exports.py +trio/_core/_generated* linguist-generated=true +# Treat generated files as binary in git diff +trio/_core/_generated* -diff From 141bd6e8e53c51ce8fcd8f830b4ec2160ae2af1d Mon Sep 17 00:00:00 2001 From: Shawnee Date: Fri, 23 Apr 2021 16:34:55 -0400 Subject: [PATCH 0674/1498] fix some typos in the code comments --- docs/source/code-of-conduct.rst | 2 +- docs/source/contributing.rst | 2 +- docs/source/design.rst | 2 +- docs/source/history.rst | 2 +- notes-to-self/subprocess-notes.txt | 2 +- notes-to-self/win-waitable-timer.py | 2 +- trio/_abc.py | 2 +- trio/_core/_run.py | 4 ++-- trio/_core/tests/test_run.py | 2 +- trio/_socket.py | 2 +- trio/_ssl.py | 2 +- trio/_threads.py | 4 ++-- trio/tests/test_highlevel_open_tcp_stream.py | 2 +- trio/tests/test_socket.py | 2 +- trio/tests/test_ssl.py | 4 ++-- 15 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/source/code-of-conduct.rst b/docs/source/code-of-conduct.rst index 4b1de91b36..8bd6b24f42 100644 --- a/docs/source/code-of-conduct.rst +++ b/docs/source/code-of-conduct.rst @@ -81,7 +81,7 @@ Examples of unacceptable behavior by participants include: - Publishing private screenshots or quotes of interactions in the context of this project without all quoted users' *explicit* consent. - Publishing of private communication that doesn't have to do with - reporting harrassment. + reporting harassment. - Any of the above even when `presented as "ironic" or "joking" `__. - Any attempt to present "reverse-ism" versions of the above as diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index b0f9e3213b..1a97dc9f44 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -458,7 +458,7 @@ each PR leads to better quality. Beyond that, it all comes down to what you feel up to. If you don't feel like you know enough to review a complex code change, then you don't have to – you can just look it over and make some comments, even -if you don't feel up to making the final merge/no-merge decison. Or +if you don't feel up to making the final merge/no-merge decision. Or you can just stick to merging trivial doc fixes and adding tags to issues, that's helpful too. If after hanging around for a while you start to feel like you have better handle on how things work and want diff --git a/docs/source/design.rst b/docs/source/design.rst index 6251f22cdb..e0a8f939b7 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -505,7 +505,7 @@ several reasons: with whatever we get. We want more control over our users' experience than that. -* Impedence mismatch: the :mod:`selectors` API isn't particularly +* Impedance mismatch: the :mod:`selectors` API isn't particularly well-fitted to how we want to use it. For example, kqueue natively treats an interest in readability of some fd as a separate thing from an interest in that same fd's writability, which neatly matches diff --git a/docs/source/history.rst b/docs/source/history.rst index 7c63b54c88..48b821bd7c 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -774,7 +774,7 @@ Bugfixes - Make trio.socket._SocketType.connect *always* close the socket on cancellation (`#247 `__) -- Fix a memory leak in :class:`trio.CapacityLimiter`, that could occurr when +- Fix a memory leak in :class:`trio.CapacityLimiter`, that could occur when ``acquire`` or ``acquire_on_behalf_of`` was cancelled. (`#548 `__) - Some version of macOS have a buggy ``getaddrinfo`` that was causing spurious diff --git a/notes-to-self/subprocess-notes.txt b/notes-to-self/subprocess-notes.txt index e33c835640..d3a5c1096c 100644 --- a/notes-to-self/subprocess-notes.txt +++ b/notes-to-self/subprocess-notes.txt @@ -4,7 +4,7 @@ # tenable. We're better off trying os.waitpid(..., os.WNOHANG), and if that # says the process is still going then spawn a thread to sit in waitpid. # ......though that waitpid is non-cancellable so ugh. this is a problem, -# becaues it's also mutating -- you only get to waitpid() once, and you have +# because it's also mutating -- you only get to waitpid() once, and you have # to do it, because zombies. I guess we could make sure the waitpid thread is # daemonic and either it gets back to us eventually (even if our first call to # 'await wait()' is cancelled, maybe another one won't be), or else we go away diff --git a/notes-to-self/win-waitable-timer.py b/notes-to-self/win-waitable-timer.py index c831761358..92bfd7a39a 100644 --- a/notes-to-self/win-waitable-timer.py +++ b/notes-to-self/win-waitable-timer.py @@ -130,7 +130,7 @@ def now_as_filetime(): # # https://docs.microsoft.com/en-us/windows/win32/sysinfo/file-times # -# This page has FILETIME convertors and can be useful for debugging: +# This page has FILETIME converters and can be useful for debugging: # # https://www.epochconverter.com/ldap # diff --git a/trio/_abc.py b/trio/_abc.py index b8e341fdaa..4da9977393 100644 --- a/trio/_abc.py +++ b/trio/_abc.py @@ -333,7 +333,7 @@ async def wait_send_all_might_not_block(self): This method is intended to aid in implementing protocols that want to delay choosing which data to send until the last moment. E.g., - suppose you're working on an implemention of a remote display server + suppose you're working on an implementation of a remote display server like `VNC `__, and the network connection is currently backed up so that if you call diff --git a/trio/_core/_run.py b/trio/_core/_run.py index ae05d81a4e..3174261fa9 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -988,7 +988,7 @@ def start_soon(self, async_fn, *args, name=None): GLOBAL_RUN_CONTEXT.runner.spawn_impl(async_fn, args, self, name) async def start(self, async_fn, *args, name=None): - r"""Creates and initalizes a child task. + r"""Creates and initializes a child task. Like :meth:`start_soon`, but blocks until the new task has finished initializing itself, and optionally returns some @@ -2067,7 +2067,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): runner.instruments.call("before_io_wait", timeout) # Driver will call io_manager.get_events(timeout) and pass it back - # in throuh the yield + # in through the yield events = yield timeout runner.io_manager.process_events(events) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 8d92e6cec4..02f5954e60 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -446,7 +446,7 @@ async def crasher(): except AssertionError: # pragma: no cover raise except BaseException as exc: - # This is ouside the outer scope, so all the Cancelled + # This is outside the outer scope, so all the Cancelled # exceptions should have been absorbed, leaving just a regular # KeyError from crasher() assert type(exc) is KeyError diff --git a/trio/_socket.py b/trio/_socket.py index fcf26e072b..c805f829ff 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -325,7 +325,7 @@ def _sniff_sockopts_for_fileno(family, type, proto, fileno): # This function will modify the given socket to match the behavior in python -# 3.7. This will become unecessary and can be removed when support for versions +# 3.7. This will become unnecessary and can be removed when support for versions # older than 3.7 is dropped. def real_socket_type(type_num): return type_num & _SOCK_TYPE_MASK diff --git a/trio/_ssl.py b/trio/_ssl.py index c4ffa3ddbe..5f52b6ff3b 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -609,7 +609,7 @@ async def do_handshake(self): certificates, select cryptographic keys, and so forth, before any actual data can be sent or received. You don't have to call this method; if you don't, then :class:`SSLStream` will automatically - peform the handshake as needed, the first time you try to send or + perform the handshake as needed, the first time you try to send or receive data. But if you want to trigger it manually – for example, because you want to look at the peer's certificate before you start talking to them – then you can call this method. diff --git a/trio/_threads.py b/trio/_threads.py index 648b87d801..356eb0d2d1 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -267,7 +267,7 @@ def from_thread_run(afn, *args, trio_token=None): - Spawn this thread from `trio.to_thread.run_sync`. Trio will automatically capture the relevant Trio token and use it when you want to re-enter Trio. - - Pass a keyword argument, ``trio_token`` specifiying a specific + - Pass a keyword argument, ``trio_token`` specifying a specific `trio.run` loop to re-enter. This is useful in case you have a "foreign" thread, spawned using some other framework, and still want to enter Trio. @@ -317,7 +317,7 @@ def from_thread_run_sync(fn, *args, trio_token=None): - Spawn this thread from `trio.to_thread.run_sync`. Trio will automatically capture the relevant Trio token and use it when you want to re-enter Trio. - - Pass a keyword argument, ``trio_token`` specifiying a specific + - Pass a keyword argument, ``trio_token`` specifying a specific `trio.run` loop to re-enter. This is useful in case you have a "foreign" thread, spawned using some other framework, and still want to enter Trio. diff --git a/trio/tests/test_highlevel_open_tcp_stream.py b/trio/tests/test_highlevel_open_tcp_stream.py index bcd3ef7f5a..eaaff3e17f 100644 --- a/trio/tests/test_highlevel_open_tcp_stream.py +++ b/trio/tests/test_highlevel_open_tcp_stream.py @@ -59,7 +59,7 @@ def fake6(i): return (AF_INET6, SOCK_STREAM, IPPROTO_TCP, "", ("::{}".format(i), 80)) for fake in fake4, fake6: - # No effect on homogenous lists + # No effect on homogeneous lists targets = [fake(0), fake(1), fake(2)] reorder_for_rfc_6555_section_5_4(targets) assert targets == [fake(0), fake(1), fake(2)] diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index f8c061ffd3..60a4f0ba51 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -606,7 +606,7 @@ async def test_SocketType_non_blocking_paths(): with assert_checkpoints(): with pytest.raises(_core.Cancelled): await ta.recv(10) - # immedate success (also checks that the previous attempt didn't + # immediate success (also checks that the previous attempt didn't # actually read anything) with assert_checkpoints(): await ta.recv(10) == b"1" diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index f160af4999..cf59b8bd0a 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1160,7 +1160,7 @@ async def test_selected_alpn_protocol_before_handshake(client_ctx): async def test_selected_alpn_protocol_when_not_set(client_ctx): - # ALPN protocol still returns None when it's not ser, + # ALPN protocol still returns None when it's not set, # instead of raising an exception client, server = ssl_memory_stream_pair(client_ctx) @@ -1185,7 +1185,7 @@ async def test_selected_npn_protocol_before_handshake(client_ctx): async def test_selected_npn_protocol_when_not_set(client_ctx): - # NPN protocol still returns None when it's not ser, + # NPN protocol still returns None when it's not set, # instead of raising an exception client, server = ssl_memory_stream_pair(client_ctx) From 05e2c0aae257c0a075e1796082799ebc3621da38 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 23 Apr 2021 17:50:24 -0400 Subject: [PATCH 0675/1498] updated race example to use variable assignment to track winner instead of memory channel. --- docs/source/reference-core.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index d52247c4ea..5b644c58c6 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -794,23 +794,23 @@ finishes first:: if not async_fns: raise ValueError("must pass at least one argument") - send_channel, receive_channel = trio.open_memory_channel(0) + winner = None - async def jockey(async_fn): - await send_channel.send(await async_fn()) + async def jockey(async_fn, cancel_scope): + nonlocal winner + await winner = async_fn() + cancel_scope.cancel() async with trio.open_nursery() as nursery: for async_fn in async_fns: - nursery.start_soon(jockey, async_fn) - winner = await receive_channel.receive() - nursery.cancel_scope.cancel() - return winner + nursery.start_soon(jockey, async_fn, nursery.cancel_scope) + + return winner This works by starting a set of tasks which each try to run their -function, and then report back the value it returns. The main task -uses ``receive_channel.receive`` to wait for one to finish; as soon as -the first task crosses the finish line, it cancels the rest, and then -returns the winning value. +function. As soon as the first function completes its execution, the task will set the nonlocal variable ``winner`` +from the outer scope to the result of the function, and cancel the other tasks using the passed in cancel scope. Once all tasks +have been cancelled (which exits the nursery block), the variable ``winner`` will be returned. Here if one or more of the racing functions raises an unhandled exception then Trio's normal handling kicks in: it cancels the others From 8b01b438f8f919d54f82636ad8df0c280974c408 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Apr 2021 05:45:00 +0000 Subject: [PATCH 0676/1498] Bump pylint from 2.7.4 to 2.8.1 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.7.4 to 2.8.1. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.7.4...pylint-2.8.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index a6a8e085eb..0d14d5ced5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5.3 +astroid==2.5.6 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -94,7 +94,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.7.4 +pylint==2.8.1 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From cc7420eb9b7c0073bd7bbbb28889a342d1700183 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Apr 2021 05:46:21 +0000 Subject: [PATCH 0677/1498] Bump black from 20.8b1 to 21.4b0 Bumps [black](https://github.com/psf/black) from 20.8b1 to 21.4b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/master/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index a6a8e085eb..31fc0c0087 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==20.8b1 ; implementation_name == "cpython" +black==21.4b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography @@ -126,12 +126,10 @@ trustme==0.7.0 typed-ast==1.4.3 ; implementation_name == "cpython" # via # -r test-requirements.in - # black # mypy typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via # -r test-requirements.in - # black # mypy wcwidth==0.2.5 # via prompt-toolkit From 3ddf6c7f4c2da8c8be9eb981d6b8eb59a9d4342c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 26 Apr 2021 05:46:58 +0000 Subject: [PATCH 0678/1498] Bump astroid from 2.5.3 to 2.5.6 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.5.3 to 2.5.6. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/astroid-2.5.3...astroid-2.5.6) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a6a8e085eb..e2c6369542 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5.3 +astroid==2.5.6 # via pylint async-generator==1.10 # via -r test-requirements.in From bb04627c2311e6cddc156019123a30e75ec1a02e Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 26 Apr 2021 10:18:05 -0400 Subject: [PATCH 0679/1498] fixed await syntax for variable assignment --- docs/source/reference-core.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 5b644c58c6..514b5072c3 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -798,7 +798,7 @@ finishes first:: async def jockey(async_fn, cancel_scope): nonlocal winner - await winner = async_fn() + winner = await async_fn() cancel_scope.cancel() async with trio.open_nursery() as nursery: From c2c35846783d3a1e791acca7a3b06651c31e387b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 26 Apr 2021 13:48:56 -0700 Subject: [PATCH 0680/1498] Remove .travis.yml Closes #1785. We'll miss you. --- .travis.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1f397d2a40..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -os: linux -language: python -dist: focal - -jobs: - include: - - python: 3.6.1 # earliest 3.6 version available on Travis - # https://github.com/pypa/setuptools/issues/2350 - env: SETUPTOOLS_USE_DISTUTILS=stdlib - dist: bionic - -script: - - ./ci.sh - -branches: - except: - - /^dependabot/.*/ From 97c4f3255a5796f9ccdca526e0f8df7ba713262c Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 26 Apr 2021 14:05:12 -0700 Subject: [PATCH 0681/1498] Reduce coverage flapping Add `# pragma: no cover` to a few places where our tests have fallbacks for OS flakiness, and that are causing coverage to randomly go up and down depending on CI host load. --- trio/testing/_check_streams.py | 7 ++++--- trio/tests/test_socket.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/trio/testing/_check_streams.py b/trio/testing/_check_streams.py index 7a9006ff43..884e3f15cd 100644 --- a/trio/testing/_check_streams.py +++ b/trio/testing/_check_streams.py @@ -329,14 +329,15 @@ async def receiver(): nursery.start_soon(s.send_all, b"123") nursery.start_soon(s.send_all, b"123") - # closing the receiver causes wait_send_all_might_not_block to return + # closing the receiver causes wait_send_all_might_not_block to return, + # with or without an exception async with _ForceCloseBoth(await clogged_stream_maker()) as (s, r): async def sender(): try: with assert_checkpoints(): await s.wait_send_all_might_not_block() - except _core.BrokenResourceError: + except _core.BrokenResourceError: # pragma: no cover pass async def receiver(): @@ -353,7 +354,7 @@ async def receiver(): try: with assert_checkpoints(): await s.wait_send_all_might_not_block() - except _core.BrokenResourceError: + except _core.BrokenResourceError: # pragma: no cover pass # Check that if a task is blocked in a send-side method, then closing diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 60a4f0ba51..d891041ab2 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -1000,7 +1000,7 @@ async def test_many_sockets(): for x in range(total // 2): try: a, b = stdlib_socket.socketpair() - except OSError as e: + except OSError as e: # pragma: no cover assert e.errno in (errno.EMFILE, errno.ENFILE) break sockets += [a, b] @@ -1011,5 +1011,5 @@ async def test_many_sockets(): nursery.cancel_scope.cancel() for sock in sockets: sock.close() - if x != total // 2 - 1: + if x != total // 2 - 1: # pragma: no cover print(f"Unable to open more than {(x-1)*2} sockets.") From 12de1803066b9ee31fb573207965821c7624f6f4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 04:15:30 +0000 Subject: [PATCH 0682/1498] Bump pylint from 2.8.1 to 2.8.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.8.1 to 2.8.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.8.1...v2.8.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 96b247f196..c75f56b549 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -94,7 +94,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.8.1 +pylint==2.8.2 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 572d02ed01c8397e97aea8d577dbd38bb5d836d2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 14:14:49 +0000 Subject: [PATCH 0683/1498] Bump black from 21.4b0 to 21.4b1 Bumps [black](https://github.com/psf/black) from 21.4b0 to 21.4b1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/master/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c75f56b549..00e156101f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==21.4b0 ; implementation_name == "cpython" +black==21.4b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography From 29d74b7eadce16acc7c270eb2143dd4d2b4f54ad Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Apr 2021 05:32:04 +0000 Subject: [PATCH 0684/1498] Bump babel from 2.9.0 to 2.9.1 Bumps [babel](https://github.com/python-babel/babel) from 2.9.0 to 2.9.1. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES) - [Commits](https://github.com/python-babel/babel/compare/v2.9.0...v2.9.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index d3ff6d487a..a04a00ed30 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -12,7 +12,7 @@ attrs==20.3.0 # via # -r docs-requirements.in # outcome -babel==2.9.0 +babel==2.9.1 # via sphinx certifi==2020.12.5 # via requests @@ -62,13 +62,13 @@ snowballstemmer==2.1.0 # via sphinx sortedcontainers==2.3.0 # via -r docs-requirements.in +sphinx-rtd-theme==0.5.2 + # via -r docs-requirements.in sphinx==3.3.1 # via # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio -sphinx_rtd_theme==0.5.2 - # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 From 36cd27f17ced09b2b3a33f25d7c160ac9bb17fe7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Apr 2021 05:33:19 +0000 Subject: [PATCH 0685/1498] Bump black from 21.4b1 to 21.4b2 Bumps [black](https://github.com/psf/black) from 21.4b1 to 21.4b2. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/master/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 00e156101f..b358c92343 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==21.4b1 ; implementation_name == "cpython" +black==21.4b2 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography From 1583d2af4301314edebc43d76b366d1f8b1717fe Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Apr 2021 15:04:05 +0000 Subject: [PATCH 0686/1498] Upgrade to GitHub-native Dependabot --- .github/dependabot.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..0c2930b120 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,36 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + allow: + - dependency-type: direct + - dependency-type: indirect + ignore: + - dependency-name: pytest + versions: + - ">= 4.6.1.a, < 4.6.2" + - dependency-name: astroid + versions: + - 2.5.2 + - dependency-name: sphinx + versions: + - 3.4.3 + - 3.5.0 + - 3.5.1 + - 3.5.2 + - 3.5.3 + - dependency-name: regex + versions: + - 2021.3.17 + - dependency-name: pygments + versions: + - 2.8.0 + - dependency-name: cryptography + versions: + - 3.4.5 + - dependency-name: pytest + versions: + - 6.2.2 From 30ea8af36400317efb57101691db8b9715dc3c95 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 5 May 2021 05:37:37 +0000 Subject: [PATCH 0687/1498] Bump pytest from 6.2.3 to 6.2.4 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.3 to 6.2.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.3...6.2.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b358c92343..152d489432 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -102,7 +102,7 @@ pyparsing==2.4.7 # via packaging pytest-cov==2.11.1 # via -r test-requirements.in -pytest==6.2.3 +pytest==6.2.4 # via # -r test-requirements.in # pytest-cov From f215661444455d85218d15fc06f1e55b1295de20 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 5 May 2021 05:38:24 +0000 Subject: [PATCH 0688/1498] Bump black from 21.4b2 to 21.5b0 Bumps [black](https://github.com/psf/black) from 21.4b2 to 21.5b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/master/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b358c92343..a49fb7571e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==21.4b2 ; implementation_name == "cpython" +black==21.5b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography From a5342463141ee8ebbdc2f0ac74fd32c3b411099a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 5 May 2021 05:48:43 +0000 Subject: [PATCH 0689/1498] Bump ipython from 7.22.0 to 7.23.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.22.0 to 7.23.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.22.0...7.23.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index fb23042ebd..971b3dd7e8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.22.0 +ipython==7.23.0 # via -r test-requirements.in isort==5.8.0 # via pylint @@ -55,6 +55,8 @@ jedi==0.18.0 # ipython lazy-object-proxy==1.6.0 # via astroid +matplotlib-inline==0.1.2 + # via ipython mccabe==0.6.1 # via # flake8 @@ -120,7 +122,9 @@ toml==0.10.2 # pylint # pytest traitlets==5.0.5 - # via ipython + # via + # ipython + # matplotlib-inline trustme==0.7.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" From e5d6470cf641d10a86f1c446eb90518575093a84 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 6 May 2021 05:30:46 +0000 Subject: [PATCH 0690/1498] Bump six from 1.15.0 to 1.16.0 Bumps [six](https://github.com/benjaminp/six) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/benjaminp/six/releases) - [Changelog](https://github.com/benjaminp/six/blob/master/CHANGES) - [Commits](https://github.com/benjaminp/six/compare/1.15.0...1.16.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 971b3dd7e8..1beccc421b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -110,7 +110,7 @@ pytest==6.2.4 # pytest-cov regex==2021.4.4 # via black -six==1.15.0 +six==1.16.0 # via pyopenssl sniffio==1.2.0 # via -r test-requirements.in From 1a3934f5e6b89b436b3cd412cc9a39460440f10d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 6 May 2021 05:31:31 +0000 Subject: [PATCH 0691/1498] Bump ipython from 7.23.0 to 7.23.1 Bumps [ipython](https://github.com/ipython/ipython) from 7.23.0 to 7.23.1. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.23.0...7.23.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 971b3dd7e8..be20e43411 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.23.0 +ipython==7.23.1 # via -r test-requirements.in isort==5.8.0 # via pylint From 2c5a9e05870ea9949290c1fc5c274267f477e0b3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 06:08:20 +0000 Subject: [PATCH 0692/1498] Bump flake8 from 3.9.1 to 3.9.2 Bumps [flake8](https://gitlab.com/pycqa/flake8) from 3.9.1 to 3.9.2. - [Release notes](https://gitlab.com/pycqa/flake8/tags) - [Commits](https://gitlab.com/pycqa/flake8/compare/3.9.1...3.9.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4fe2d37629..f8f257fe44 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -33,7 +33,7 @@ cryptography==3.4.7 # trustme decorator==5.0.7 # via ipython -flake8==3.9.1 +flake8==3.9.2 # via -r test-requirements.in idna==2.10 # via From 9d3681f1c9452ec0d8b6495bd04e0134be5d8a55 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 11 May 2021 05:45:19 +0000 Subject: [PATCH 0693/1498] Bump black from 21.5b0 to 21.5b1 Bumps [black](https://github.com/psf/black) from 21.5b0 to 21.5b1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f8f257fe44..6db10e4100 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==21.5b0 ; implementation_name == "cpython" +black==21.5b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography From 14a784d92a5f167083e91b26977a4037b4c46923 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 05:35:07 +0000 Subject: [PATCH 0694/1498] Bump decorator from 5.0.7 to 5.0.9 Bumps [decorator](https://github.com/micheles/decorator) from 5.0.7 to 5.0.9. - [Release notes](https://github.com/micheles/decorator/releases) - [Changelog](https://github.com/micheles/decorator/blob/master/CHANGES.md) - [Commits](https://github.com/micheles/decorator/compare/5.0.7...5.0.9) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6db10e4100..55ddc4685a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -31,7 +31,7 @@ cryptography==3.4.7 # via # pyopenssl # trustme -decorator==5.0.7 +decorator==5.0.9 # via ipython flake8==3.9.2 # via -r test-requirements.in From 5ed4e0f6b19b652c91b2e936e5900a2bfb6e5618 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 06:06:36 +0000 Subject: [PATCH 0695/1498] Bump markupsafe from 1.1.1 to 2.0.0 Bumps [markupsafe](https://github.com/pallets/markupsafe) from 1.1.1 to 2.0.0. - [Release notes](https://github.com/pallets/markupsafe/releases) - [Changelog](https://github.com/pallets/markupsafe/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/markupsafe/compare/1.1.1...2.0.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index a04a00ed30..f628b08f55 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -42,7 +42,7 @@ jinja2==2.11.3 # via # sphinx # towncrier -markupsafe==1.1.1 +markupsafe==2.0.0 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in From 16decc5aa1ac03cbd517b5224cb08914ab29aeb0 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 17 May 2021 10:16:57 +0400 Subject: [PATCH 0696/1498] Trigger sr.ht build From fb70cccee976261c545d3d64a9f0dab66faf4f7f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 19 May 2021 05:32:17 +0000 Subject: [PATCH 0697/1498] Bump jinja2 from 2.11.3 to 3.0.1 Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.3 to 3.0.1. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.11.3...3.0.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index a04a00ed30..bbd3b06b30 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -38,11 +38,11 @@ immutables==0.15 # via -r docs-requirements.in incremental==21.3.0 # via towncrier -jinja2==2.11.3 +jinja2==3.0.1 # via # sphinx # towncrier -markupsafe==1.1.1 +markupsafe==2.0.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in From 60c63014f86dcaf1da2397ce5838dd0b37e23d45 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 19 May 2021 10:04:41 +0400 Subject: [PATCH 0698/1498] Bump markupsafe from 2.0.0 to 2.0.1 --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f628b08f55..cb90efa5bb 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -42,7 +42,7 @@ jinja2==2.11.3 # via # sphinx # towncrier -markupsafe==2.0.0 +markupsafe==2.0.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in From 2a4350781cb6bf3498f7fe305bda850e8b3107f0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 20 May 2021 05:28:05 +0000 Subject: [PATCH 0699/1498] Bump click from 7.1.2 to 8.0.1 Bumps [click](https://github.com/pallets/click) from 7.1.2 to 8.0.1. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/7.1.2...8.0.1) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index bbd3b06b30..3b21cbd928 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -20,7 +20,7 @@ chardet==4.0.0 # via requests click-default-group==1.2.2 # via towncrier -click==7.1.2 +click==8.0.1 # via # click-default-group # towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 55ddc4685a..fde04fa3ff 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,7 +23,7 @@ black==21.5b1 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography -click==7.1.2 +click==8.0.1 # via black coverage==5.5 # via pytest-cov From 4b076a344f24af44c4a029a0285a5349dcd44a56 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 20 May 2021 05:38:12 +0000 Subject: [PATCH 0700/1498] Bump pytest-cov from 2.11.1 to 2.12.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.11.1 to 2.12.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.11.1...v2.12.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index fde04fa3ff..b0f754cbd2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ cffi==1.14.5 # via cryptography click==8.0.1 # via black -coverage==5.5 +coverage[toml]==5.5 # via pytest-cov cryptography==3.4.7 # via @@ -102,7 +102,7 @@ pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.11.1 +pytest-cov==2.12.0 # via -r test-requirements.in pytest==6.2.4 # via @@ -119,6 +119,7 @@ sortedcontainers==2.3.0 toml==0.10.2 # via # black + # coverage # pylint # pytest traitlets==5.0.5 From d37a8dea9a870643cd29df8ac7d12e67e92d9ee6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 20 May 2021 05:49:26 +0000 Subject: [PATCH 0701/1498] Bump sortedcontainers from 2.3.0 to 2.4.0 Bumps [sortedcontainers](https://github.com/grantjenks/python-sortedcontainers) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/grantjenks/python-sortedcontainers/releases) - [Changelog](https://github.com/grantjenks/python-sortedcontainers/blob/master/HISTORY.rst) - [Commits](https://github.com/grantjenks/python-sortedcontainers/compare/v2.3.0...v2.4.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 3b21cbd928..c835313b95 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -60,7 +60,7 @@ sniffio==1.2.0 # via -r docs-requirements.in snowballstemmer==2.1.0 # via sphinx -sortedcontainers==2.3.0 +sortedcontainers==2.4.0 # via -r docs-requirements.in sphinx-rtd-theme==0.5.2 # via -r docs-requirements.in diff --git a/test-requirements.txt b/test-requirements.txt index b0f754cbd2..21e3b191f1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -114,7 +114,7 @@ six==1.16.0 # via pyopenssl sniffio==1.2.0 # via -r test-requirements.in -sortedcontainers==2.3.0 +sortedcontainers==2.4.0 # via -r test-requirements.in toml==0.10.2 # via From a8f091f1a55e48428ee35c01214c2a4dfb39b1bf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 05:34:07 +0000 Subject: [PATCH 0702/1498] Bump sphinxcontrib-serializinghtml from 1.1.4 to 1.1.5 Bumps [sphinxcontrib-serializinghtml](http://sphinx-doc.org/) from 1.1.4 to 1.1.5. Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c835313b95..2e2df8902c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -79,7 +79,7 @@ sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in From 3c3062fecea4be1d7182f90848c3515f34b18629 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 05:35:01 +0000 Subject: [PATCH 0703/1498] Bump sphinxcontrib-htmlhelp from 1.0.3 to 2.0.0 Bumps [sphinxcontrib-htmlhelp](http://sphinx-doc.org/) from 1.0.3 to 2.0.0. Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c835313b95..1ba5aa1e62 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -73,7 +73,7 @@ sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx From 3362898552a9bfa0b2d8b8230fba490d220e3906 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 27 May 2021 05:24:31 +0000 Subject: [PATCH 0704/1498] Bump urllib3 from 1.26.4 to 1.26.5 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.4 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.4...1.26.5) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 920ddcaefd..a07264b13e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -87,5 +87,5 @@ toml==0.10.2 # via towncrier towncrier==21.3.0 # via -r docs-requirements.in -urllib3==1.26.4 +urllib3==1.26.5 # via requests From 63d15683515b8acca833a44b59e652dc2d9986d0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 05:46:27 +0000 Subject: [PATCH 0705/1498] Bump ipython from 7.23.1 to 7.24.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.23.1 to 7.24.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.23.1...7.24.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 21e3b191f1..8a9ad5c974 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.23.1 +ipython==7.24.0 # via -r test-requirements.in isort==5.8.0 # via pylint From 53f6a1d6ffb5adba99ae166625754459e943f083 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 05:47:09 +0000 Subject: [PATCH 0706/1498] Bump certifi from 2020.12.5 to 2021.5.30 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.12.5 to 2021.5.30. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.12.05...2021.05.30) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index a07264b13e..d4988bfabc 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==20.3.0 # outcome babel==2.9.1 # via sphinx -certifi==2020.12.5 +certifi==2021.5.30 # via requests chardet==4.0.0 # via requests From 6fa7af4b8c14f4ea6e6cb02383696cb0df01ca06 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 05:47:59 +0000 Subject: [PATCH 0707/1498] Bump astroid from 2.5.6 to 2.5.7 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.5.6 to 2.5.7. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/astroid-2.5.6...v2.5.7) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 21e3b191f1..c83293e23d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5.6 +astroid==2.5.7 # via pylint async-generator==1.10 # via -r test-requirements.in From 99002069126b634a793950267dd3340db715c4a5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 1 Jun 2021 05:32:18 +0000 Subject: [PATCH 0708/1498] Bump black from 21.5b1 to 21.5b2 Bumps [black](https://github.com/psf/black) from 21.5b1 to 21.5b2. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 350befaf88..3691bdcd44 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==21.5b1 ; implementation_name == "cpython" +black==21.5b2 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography From 52f595a14bb96e3599d4dfcf0284f62d3355bdbf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 1 Jun 2021 05:34:09 +0000 Subject: [PATCH 0709/1498] Bump pylint from 2.8.2 to 2.8.3 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.8.2 to 2.8.3. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.8.2...v2.8.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 350befaf88..cd16670f2a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5.7 +astroid==2.5.6 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -96,7 +96,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.8.2 +pylint==2.8.3 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From ca0a824df481824c590a382015c88fa673ef92ed Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 2 Jun 2021 05:18:19 +0000 Subject: [PATCH 0710/1498] Bump pytest-cov from 2.12.0 to 2.12.1 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.12.0 to 2.12.1. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.12.0...v2.12.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 98206bc25a..2ddd80163d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ cffi==1.14.5 # via cryptography click==8.0.1 # via black -coverage[toml]==5.5 +coverage==5.5 # via pytest-cov cryptography==3.4.7 # via @@ -102,7 +102,7 @@ pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.12.0 +pytest-cov==2.12.1 # via -r test-requirements.in pytest==6.2.4 # via @@ -119,9 +119,9 @@ sortedcontainers==2.4.0 toml==0.10.2 # via # black - # coverage # pylint # pytest + # pytest-cov traitlets==5.0.5 # via # ipython From 2d7eb41ba028067006cca3529407fb231a405673 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 3 Jun 2021 05:29:38 +0000 Subject: [PATCH 0711/1498] Bump ipython from 7.24.0 to 7.24.1 Bumps [ipython](https://github.com/ipython/ipython) from 7.24.0 to 7.24.1. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.24.0...7.24.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2ddd80163d..f137c41785 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.24.0 +ipython==7.24.1 # via -r test-requirements.in isort==5.8.0 # via pylint From c069b12d32444eded906d073b6f53ecf24091771 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 8 Jun 2021 21:24:37 +0100 Subject: [PATCH 0712/1498] avoid deprecated asyncio.Future() --- docs/source/reference-lowlevel.rst | 2 +- notes-to-self/aio-guest-test.py | 2 +- trio/_core/tests/test_guest_mode.py | 2 +- trio/_core/tests/test_run.py | 7 +++---- trio/_core/tests/tutil.py | 8 +++++++- trio/tests/test_util.py | 9 ++++++--- 6 files changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index ebd0f57c03..2fe773cbb7 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -785,7 +785,7 @@ Here's how we'd extend our asyncio example to implement this pattern: asyncio_loop.call_soon_threadsafe(fn) # Revised 'done' callback: set a Future - done_fut = asyncio.Future() + done_fut = asyncio_loop.create_future() def done_callback(trio_main_outcome): done_fut.set_result(trio_main_outcome) diff --git a/notes-to-self/aio-guest-test.py b/notes-to-self/aio-guest-test.py index 5e9b398132..b64a11bd04 100644 --- a/notes-to-self/aio-guest-test.py +++ b/notes-to-self/aio-guest-test.py @@ -4,7 +4,7 @@ async def aio_main(): loop = asyncio.get_running_loop() - trio_done_fut = asyncio.Future() + trio_done_fut = loop.create_future() def trio_done_callback(main_outcome): print(f"trio_main finished: {main_outcome!r}") trio_done_fut.set_result(main_outcome) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 0184ff3103..b06849a836 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -323,7 +323,7 @@ def aiotrio_run(trio_fn, *, pass_not_threadsafe=True, **start_guest_run_kwargs): loop = asyncio.new_event_loop() async def aio_main(): - trio_done_fut = asyncio.Future() + trio_done_fut = loop.create_future() def trio_done_callback(main_outcome): print(f"trio_fn finished: {main_outcome!r}") diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 02f5954e60..c56aaaa092 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -24,6 +24,7 @@ ignore_coroutine_never_awaited_warnings, buggy_pypy_asyncgens, restore_unraisablehook, + create_asyncio_future_in_new_loop, ) from ... import _core @@ -1574,9 +1575,7 @@ async def async_gen(arg): # pragma: no cover def test_calling_asyncio_function_gives_nice_error(): async def child_xyzzy(): - import asyncio - - await asyncio.Future() + await create_asyncio_future_in_new_loop() async def misguided(): await child_xyzzy() @@ -1598,7 +1597,7 @@ async def test_asyncio_function_inside_nursery_does_not_explode(): import asyncio nursery.start_soon(sleep_forever) - await asyncio.Future() + await create_asyncio_future_in_new_loop() assert "asyncio" in str(excinfo.value) diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 84a5a9a9fb..3e8053f71f 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -1,4 +1,5 @@ # Utilities for testing +import asyncio import socket as stdlib_socket import threading import os @@ -7,7 +8,7 @@ import pytest import warnings -from contextlib import contextmanager +from contextlib import contextmanager, closing import gc @@ -139,3 +140,8 @@ def check_sequence_matches(seq, template): and os.uname().release[:4] < "12.2", reason="hangs on FreeBSD 12.1 and earlier, due to FreeBSD bug #246350", ) + + +def create_asyncio_future_in_new_loop(): + with closing(asyncio.new_event_loop()) as loop: + return loop.create_future() diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 2ea0a1e287..2d57e0ebfc 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -3,7 +3,10 @@ import trio from .. import _core -from .._core.tests.tutil import ignore_coroutine_never_awaited_warnings +from .._core.tests.tutil import ( + ignore_coroutine_never_awaited_warnings, + create_asyncio_future_in_new_loop, +) from .._util import ( signal_raise, ConflictDetector, @@ -114,11 +117,11 @@ def generator_based_coro(): # pragma: no cover assert "asyncio" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: - coroutine_or_error(asyncio.Future()) + coroutine_or_error(create_asyncio_future_in_new_loop()) assert "asyncio" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: - coroutine_or_error(lambda: asyncio.Future()) + coroutine_or_error(create_asyncio_future_in_new_loop) assert "asyncio" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: From 970034e32a4c68566135def068236a636ffa0aae Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 8 Jun 2021 21:26:48 +0100 Subject: [PATCH 0713/1498] filterwarnings distutils deprecation warnings --- trio/tests/test_exports.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 374ce8c044..ef7edb4ccb 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -1,3 +1,4 @@ +import re import sys import importlib import types @@ -69,6 +70,17 @@ def public_modules(module): ) @pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) @pytest.mark.parametrize("tool", ["pylint", "jedi"]) +@pytest.mark.filterwarnings( + "ignore:" + + re.escape( + "The distutils package is deprecated and slated for removal in Python 3.12. " + "Use setuptools or check PEP 632 for potential alternatives" + ) + + ":DeprecationWarning", + "ignore:" + + re.escape("The distutils.sysconfig module is deprecated, use sysconfig instead") + + ":DeprecationWarning", +) def test_static_tool_sees_all_symbols(tool, modname): module = importlib.import_module(modname) From d37fe5121cbdfb533747b8b1c9020bd381cb1445 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 8 Jun 2021 21:27:13 +0100 Subject: [PATCH 0714/1498] filterwarnings NPN deprecation warnings --- trio/tests/test_ssl.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index cf59b8bd0a..665024bed2 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1,3 +1,5 @@ +import re + import pytest import threading @@ -1184,6 +1186,9 @@ async def test_selected_npn_protocol_before_handshake(client_ctx): server.selected_npn_protocol() +@pytest.mark.filterwarnings( + r"ignore: ssl module. NPN is deprecated, use ALPN instead:UserWarning" +) async def test_selected_npn_protocol_when_not_set(client_ctx): # NPN protocol still returns None when it's not set, # instead of raising an exception From 3794f88328f3baf3f0fb4f4d716de0a7089fcdb5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 9 Jun 2021 05:39:19 +0000 Subject: [PATCH 0715/1498] Bump mypy from 0.812 to 0.901 Bumps [mypy](https://github.com/python/mypy) from 0.812 to 0.901. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.812...v0.901) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index f137c41785..3a99ea6d72 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -66,7 +66,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # -r test-requirements.in # black # mypy -mypy==0.812 ; implementation_name == "cpython" +mypy==0.901 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in @@ -119,6 +119,7 @@ sortedcontainers==2.4.0 toml==0.10.2 # via # black + # mypy # pylint # pytest # pytest-cov @@ -129,9 +130,7 @@ traitlets==5.0.5 trustme==0.7.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" - # via - # -r test-requirements.in - # mypy + # via -r test-requirements.in typing-extensions==3.7.4.3 ; implementation_name == "cpython" # via # -r test-requirements.in From c64f4134d7ef9f61e39e81861a0de31274c5137b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 9 Jun 2021 05:47:10 +0000 Subject: [PATCH 0716/1498] Bump typing-extensions from 3.7.4.3 to 3.10.0.0 Bumps [typing-extensions](https://github.com/python/typing) from 3.7.4.3 to 3.10.0.0. - [Release notes](https://github.com/python/typing/releases) - [Commits](https://github.com/python/typing/compare/3.7.4.3...3.10.0.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3a99ea6d72..616d101b44 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -131,7 +131,7 @@ trustme==0.7.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" # via -r test-requirements.in -typing-extensions==3.7.4.3 ; implementation_name == "cpython" +typing-extensions==3.10.0.0 ; implementation_name == "cpython" # via # -r test-requirements.in # mypy From 8286fc96e3dfb2cbd94792f1ab6579c2e31c6b38 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 9 Jun 2021 05:58:30 +0000 Subject: [PATCH 0717/1498] Bump trustme from 0.7.0 to 0.8.0 Bumps [trustme](https://github.com/python-trio/trustme) from 0.7.0 to 0.8.0. - [Release notes](https://github.com/python-trio/trustme/releases) - [Commits](https://github.com/python-trio/trustme/compare/v0.7.0...v0.8.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 616d101b44..6629362844 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -127,7 +127,7 @@ traitlets==5.0.5 # via # ipython # matplotlib-inline -trustme==0.7.0 +trustme==0.8.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" # via -r test-requirements.in From 429f3fa5f87c6c519c74e3b92c65bbdaca12e2e2 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 10 Jun 2021 20:20:41 +0100 Subject: [PATCH 0718/1498] remove typed_ast git dep --- ci.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci.sh b/ci.sh index 0d5e77e6b4..b4f56f7114 100755 --- a/ci.sh +++ b/ci.sh @@ -69,8 +69,6 @@ python -c "import sys, struct, ssl; print('#' * 70); print('python:', sys.versio python -m pip install -U pip setuptools wheel python -m pip --version -python -m pip install 'typed-ast @ git+https://github.com/python/typed_ast; python_version >= "3.10"' - python setup.py sdist --formats=zip python -m pip install dist/*.zip From 005826a2ed659c15ea8635694ce6e49fafd005c2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 11 Jun 2021 05:23:44 +0000 Subject: [PATCH 0719/1498] Bump black from 21.5b2 to 21.6b0 Bumps [black](https://github.com/psf/black) from 21.5b2 to 21.6b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6629362844..1f4635d492 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==21.5b2 ; implementation_name == "cpython" +black==21.6b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography From ba4ae69772c9f30da9ec0cd939333c515e785257 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 11 Jun 2021 05:24:58 +0000 Subject: [PATCH 0720/1498] Bump mypy from 0.901 to 0.902 Bumps [mypy](https://github.com/python/mypy) from 0.901 to 0.902. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.901...v0.902) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6629362844..285826ccfc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -66,7 +66,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # -r test-requirements.in # black # mypy -mypy==0.901 ; implementation_name == "cpython" +mypy==0.902 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in From 399c850ab9fe841b888208d37b35b796c5f576ea Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 11 Jun 2021 09:33:48 +0100 Subject: [PATCH 0721/1498] collect telemetry on bpo-32219 --- trio/tests/test_ssl.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index cf59b8bd0a..1a2a1dcc83 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1,3 +1,5 @@ +import sys + import pytest import threading @@ -98,14 +100,20 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): # respond in kind but it's legal for them to have already # gone away. exceptions = (BrokenPipeError, ssl.SSLZeroReturnError) - # Under unclear conditions, CPython sometimes raises - # SSLWantWriteError here. This is a bug (bpo-32219), but - # it's not our bug, so ignore it. - exceptions += (ssl.SSLWantWriteError,) try: wrapped.unwrap() except exceptions: pass + except ssl.SSLWantWriteError: # pragma: no cover + # Under unclear conditions, CPython sometimes raises + # SSLWantWriteError here. This is a bug (bpo-32219), + # but it's not our bug, so ignore it. + if sys.version_info >= (3, 8): + pytest.fail( + "still an issue on recent python versions " + "add a comment to " + "https://bugs.python.org/issue32219" + ) return wrapped.sendall(data) # This is an obscure workaround for an openssl bug. In server mode, in From 8bc7d0620a86791f3cd0931652c7b15432593d4c Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 11 Jun 2021 17:42:03 +0100 Subject: [PATCH 0722/1498] Update trio/tests/test_ssl.py Co-authored-by: Kyle Altendorf --- trio/tests/test_ssl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 1a2a1dcc83..ffe5388a79 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -107,7 +107,10 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): except ssl.SSLWantWriteError: # pragma: no cover # Under unclear conditions, CPython sometimes raises # SSLWantWriteError here. This is a bug (bpo-32219), - # but it's not our bug, so ignore it. + # but it's not our bug. Christian Heimes thinks + # it's fixed in 'recent' CPython versions so we fail + # the test for those and ignore it for earlier + # versions. if sys.version_info >= (3, 8): pytest.fail( "still an issue on recent python versions " From 66fc4190f3ecb290c64d5f9ec15fb676e3e701ae Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 11 Jun 2021 17:44:11 +0100 Subject: [PATCH 0723/1498] only ignore buggy exception on old versions of cpython --- trio/tests/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index ffe5388a79..573b6bd690 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -111,7 +111,7 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): # it's fixed in 'recent' CPython versions so we fail # the test for those and ignore it for earlier # versions. - if sys.version_info >= (3, 8): + if sys.platform != "cpython" or sys.version_info >= (3, 8): pytest.fail( "still an issue on recent python versions " "add a comment to " From add57b9ec5a70217373ae1624ddc6e30f30b02ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 14 Jun 2021 10:24:40 +0200 Subject: [PATCH 0724/1498] :construction_worker: Trigger CI to confirm that master has a CI issue with the Fedora job From 27612eaf1517282909701cea1847d021ae76ccc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 14 Jun 2021 10:51:01 +0200 Subject: [PATCH 0725/1498] :green_heart: Make Fedora build use Python 3.9 to avoid cffi build from source --- .builds/fedora.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.builds/fedora.yml b/.builds/fedora.yml index eddb2368a0..e30f7706ce 100644 --- a/.builds/fedora.yml +++ b/.builds/fedora.yml @@ -1,12 +1,14 @@ image: fedora/rawhide packages: - - python3-devel + # Comment out to avoid Python 3.10 until cffi has built wheels for Python 3.10 + # - python3-devel + - python3.9 - python3-pip sources: - https://github.com/python-trio/trio tasks: - test: | - python3 -m venv venv + python3.9 -m venv venv source venv/bin/activate cd trio CI_BUILD_ID=$JOB_ID CI_BUILD_URL=$JOB_URL ./ci.sh From e04ace7c3685e3ac14c3a93226d413c001cecaba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 14 Jun 2021 10:12:17 +0200 Subject: [PATCH 0726/1498] :memo: Add small description of call stack in tutorial --- docs/source/tutorial.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 82d4663a2d..279b33d988 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -600,7 +600,10 @@ suddenly we're back in :func:`trio.run` deciding what to run next. How does this happen? The secret is that :func:`trio.run` and :func:`trio.sleep` work together to make it happen: :func:`trio.sleep` has access to some special magic that lets it pause its entire -call stack, so it sends a note to :func:`trio.run` requesting to be +call stack, which is all the things in memory up to that specific point +in the code, including the variables created in the parent function that +called :func:`trio.sleep`, the exact point in that function where it was +called, etc. so it sends a note to :func:`trio.run` requesting to be woken again after 1 second, and then suspends the task. And once the task is suspended, Python gives control back to :func:`trio.run`, which decides what to do next. (If this sounds similar to the way that From a448ea1d1df37564b292e8479e419f8e3d2b35ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 14 Jun 2021 12:25:16 +0200 Subject: [PATCH 0727/1498] :memo: Reword tutorial, remove reference to call stack, from code review --- docs/source/tutorial.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 279b33d988..4622e257e5 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -599,11 +599,8 @@ Each task runs until it hits the call to :func:`trio.sleep`, and then suddenly we're back in :func:`trio.run` deciding what to run next. How does this happen? The secret is that :func:`trio.run` and :func:`trio.sleep` work together to make it happen: :func:`trio.sleep` -has access to some special magic that lets it pause its entire -call stack, which is all the things in memory up to that specific point -in the code, including the variables created in the parent function that -called :func:`trio.sleep`, the exact point in that function where it was -called, etc. so it sends a note to :func:`trio.run` requesting to be +has access to some special magic that lets it pause itself, +so it sends a note to :func:`trio.run` requesting to be woken again after 1 second, and then suspends the task. And once the task is suspended, Python gives control back to :func:`trio.run`, which decides what to do next. (If this sounds similar to the way that From 71badc7f9586d763970b0d8557d5504e90a4d3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 14 Jun 2021 16:27:49 +0200 Subject: [PATCH 0728/1498] :pencil2: Fix a couple of typos in tutorial --- docs/source/tutorial.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 4622e257e5..aedba8244a 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -622,7 +622,7 @@ between the implementation of generators and async functions.) Only async functions have access to the special magic for suspending a task, so only async functions can cause the program to switch to a -different task. What this means if a call *doesn't* have an ``await`` +different task. What this means is that if a call *doesn't* have an ``await`` on it, then you know that it *can't* be a place where your task will be suspended. This makes tasks much `easier to reason about `__ than @@ -639,7 +639,7 @@ wouldn't have been able to pause at the end and wait for the children to finish; we need our cleanup function to be async, which is exactly what ``async with`` gives us. -Now, back to our execution trace. To recap: at this point ``parent`` +Now, back to our execution point. To recap: at this point ``parent`` is waiting on ``child1`` and ``child2``, and both children are sleeping. So :func:`trio.run` checks its notes, and sees that there's nothing to be done until those sleeps finish – unless possibly some @@ -1092,7 +1092,7 @@ up, and ``send_all`` will block until the remote side calls Now let's think about this from the server's point of view. Each time it calls ``receive_some``, it gets some data that it needs to send -back. And until it sends it back, the data is sitting around takes up +back. And until it sends it back, the data that is sitting around takes up memory. Computers have finite amounts of RAM, so if our server is well behaved then at some point it needs to stop calling ``receive_some`` until it gets rid of some of the old data by doing its own call to From c575703c1bf13e9a986503b96672da8d567fc520 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 14 Jun 2021 19:34:33 +0100 Subject: [PATCH 0729/1498] fix sys.platform/sys.implementation.name confusion --- trio/tests/test_ssl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 573b6bd690..1a2e66212d 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -111,7 +111,10 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): # it's fixed in 'recent' CPython versions so we fail # the test for those and ignore it for earlier # versions. - if sys.platform != "cpython" or sys.version_info >= (3, 8): + if ( + sys.implementation.name != "cpython" + or sys.version_info >= (3, 8) + ): pytest.fail( "still an issue on recent python versions " "add a comment to " From 0cb25fae4248263dd14bc84586207bfc7ec776dc Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 15 Jun 2021 13:24:53 +0400 Subject: [PATCH 0730/1498] Bump version to 0.19.0 --- docs/source/history.rst | 23 ++++++++++++++++++++++ newsfragments/1864.bugfix.rst | 2 -- newsfragments/1924.bugfix.rst | 1 - newsfragments/1927.feature.rst | 1 - newsfragments/1948.feature.rst | 5 ----- newsfragments/README.rst | 35 ---------------------------------- trio/_version.py | 2 +- 7 files changed, 24 insertions(+), 45 deletions(-) delete mode 100644 newsfragments/1864.bugfix.rst delete mode 100644 newsfragments/1924.bugfix.rst delete mode 100644 newsfragments/1927.feature.rst delete mode 100644 newsfragments/1948.feature.rst delete mode 100644 newsfragments/README.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 48b821bd7c..0579d4420d 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,29 @@ Release history .. towncrier release notes start +Trio 0.19.0 (2021-06-15) +------------------------ + +Features +~~~~~~~~ + +- Trio now supports Python 3.10. (`#1921 `__) +- Use slots for :class:`~.lowlevel.Task` which should make them slightly smaller and faster. (`#1927 `__) +- Make :class:`~.Event` more lightweight by using less objects (about 2 rather + than 5, including a nested ParkingLot and attribute dicts) and simpler + structures (set rather than OrderedDict). This may benefit applications that + create a large number of event instances, such as with the "replace event + object on every set()" idiom. (`#1948 `__) + + +Bugfixes +~~~~~~~~ + +- The event loop now holds on to references of coroutine frames for only + the minimum necessary period of time. (`#1864 `__) +- The :class:`~.lowlevel.TrioToken` class can now be used as a target of a weak reference. (`#1924 `__) + + Trio 0.18.0 (2021-01-11) ------------------------ diff --git a/newsfragments/1864.bugfix.rst b/newsfragments/1864.bugfix.rst deleted file mode 100644 index 440984a091..0000000000 --- a/newsfragments/1864.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -The event loop now holds on to references of coroutine frames for only -the minimum necessary period of time. \ No newline at end of file diff --git a/newsfragments/1924.bugfix.rst b/newsfragments/1924.bugfix.rst deleted file mode 100644 index 351d424865..0000000000 --- a/newsfragments/1924.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -The :class:`~.lowlevel.TrioToken` class can now be used as a target of a weak reference. diff --git a/newsfragments/1927.feature.rst b/newsfragments/1927.feature.rst deleted file mode 100644 index cd5fc4e9f9..0000000000 --- a/newsfragments/1927.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Use slots for :class:`~.lowlevel.Task` which should make them slightly smaller and faster. diff --git a/newsfragments/1948.feature.rst b/newsfragments/1948.feature.rst deleted file mode 100644 index 6e8b7616b9..0000000000 --- a/newsfragments/1948.feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -Make :class:`~.Event` more lightweight by using less objects (about 2 rather -than 5, including a nested ParkingLot and attribute dicts) and simpler -structures (set rather than OrderedDict). This may benefit applications that -create a large number of event instances, such as with the "replace event -object on every set()" idiom. diff --git a/newsfragments/README.rst b/newsfragments/README.rst deleted file mode 100644 index 349e67eec0..0000000000 --- a/newsfragments/README.rst +++ /dev/null @@ -1,35 +0,0 @@ -This directory collects "newsfragments": short files that each contain -a snippet of ReST-formatted text that will be added to the next -release notes. This should be a description of aspects of the change -(if any) that are relevant to users. (This contrasts with your commit -message and PR description, which are a description of the change as -relevant to people working on the code itself.) - -Each file should be named like ``..rst``, where -```` is an issue number, and ```` is one of: - -* ``headline``: a major new feature we want to highlight for users -* ``breaking``: any breaking changes that happen without a proper - deprecation period (note: deprecations, and removal of previously - deprecated features after an appropriate time, go in the - ``deprecated`` category instead) -* ``feature``: any new feature that doesn't qualify for ``headline`` -* ``bugfix`` -* ``doc`` -* ``deprecated`` -* ``misc`` - -So for example: ``123.headline.rst``, ``456.bugfix.rst``, -``789.deprecated.rst`` - -If your PR fixes an issue, use that number here. If there is no issue, -then after you submit the PR and get the PR number you can add a -newsfragment using that instead. - -Your text can use all the same markup that we use in our Sphinx docs. -For example, you can use double-backticks to mark code snippets, or -single-backticks to link to a function/class/module. - -To check how your formatting looks, the easiest way is to make the PR, -and then after the CI checks run, click on the "Read the Docs build" -details link, and navigate to the release history page. diff --git a/trio/_version.py b/trio/_version.py index 48369a7a2b..3dc7bd1c1a 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.18.0+dev" +__version__ = "0.19.0" From 179aa06dfba605e61c09af445f028099857f59b7 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 15 Jun 2021 14:15:41 +0400 Subject: [PATCH 0731/1498] Bump version to 0.19.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index 3dc7bd1c1a..c99aa04a7e 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.19.0" +__version__ = "0.19.0+dev" From 01240e57ed2678570641bfa763abe77137dfbe9e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 18 Jun 2021 05:28:44 +0000 Subject: [PATCH 0732/1498] Bump prompt-toolkit from 3.0.18 to 3.0.19 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.18 to 3.0.19. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.18...3.0.19) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f23fd9a9f6..b70f689357 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -82,7 +82,7 @@ pickleshare==0.7.5 # via ipython pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.18 +prompt-toolkit==3.0.19 # via ipython ptyprocess==0.7.0 # via pexpect From 0639f87e58bae400070c5642f6fb339c7bb6e704 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 11:33:12 -0400 Subject: [PATCH 0733/1498] update tests for SSL related deprecations in 3.10 --- trio/tests/test_ssl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 75f99df137..933e0be470 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -80,7 +80,10 @@ def client_ctx(request): if request.param in ["default", "tls13"]: return ctx elif request.param == "tls12": - ctx.options |= ssl.OP_NO_TLSv1_3 + if sys.version_info >= (3, 7): + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + else: + ctx.options |= ssl.OP_NO_TLSv1_3 return ctx else: # pragma: no cover assert False @@ -1200,7 +1203,8 @@ async def test_selected_npn_protocol_before_handshake(client_ctx): @pytest.mark.filterwarnings( - r"ignore: ssl module. NPN is deprecated, use ALPN instead:UserWarning" + r"ignore: ssl module. NPN is deprecated, use ALPN instead:UserWarning", + r"ignore:ssl NPN is deprecated, use ALPN instead:DeprecationWarning", ) async def test_selected_npn_protocol_when_not_set(client_ctx): # NPN protocol still returns None when it's not set, From 00e6a6689c258eb43c51ade59ed6fb7907804a63 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 22 Jun 2021 16:40:22 +0000 Subject: [PATCH 0734/1498] Bump isort from 5.8.0 to 5.9.1 Bumps [isort](https://github.com/pycqa/isort) from 5.8.0 to 5.9.1. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.8.0...5.9.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b70f689357..3947d603f5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ ipython-genutils==0.2.0 # via traitlets ipython==7.24.1 # via -r test-requirements.in -isort==5.8.0 +isort==5.9.1 # via pylint jedi==0.18.0 # via From 9ab8ff48f28d9022a8c4786e668d5d99e75d55ac Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 22 Jun 2021 19:57:49 +0100 Subject: [PATCH 0735/1498] remove unused global trio._core._parking_lot._counter according to git -S the last occurance of next(_counter) was removed in 9cc347779ff520299f9b6639bc75e5cae2d898f7 --- trio/_core/_parking_lot.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/trio/_core/_parking_lot.py b/trio/_core/_parking_lot.py index e0f57ff8fa..f38123540f 100644 --- a/trio/_core/_parking_lot.py +++ b/trio/_core/_parking_lot.py @@ -70,15 +70,12 @@ # # See: https://github.com/python-trio/trio/issues/53 -from itertools import count import attr from collections import OrderedDict from .. import _core from .._util import Final -_counter = count() - @attr.s(frozen=True, slots=True) class _ParkingLotStatistics: From 8c8990590c370c5b2fd6ea0484d7265afe9d0a95 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 23 Jun 2021 05:21:11 +0000 Subject: [PATCH 0736/1498] Bump mypy from 0.902 to 0.910 Bumps [mypy](https://github.com/python/mypy) from 0.902 to 0.910. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.902...v0.910) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3947d603f5..82211a0a24 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -66,7 +66,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # -r test-requirements.in # black # mypy -mypy==0.902 ; implementation_name == "cpython" +mypy==0.910 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in From 7e2ea2de3db29ec396a81013c60f2b3a6bc76650 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 05:29:53 +0000 Subject: [PATCH 0737/1498] Bump ipython from 7.24.1 to 7.25.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.24.1 to 7.25.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.24.1...7.25.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 82211a0a24..89428ac435 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -45,7 +45,7 @@ iniconfig==1.1.1 # via pytest ipython-genutils==0.2.0 # via traitlets -ipython==7.24.1 +ipython==7.25.0 # via -r test-requirements.in isort==5.9.1 # via pylint From 1f523528976978d9f1aa5695f618f0b76fc616aa Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 05:30:44 +0000 Subject: [PATCH 0738/1498] Bump urllib3 from 1.26.5 to 1.26.6 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.5 to 1.26.6. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.6/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.5...1.26.6) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index d4988bfabc..2f3080e746 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -87,5 +87,5 @@ toml==0.10.2 # via towncrier towncrier==21.3.0 # via -r docs-requirements.in -urllib3==1.26.5 +urllib3==1.26.6 # via requests From 97279640615524ec4ee1bc60fc27775c26831eb4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 1 Jul 2021 05:30:59 +0000 Subject: [PATCH 0739/1498] Bump pylint from 2.8.3 to 2.9.1 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.8.3 to 2.9.1. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.8.3...v2.9.1) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 89428ac435..132f20e4ce 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.5.6 +astroid==2.6.2 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -96,7 +96,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.8.3 +pylint==2.9.1 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 490ae437a7d82fcd38bf2fc871a5deb5e9fe2211 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 2 Jul 2021 05:21:26 +0000 Subject: [PATCH 0740/1498] Bump regex from 2021.4.4 to 2021.7.1 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2021.4.4 to 2021.7.1. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 132f20e4ce..ef744c1433 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -108,7 +108,7 @@ pytest==6.2.4 # via # -r test-requirements.in # pytest-cov -regex==2021.4.4 +regex==2021.7.1 # via black six==1.16.0 # via pyopenssl From f20d574a82c12112eefef13501ea3892c0c8a4a2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 2 Jul 2021 05:22:24 +0000 Subject: [PATCH 0741/1498] Bump pylint from 2.9.1 to 2.9.3 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.9.1 to 2.9.3. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.9.1...v2.9.3) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 132f20e4ce..9f7406c32d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -96,7 +96,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.9.1 +pylint==2.9.3 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 3960130b99272a905dfb6e3ffd851ffe4e77df9c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 05:27:49 +0000 Subject: [PATCH 0742/1498] Bump packaging from 20.9 to 21.0 Bumps [packaging](https://github.com/pypa/packaging) from 20.9 to 21.0. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/20.9...21.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 2f3080e746..ad5f9348ef 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -46,7 +46,7 @@ markupsafe==2.0.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in -packaging==20.9 +packaging==21.0 # via sphinx pygments==2.7.4 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index b3a7d8f4cb..3d8d289800 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -70,7 +70,7 @@ mypy==0.910 ; implementation_name == "cpython" # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in -packaging==20.9 +packaging==21.0 # via pytest parso==0.8.2 # via jedi From c1956389ffe0d08c29fca019d8f1585a1278187f Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 5 Jul 2021 19:29:13 +0100 Subject: [PATCH 0743/1498] add starlette and aiometer to awesome-trio --- docs/source/awesome-trio-libraries.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index fd203bf3f7..895742a0c7 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -36,6 +36,7 @@ Web and HTML * `pyscalpel `__ - A fast and powerful webscraping library. * `muffin `_ - Muffin is a fast, simple ASGI web-framework * `asgi-tools `_ - Tools to quickly build lightest ASGI apps (also contains a test client with lifespan, websocket support) +* `starlette `_ - The little ASGI framework that shines. Database @@ -96,6 +97,7 @@ Tools and Utilities * `tricycle `__ - This is a library of interesting-but-maybe-not-yet-fully-proven extensions to Trio. * `tenacity `__ - Retrying library for Python with async/await support. * `perf-timer `__ - A code timer with Trio async support (see ``TrioPerfTimer``). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers ``trio_perf_counter()`` for low-level timing. +* `aiometer `__ - Execute lots of tasks concurrently while controlling concurrency limits Trio/Asyncio Interoperability From 4ca78452b937cfc8755d00b76a879264743bf3bc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 6 Jul 2021 05:24:06 +0000 Subject: [PATCH 0744/1498] Bump regex from 2021.7.1 to 2021.7.6 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2021.7.1 to 2021.7.6. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3d8d289800..11c5a03748 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -108,7 +108,7 @@ pytest==6.2.4 # via # -r test-requirements.in # pytest-cov -regex==2021.7.1 +regex==2021.7.6 # via black six==1.16.0 # via pyopenssl From c04fd877057363a3598cf9b7232e65fa36e6e7e8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 9 Jul 2021 05:17:16 +0000 Subject: [PATCH 0745/1498] Bump isort from 5.9.1 to 5.9.2 Bumps [isort](https://github.com/pycqa/isort) from 5.9.1 to 5.9.2. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.9.1...5.9.2) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 11c5a03748..765e87e025 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ ipython-genutils==0.2.0 # via traitlets ipython==7.25.0 # via -r test-requirements.in -isort==5.9.1 +isort==5.9.2 # via pylint jedi==0.18.0 # via From 356db30e901fcde82b8fd0acdd3c109ca61e2156 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 1 Jun 2020 19:37:43 -0700 Subject: [PATCH 0746/1498] First pass at implementing nursery.start(run_process, ...) --- trio/_subprocess.py | 72 ++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 876cc0d7c9..886e63edc4 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -424,6 +424,7 @@ async def run_process( capture_stderr=False, check=True, deliver_cancel=None, + task_status=trio.TASK_STATUS_IGNORED, **options, ): """Run ``command`` in a subprocess, wait for it to complete, and @@ -568,7 +569,7 @@ async def my_deliver_cancel(process): if isinstance(stdin, str): raise UnicodeError("process stdin must be bytes, not str") - if stdin == subprocess.PIPE: + if stdin == subprocess.PIPE and task_status is trio.TASK_STATUS_IGNORED: raise ValueError( "stdin=subprocess.PIPE doesn't make sense since the pipe " "is internal to run_process(); pass the actual data you " @@ -603,44 +604,49 @@ async def my_deliver_cancel(process): stdout_chunks = [] stderr_chunks = [] - async with await open_process(command, **options) as proc: - - async def feed_input(): - async with proc.stdin: - try: - await proc.stdin.send_all(input) - except trio.BrokenResourceError: - pass + async def feed_input(stream): + async with stream: + try: + await stream.send_all(input) + except trio.BrokenResourceError: + pass - async def read_output(stream, chunks): - async with stream: - async for chunk in stream: - chunks.append(chunk) + async def read_output(stream, chunks): + async with stream: + async for chunk in stream: + chunks.append(chunk) - async with trio.open_nursery() as nursery: - if proc.stdin is not None: - nursery.start_soon(feed_input) - if proc.stdout is not None: + async with trio.open_nursery() as nursery: + proc = await open_process(command, **options) + try: + if input is not None: + nursery.start_soon(feed_input, proc.stdin) + proc.stdin = None + proc.stdio = None + if capture_stdout: nursery.start_soon(read_output, proc.stdout, stdout_chunks) - if proc.stderr is not None: + proc.stdout = None + proc.stdio = None + if capture_stderr: nursery.start_soon(read_output, proc.stderr, stderr_chunks) - try: + proc.stderr = None + task_status.started(proc) + await proc.wait() + except BaseException: + with trio.CancelScope(shield=True): + killer_cscope = trio.CancelScope(shield=True) + + async def killer(): + with killer_cscope: + await deliver_cancel(proc) + + nursery.start_soon(killer) await proc.wait() - except trio.Cancelled: - with trio.CancelScope(shield=True): - killer_cscope = trio.CancelScope(shield=True) - - async def killer(): - with killer_cscope: - await deliver_cancel(proc) - - nursery.start_soon(killer) - await proc.wait() - killer_cscope.cancel() - raise + killer_cscope.cancel() + raise - stdout = b"".join(stdout_chunks) if proc.stdout is not None else None - stderr = b"".join(stderr_chunks) if proc.stderr is not None else None + stdout = b"".join(stdout_chunks) if capture_stdout else None + stderr = b"".join(stderr_chunks) if capture_stderr else None if proc.returncode and check: raise subprocess.CalledProcessError( From 0734cad2ce7de8d1f0b710b79ea5ed9cdbec6eae Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 2 Jun 2020 00:10:32 -0700 Subject: [PATCH 0747/1498] Deprecate Process.aclose and 'async with Process' --- docs/source/reference-io.rst | 2 - newsfragments/1104.removal.rst | 4 + trio/_subprocess.py | 34 ++++++--- trio/tests/test_subprocess.py | 131 +++++++++++++++++++++------------ 4 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 newsfragments/1104.removal.rst diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index ef971f6ea8..a0b8c77879 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -735,8 +735,6 @@ with it using the `Process` interface. .. autoattribute:: returncode - .. automethod:: aclose - .. automethod:: wait .. automethod:: poll diff --git a/newsfragments/1104.removal.rst b/newsfragments/1104.removal.rst new file mode 100644 index 0000000000..0ffb50fd3f --- /dev/null +++ b/newsfragments/1104.removal.rst @@ -0,0 +1,4 @@ +The `~Process.aclose` method on `Process` has been deprecated, and +``async with process_obj`` has also been deprecated. We recommend +switching to the new ``nursery.start(trio.run_process, ...)`` API +instead, which is more powerful and predictable. diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 886e63edc4..a4c79a2b8f 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -16,6 +16,7 @@ create_pipe_to_child_stdin, create_pipe_from_child_output, ) +from ._deprecate import deprecated from ._util import NoPublicConstructor import trio @@ -66,22 +67,19 @@ def pidfd_open(fd: int, flags: int) -> int: class Process(AsyncResource, metaclass=NoPublicConstructor): r"""A child process. Like :class:`subprocess.Popen`, but async. - This class has no public constructor. To create a child process, use - `open_process`:: + This class has no public constructor. The most common way to create a + `Process` is to combine `Nursery.start` with `run_process`:: - process = await trio.open_process(...) + process = await nursery.start(run_process, ...) - `Process` implements the `~trio.abc.AsyncResource` interface. In order to - make sure your process doesn't end up getting abandoned by mistake or - after an exception, you can use ``async with``:: + This way, `run_process` supervises the process, and makes sure that it is + cleaned up, while optionally checking the output, feeding it input, and so + on. - async with await trio.open_process(...) as process: - ... + If you need more control – for example, because you want to spawn a child + process that outlives your program – then you can use `open_process`:: - "Closing" a :class:`Process` will close any pipes to the child and wait - for it to exit; if cancelled, the child will be forcibly killed and we - will ensure it has finished exiting before allowing the cancellation to - propagate. + process = await trio.open_process(...) Attributes: args (str or list): The ``command`` passed at construction time, @@ -180,6 +178,18 @@ def returncode(self): self._close_pidfd() return result + @deprecated( + "0.16.0", + thing="using trio.Process as an async context manager", + issue=1104, + instead="run_process or nursery.start(run_process, ...)", + ) + async def __aenter__(self): + return self + + @deprecated( + "0.16.0", issue=1104, instead="run_process or nursery.start(run_process, ...)" + ) async def aclose(self): """Close any pipes we have to the process (both input and output) and wait for it to exit. diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 7ba794a428..1a5cf17196 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -5,6 +5,7 @@ import pytest import random from functools import partial +from async_generator import asynccontextmanager from .. import ( _core, @@ -16,6 +17,7 @@ open_process, run_process, TrioDeprecationWarning, + ClosedResourceError, ) from .._core.tests.tutil import slow, skip_if_fbsd_pipes_broken from ..testing import wait_all_tasks_blocked @@ -47,16 +49,25 @@ def got_signal(proc, sig): return proc.returncode != 0 +@asynccontextmanager +async def killing(proc): + try: + yield proc + finally: + proc.kill() + + async def test_basic(): - async with await open_process(EXIT_TRUE) as proc: - pass + async with killing(await open_process(EXIT_TRUE)) as proc: + await proc.wait() assert isinstance(proc, Process) assert proc._pidfd is None assert proc.returncode == 0 assert repr(proc) == f"" - async with await open_process(EXIT_FALSE) as proc: - pass + async with killing(await open_process(EXIT_FALSE)) as proc: + await proc.wait() + await proc.wait() assert proc.returncode == 1 assert repr(proc) == "".format( EXIT_FALSE, "exited with status 1" @@ -64,19 +75,19 @@ async def test_basic(): async def test_auto_update_returncode(): - p = await open_process(SLEEP(9999)) - assert p.returncode is None - assert "running" in repr(p) - p.kill() - p._proc.wait() - assert p.returncode is not None - assert "exited" in repr(p) - assert p._pidfd is None - assert p.returncode is not None + async with killing(await open_process(SLEEP(9999))) as p: + assert p.returncode is None + assert "running" in repr(p) + p.kill() + p._proc.wait() + assert p.returncode is not None + assert "exited" in repr(p) + assert p._pidfd is None + assert p.returncode is not None async def test_multi_wait(): - async with await open_process(SLEEP(10)) as proc: + async with killing(await open_process(SLEEP(10))) as proc: # Check that wait (including multi-wait) tolerates being cancelled async with _core.open_nursery() as nursery: nursery.start_soon(proc.wait) @@ -94,7 +105,21 @@ async def test_multi_wait(): proc.kill() -async def test_kill_when_context_cancelled(): +# Test for deprecated 'async with process:' semantics +async def test_async_with_basics_deprecated(recwarn): + async with await open_process( + CAT, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) as proc: + pass + assert proc.returncode == 0 + with pytest.raises(ClosedResourceError): + await proc.stdin.send_all(b"x") + with pytest.raises(ClosedResourceError): + await proc.stdout.receive_some() + + +# Test for deprecated 'async with process:' semantics +async def test_kill_when_context_cancelled(recwarn): with move_on_after(100) as scope: async with await open_process(SLEEP(10)) as proc: assert proc.poll() is None @@ -115,11 +140,13 @@ async def test_kill_when_context_cancelled(): async def test_pipes(): - async with await open_process( - COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + async with killing( + await open_process( + COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) ) as proc: msg = b"the quick brown fox jumps over the lazy dog" @@ -156,20 +183,22 @@ async def test_interactive(): # out: EOF # err: EOF - async with await open_process( - python( - "idx = 0\n" - "while True:\n" - " line = sys.stdin.readline()\n" - " if line == '': break\n" - " request = int(line.strip())\n" - " print(str(idx * 2) * request)\n" - " print(str(idx * 2 + 1) * request * 2, file=sys.stderr)\n" - " idx += 1\n" - ), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + async with killing( + await open_process( + python( + "idx = 0\n" + "while True:\n" + " line = sys.stdin.readline()\n" + " if line == '': break\n" + " request = int(line.strip())\n" + " print(str(idx * 2) * request)\n" + " print(str(idx * 2 + 1) * request * 2, file=sys.stderr)\n" + " idx += 1\n" + ), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) ) as proc: newline = b"\n" if posix else b"\r\n" @@ -279,11 +308,13 @@ async def test_run_with_broken_pipe(): async def test_stderr_stdout(): - async with await open_process( - COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + async with killing( + await open_process( + COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) ) as proc: assert proc.stdout is not None assert proc.stderr is None @@ -312,23 +343,26 @@ async def test_stderr_stdout(): # this one hits the branch where stderr=STDOUT but stdout # is not redirected - async with await open_process( - CAT, stdin=subprocess.PIPE, stderr=subprocess.STDOUT + async with killing( + await open_process(CAT, stdin=subprocess.PIPE, stderr=subprocess.STDOUT) ) as proc: assert proc.stdout is None assert proc.stderr is None await proc.stdin.aclose() + await proc.wait() assert proc.returncode == 0 if posix: try: r, w = os.pipe() - async with await open_process( - COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, - stdin=subprocess.PIPE, - stdout=w, - stderr=subprocess.STDOUT, + async with killing( + await open_process( + COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, + stdin=subprocess.PIPE, + stdout=w, + stderr=subprocess.STDOUT, + ) ) as proc: os.close(w) assert proc.stdio is None @@ -359,8 +393,9 @@ async def test_errors(): async def test_signals(): async def test_one_signal(send_it, signum): with move_on_after(1.0) as scope: - async with await open_process(SLEEP(3600)) as proc: + async with killing(await open_process(SLEEP(3600))) as proc: send_it(proc) + await proc.wait() assert not scope.cancelled_caught if posix: assert proc.returncode == -signum @@ -387,7 +422,7 @@ async def test_wait_reapable_fails(): # With SIGCHLD disabled, the wait() syscall will wait for the # process to exit but then fail with ECHILD. Make sure we # support this case as the stdlib subprocess module does. - async with await open_process(SLEEP(3600)) as proc: + async with killing(await open_process(SLEEP(3600))) as proc: async with _core.open_nursery() as nursery: nursery.start_soon(proc.wait) await wait_all_tasks_blocked() From de51f9c11f8fd25e5130d586d6ae038963c78548 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 2 Jun 2020 17:35:25 -0700 Subject: [PATCH 0748/1498] Attempt to fix CI failures --- newsfragments/1104.removal.rst | 8 ++++---- trio/tests/test_subprocess.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/newsfragments/1104.removal.rst b/newsfragments/1104.removal.rst index 0ffb50fd3f..406230ed08 100644 --- a/newsfragments/1104.removal.rst +++ b/newsfragments/1104.removal.rst @@ -1,4 +1,4 @@ -The `~Process.aclose` method on `Process` has been deprecated, and -``async with process_obj`` has also been deprecated. We recommend -switching to the new ``nursery.start(trio.run_process, ...)`` API -instead, which is more powerful and predictable. +The ``aclose`` method on `Process` has been deprecated, and ``async +with process_obj`` has also been deprecated. We recommend switching to +the new ``nursery.start(trio.run_process, ...)`` API instead, which is +more powerful and predictable. diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 1a5cf17196..27a2e1985d 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -111,7 +111,7 @@ async def test_async_with_basics_deprecated(recwarn): CAT, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) as proc: pass - assert proc.returncode == 0 + assert proc.returncode is not None with pytest.raises(ClosedResourceError): await proc.stdin.send_all(b"x") with pytest.raises(ClosedResourceError): @@ -238,6 +238,7 @@ async def drain_one(stream, count, digit): await proc.stdin.aclose() assert await proc.stdout.receive_some(1) == b"" assert await proc.stderr.receive_some(1) == b"" + await proc.wait() assert proc.returncode == 0 From 3fae5aa84f43dfcad1b13604f1ceadf58b96c6f6 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 5 Jun 2020 09:44:02 -0700 Subject: [PATCH 0749/1498] Redo docs --- docs/source/reference-io.rst | 99 +++++++++--------------- trio/_subprocess.py | 145 ++++++++++++++++++++++++----------- 2 files changed, 135 insertions(+), 109 deletions(-) diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index a0b8c77879..77ae4cbd88 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -664,22 +664,43 @@ Spawning subprocesses Trio provides support for spawning other programs as subprocesses, communicating with them via pipes, sending them signals, and waiting -for them to exit. The interface for doing so consists of two layers: +for them to exit. -* :func:`trio.run_process` runs a process from start to - finish and returns a :class:`~subprocess.CompletedProcess` object describing - its outputs and return value. This is what you should reach for if you - want to run a process to completion before continuing, while possibly - sending it some input or capturing its output. It is modelled after - the standard :func:`subprocess.run` with some additional features - and safer defaults. +Most of the time, this is done through our high-level interface, +`trio.run_process`. It lets you either run a process to completion +while optionally capturing the output, or else run it in a background +task and interact with it while it's running: -* `trio.open_process` starts a process in the background and returns a - `Process` object to let you interact with it. Using it requires a - bit more code than `run_process`, but exposes additional - capabilities: back-and-forth communication, processing output as - soon as it is generated, and so forth. It is modelled after the - standard library :class:`subprocess.Popen`. +.. autofunction:: trio.run_process + +.. autoclass:: trio.Process + + .. autoattribute:: returncode + + .. automethod:: wait + + .. automethod:: poll + + .. automethod:: kill + + .. automethod:: terminate + + .. automethod:: send_signal + + .. note:: :meth:`~subprocess.Popen.communicate` is not provided as a + method on :class:`~trio.Process` objects; call :func:`~trio.run_process` + normally for simple capturing, or write the loop yourself if you + have unusual needs. :meth:`~subprocess.Popen.communicate` has + quite unusual cancellation behavior in the standard library (on + some platforms it spawns a background thread which continues to + read from the child process even after the timeout has expired) + and we wanted to provide an interface with fewer surprises. + +If `trio.run_process` is too limiting, we also offer a low-level API, +`trio.open_process`. For example, use `open_process` if you want to +spawn a child process that outlives the parent process: + +.. autofunction:: trio.open_process .. _subprocess-options: @@ -705,56 +726,6 @@ with a process, so it does not support the ``encoding``, ``errors``, options. -Running a process and waiting for it to finish -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The basic interface for running a subprocess start-to-finish is -:func:`trio.run_process`. It always waits for the subprocess to exit -before returning, so there's no need to worry about leaving a process -running by mistake after you've gone on to do other things. -:func:`~trio.run_process` is similar to the standard library -:func:`subprocess.run` function, but tries to have safer defaults: -with no options, the subprocess's input is empty rather than coming -from the user's terminal, and a failure in the subprocess will be -propagated as a :exc:`subprocess.CalledProcessError` exception. Of -course, these defaults can be changed where necessary. - -.. autofunction:: trio.run_process - - -Interacting with a process as it runs -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want more control than :func:`~trio.run_process` affords, you -can use `trio.open_process` to spawn a subprocess, and then interact -with it using the `Process` interface. - -.. autofunction:: trio.open_process - -.. autoclass:: trio.Process - - .. autoattribute:: returncode - - .. automethod:: wait - - .. automethod:: poll - - .. automethod:: kill - - .. automethod:: terminate - - .. automethod:: send_signal - - .. note:: :meth:`~subprocess.Popen.communicate` is not provided as a - method on :class:`~trio.Process` objects; use :func:`~trio.run_process` - instead, or write the loop yourself if you have unusual - needs. :meth:`~subprocess.Popen.communicate` has quite unusual - cancellation behavior in the standard library (on some platforms it - spawns a background thread which continues to read from the child - process even after the timeout has expired) and we wanted to - provide an interface with fewer surprises. - - .. _subprocess-quoting: Quoting: more than you wanted to know diff --git a/trio/_subprocess.py b/trio/_subprocess.py index a4c79a2b8f..aab9b84266 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -67,19 +67,20 @@ def pidfd_open(fd: int, flags: int) -> int: class Process(AsyncResource, metaclass=NoPublicConstructor): r"""A child process. Like :class:`subprocess.Popen`, but async. - This class has no public constructor. The most common way to create a - `Process` is to combine `Nursery.start` with `run_process`:: + This class has no public constructor. The most common way to get a + `Process` object is to combine `Nursery.start` with `run_process`:: - process = await nursery.start(run_process, ...) + process_object = await nursery.start(run_process, ...) - This way, `run_process` supervises the process, and makes sure that it is - cleaned up, while optionally checking the output, feeding it input, and so - on. + This way, `run_process` supervises the process and makes sure that it is + cleaned up properly, while optionally checking the return value, feeding + it input, and so on. If you need more control – for example, because you want to spawn a child - process that outlives your program – then you can use `open_process`:: + process that outlives your program – then another option is to use + `open_process`:: - process = await trio.open_process(...) + process_object = await trio.open_process(...) Attributes: args (str or list): The ``command`` passed at construction time, @@ -437,21 +438,39 @@ async def run_process( task_status=trio.TASK_STATUS_IGNORED, **options, ): - """Run ``command`` in a subprocess, wait for it to complete, and - return a :class:`subprocess.CompletedProcess` instance describing - the results. - - If cancelled, :func:`run_process` terminates the subprocess and - waits for it to exit before propagating the cancellation, like - :meth:`Process.aclose`. - - **Input:** The subprocess's standard input stream is set up to - receive the bytes provided as ``stdin``. Once the given input has - been fully delivered, or if none is provided, the subprocess will - receive end-of-file when reading from its standard input. - Alternatively, if you want the subprocess to read its - standard input from the same place as the parent Trio process, you - can pass ``stdin=None``. + """Run ``command`` in a subprocess and wait for it to complete. + + This function can be called in two different ways. + + One option is a direct call, like:: + + completed_process_info = await trio.run_process(...) + + In this case, it returns a :class:`subprocess.CompletedProcess` instance + describing the results. Use this if you want to treat a process like a + function call. + + The other option is to run it as a task using `Nursery.start` – the enhanced version + of `~Nursery.start_soon` that lets a task pass back a value during startup:: + + process = await nursery.start(trio.run_process, ...) + + In this case, `~Nursery.start` returns a `Process` object that you can use + to interact with the process while it's running. Use this if you want to + treat a process like a background task. + + Either way, `run_process` makes sure that the process has exited before + returning, handles cancellation, optionally checks for errors, and + provides some convenient shorthands for dealing with the child's + input/output. + + **Input:** `run_process` supports all the same ``stdin=`` arguments as + `subprocess.Popen`. In addition, if you simply want to pass in some fixed + data, you can pass a plain `bytes` object, and `run_process` will take + care of setting up a pipe, feeding in the data you gave, and then sending + end-of-file. The default is ``b""``, which means that the child will receive + an empty stdin. If you want the child to instead read from the parent's + stdin, use ``stdin=None``. **Output:** By default, any output produced by the subprocess is passed through to the standard output and error streams of the @@ -481,8 +500,28 @@ async def run_process( the :attr:`~subprocess.CalledProcessError.stdout` and :attr:`~subprocess.CalledProcessError.stderr` attributes of that exception. To disable this behavior, so that :func:`run_process` - returns normally even if the subprocess exits abnormally, pass - ``check=False``. + returns normally even if the subprocess exits abnormally, pass ``check=False``. + + Note that this can make the ``capture_stdout`` and ``capture_stderr`` + arguments useful even when starting `run_process` as a task: if you only + care about the output if the process fails, then you can enable capturing + and then read the output off of the `~subprocess.CalledProcessError`. + + **Cancellation:** If cancelled, `run_process` sends a termination + request to the subprocess, then waits for it to fully exit. The + ``deliver_cancel`` argument lets you control how the process is terminated. + + .. note:: `run_process` is intentionally similar to the standard library + `subprocess.run`, but some of the defaults are different. Specifically, we + default to: + + - ``check=True``, because `"errors should never pass silently / unless + explicitly silenced "`__. + + - ``stdin=b""``, because it produces less-confusing results if a subprocess + unexpectedly tries to read from stdin. + + To get the `subprocess.run` semantics, use ``check=False, stdin=None``. Args: command (list or str): The command to run. Typically this is a @@ -493,24 +532,26 @@ async def run_process( be a string, which will be parsed following platform-dependent :ref:`quoting rules `. - stdin (:obj:`bytes`, file descriptor, or None): The bytes to provide to - the subprocess on its standard input stream, or ``None`` if the - subprocess's standard input should come from the same place as - the parent Trio process's standard input. As is the case with - the :mod:`subprocess` module, you can also pass a - file descriptor or an object with a ``fileno()`` method, - in which case the subprocess's standard input will come from - that file. + stdin (:obj:`bytes`, subprocess.PIPE, file descriptor, or None): The + bytes to provide to the subprocess on its standard input stream, or + ``None`` if the subprocess's standard input should come from the + same place as the parent Trio process's standard input. As is the + case with the :mod:`subprocess` module, you can also pass a file + descriptor or an object with a ``fileno()`` method, in which case + the subprocess's standard input will come from that file. And when + starting `run_process` as a background task, you can use + ``stdin=subprocess.PIPE``, in which case `Process.stdin` will be a + `~trio.abc.SendStream` that you can use to send data to the child. capture_stdout (bool): If true, capture the bytes that the subprocess writes to its standard output stream and return them in the - :attr:`~subprocess.CompletedProcess.stdout` attribute - of the returned :class:`~subprocess.CompletedProcess` object. + `~subprocess.CompletedProcess.stdout` attribute of the returned + `subprocess.CompletedProcess` or `subprocess.CalledProcessError`. capture_stderr (bool): If true, capture the bytes that the subprocess writes to its standard error stream and return them in the - :attr:`~subprocess.CompletedProcess.stderr` attribute - of the returned :class:`~subprocess.CompletedProcess` object. + `~subprocess.CompletedProcess.stderr` attribute of the returned + `~subprocess.CompletedProcess` or `subprocess.CalledProcessError`. check (bool): If false, don't validate that the subprocess exits successfully. You should be sure to check the @@ -555,8 +596,11 @@ async def my_deliver_cancel(process): ``stdout=subprocess.DEVNULL``, or file descriptors. Returns: - A :class:`subprocess.CompletedProcess` instance describing the - return code and outputs. + + When called normally – a `subprocess.CompletedProcess` instance + describing the return code and outputs. + + When called via `Nursery.start` – a `trio.Process` instance. Raises: UnicodeError: if ``stdin`` is specified as a Unicode string, rather @@ -579,12 +623,23 @@ async def my_deliver_cancel(process): if isinstance(stdin, str): raise UnicodeError("process stdin must be bytes, not str") - if stdin == subprocess.PIPE and task_status is trio.TASK_STATUS_IGNORED: - raise ValueError( - "stdin=subprocess.PIPE doesn't make sense since the pipe " - "is internal to run_process(); pass the actual data you " - "want to send over that pipe instead" - ) + if task_status is trio.TASK_STATUS_IGNORED: + if stdin is subprocess.PIPE: + raise ValueError( + "stdin=subprocess.PIPE doesn't make sense without " + "nursery.start, since there's no way to access the " + "pipe; pass the data you want to send or use nursery.start" + ) + if options.get("stdout") is subprocess.PIPE: + raise ValueError( + "stdout=subprocess.PIPE doesn't make sense without " + "nursery.start, since there's no way to access the pipe" + ) + if options.get("stderr") is subprocess.PIPE: + raise ValueError( + "stderr=subprocess.PIPE doesn't make sense without " + "nursery.start, since there's no way to access the pipe" + ) if isinstance(stdin, (bytes, bytearray, memoryview)): input = stdin options["stdin"] = subprocess.PIPE From 5bd9600666961c4820d9fffc5d3f9aef2cfd66d1 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 09:24:10 -0700 Subject: [PATCH 0750/1498] fix ReST syntax --- trio/_subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index aab9b84266..44ebc0997c 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -516,7 +516,7 @@ async def run_process( default to: - ``check=True``, because `"errors should never pass silently / unless - explicitly silenced "`__. + explicitly silenced" `__. - ``stdin=b""``, because it produces less-confusing results if a subprocess unexpectedly tries to read from stdin. From 14a4b916ec624c77497da8aaae41bff3e8105d82 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 09:27:16 -0700 Subject: [PATCH 0751/1498] In test helper, wait for process to be fully dead before continuing Fixes some race conditions in tests that make assertions about proc.returncode after exiting the `async with killing` block. --- trio/tests/test_subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 27a2e1985d..2fdd5a2e45 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -55,6 +55,7 @@ async def killing(proc): yield proc finally: proc.kill() + await proc.wait() async def test_basic(): @@ -67,7 +68,6 @@ async def test_basic(): async with killing(await open_process(EXIT_FALSE)) as proc: await proc.wait() - await proc.wait() assert proc.returncode == 1 assert repr(proc) == "".format( EXIT_FALSE, "exited with status 1" From 7e742f310febb925d9428af614e0c690991f0a1f Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 10:59:34 -0700 Subject: [PATCH 0752/1498] Tests for nursery.start(run_subprocess, ...) --- trio/tests/test_subprocess.py | 109 +++++++++++++++++++++------------- 1 file changed, 68 insertions(+), 41 deletions(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 2fdd5a2e45..86cde0773e 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -50,7 +50,8 @@ def got_signal(proc, sig): @asynccontextmanager -async def killing(proc): +async def open_process_then_kill(*args, **kwargs): + proc = await open_process(*args, **kwargs) try: yield proc finally: @@ -58,15 +59,32 @@ async def killing(proc): await proc.wait() -async def test_basic(): - async with killing(await open_process(EXIT_TRUE)) as proc: +@asynccontextmanager +async def run_process_in_nursery(*args, **kwargs): + async with _core.open_nursery() as nursery: + kwargs.setdefault("check", False) + proc = await nursery.start(partial(run_process, *args, **kwargs)) + yield proc + nursery.cancel_scope.cancel() + + +background_process_param = pytest.mark.parametrize( + "background_process", + [open_process_then_kill, run_process_in_nursery], + ids=["open_process", "run_process in nursery"] +) + + +@background_process_param +async def test_basic(background_process): + async with background_process(EXIT_TRUE) as proc: await proc.wait() assert isinstance(proc, Process) assert proc._pidfd is None assert proc.returncode == 0 assert repr(proc) == f"" - async with killing(await open_process(EXIT_FALSE)) as proc: + async with background_process(EXIT_FALSE) as proc: await proc.wait() assert proc.returncode == 1 assert repr(proc) == "".format( @@ -74,8 +92,9 @@ async def test_basic(): ) -async def test_auto_update_returncode(): - async with killing(await open_process(SLEEP(9999))) as p: +@background_process_param +async def test_auto_update_returncode(background_process): + async with background_process(SLEEP(9999)) as p: assert p.returncode is None assert "running" in repr(p) p.kill() @@ -86,8 +105,9 @@ async def test_auto_update_returncode(): assert p.returncode is not None -async def test_multi_wait(): - async with killing(await open_process(SLEEP(10))) as proc: +@background_process_param +async def test_multi_wait(background_process): + async with background_process(SLEEP(10)) as proc: # Check that wait (including multi-wait) tolerates being cancelled async with _core.open_nursery() as nursery: nursery.start_soon(proc.wait) @@ -139,14 +159,13 @@ async def test_kill_when_context_cancelled(recwarn): ) -async def test_pipes(): - async with killing( - await open_process( - COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) +@background_process_param +async def test_pipes(background_process): + async with background_process( + COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) as proc: msg = b"the quick brown fox jumps over the lazy dog" @@ -171,7 +190,8 @@ async def check_output(stream, expected): assert 0 == await proc.wait() -async def test_interactive(): +@background_process_param +async def test_interactive(background_process): # Test some back-and-forth with a subprocess. This one works like so: # in: 32\n # out: 0000...0000\n (32 zeroes) @@ -183,8 +203,7 @@ async def test_interactive(): # out: EOF # err: EOF - async with killing( - await open_process( + async with background_process( python( "idx = 0\n" "while True:\n" @@ -198,7 +217,6 @@ async def test_interactive(): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) ) as proc: newline = b"\n" if posix else b"\r\n" @@ -239,6 +257,7 @@ async def drain_one(stream, count, digit): assert await proc.stdout.receive_some(1) == b"" assert await proc.stderr.receive_some(1) == b"" await proc.wait() + assert proc.returncode == 0 @@ -308,14 +327,13 @@ async def test_run_with_broken_pipe(): assert result.stdout is result.stderr is None -async def test_stderr_stdout(): - async with killing( - await open_process( - COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) +@background_process_param +async def test_stderr_stdout(background_process): + async with background_process( + COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, ) as proc: assert proc.stdout is not None assert proc.stderr is None @@ -344,8 +362,8 @@ async def test_stderr_stdout(): # this one hits the branch where stderr=STDOUT but stdout # is not redirected - async with killing( - await open_process(CAT, stdin=subprocess.PIPE, stderr=subprocess.STDOUT) + async with background_process( + CAT, stdin=subprocess.PIPE, stderr=subprocess.STDOUT ) as proc: assert proc.stdout is None assert proc.stderr is None @@ -357,13 +375,11 @@ async def test_stderr_stdout(): try: r, w = os.pipe() - async with killing( - await open_process( - COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, - stdin=subprocess.PIPE, - stdout=w, - stderr=subprocess.STDOUT, - ) + async with background_process( + COPY_STDIN_TO_STDOUT_AND_BACKWARD_TO_STDERR, + stdin=subprocess.PIPE, + stdout=w, + stderr=subprocess.STDOUT, ) as proc: os.close(w) assert proc.stdio is None @@ -391,10 +407,11 @@ async def test_errors(): await open_process("ls", shell=False) -async def test_signals(): +@background_process_param +async def test_signals(background_process): async def test_one_signal(send_it, signum): with move_on_after(1.0) as scope: - async with killing(await open_process(SLEEP(3600))) as proc: + async with background_process(SLEEP(3600)) as proc: send_it(proc) await proc.wait() assert not scope.cancelled_caught @@ -417,13 +434,14 @@ async def test_one_signal(send_it, signum): @pytest.mark.skipif(not posix, reason="POSIX specific") -async def test_wait_reapable_fails(): +@background_process_param +async def test_wait_reapable_fails(background_process): old_sigchld = signal.signal(signal.SIGCHLD, signal.SIG_IGN) try: # With SIGCHLD disabled, the wait() syscall will wait for the # process to exit but then fail with ECHILD. Make sure we # support this case as the stdlib subprocess module does. - async with killing(await open_process(SLEEP(3600))) as proc: + async with background_process(SLEEP(3600)) as proc: async with _core.open_nursery() as nursery: nursery.start_soon(proc.wait) await wait_all_tasks_blocked() @@ -516,3 +534,12 @@ async def test_warn_on_cancel_SIGKILL_escalation(autojump_clock, monkeypatch): nursery.start_soon(run_process, SLEEP(9999)) await wait_all_tasks_blocked() nursery.cancel_scope.cancel() + + +# the background_process_param exercises a lot of run_process cases, but it uses +# check=False, so lets have a test that uses check=True as well +async def test_run_process_background_fail(): + with pytest.raises(subprocess.CalledProcessError): + async with _core.open_nursery() as nursery: + proc = await nursery.start(run_process, EXIT_FALSE) + assert proc.returncode == 1 From c48123abbe8f8c11b2b4a50dcc3ace000e430905 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 11:02:08 -0700 Subject: [PATCH 0753/1498] blacken --- trio/tests/test_subprocess.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 86cde0773e..776510d9d5 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -71,7 +71,7 @@ async def run_process_in_nursery(*args, **kwargs): background_process_param = pytest.mark.parametrize( "background_process", [open_process_then_kill, run_process_in_nursery], - ids=["open_process", "run_process in nursery"] + ids=["open_process", "run_process in nursery"], ) @@ -204,19 +204,19 @@ async def test_interactive(background_process): # err: EOF async with background_process( - python( - "idx = 0\n" - "while True:\n" - " line = sys.stdin.readline()\n" - " if line == '': break\n" - " request = int(line.strip())\n" - " print(str(idx * 2) * request)\n" - " print(str(idx * 2 + 1) * request * 2, file=sys.stderr)\n" - " idx += 1\n" - ), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + python( + "idx = 0\n" + "while True:\n" + " line = sys.stdin.readline()\n" + " if line == '': break\n" + " request = int(line.strip())\n" + " print(str(idx * 2) * request)\n" + " print(str(idx * 2 + 1) * request * 2, file=sys.stderr)\n" + " idx += 1\n" + ), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) as proc: newline = b"\n" if posix else b"\r\n" From a7e771000acf9050bbe8177d4baead9626178fff Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 11:03:54 -0700 Subject: [PATCH 0754/1498] Add feature newsfragment --- newsfragments/1104.feature.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 newsfragments/1104.feature.rst diff --git a/newsfragments/1104.feature.rst b/newsfragments/1104.feature.rst new file mode 100644 index 0000000000..b2d4ceaa11 --- /dev/null +++ b/newsfragments/1104.feature.rst @@ -0,0 +1,3 @@ +You can now conveniently spawn a child process in a background task +and interact it with on the fly using ``process = await +nursery.start(run_process, ...)``. See `run_process` for more details. From 32b1c13a2b91e0af282bfb8c201837d5a5fd2479 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 11:59:04 -0700 Subject: [PATCH 0755/1498] Add a bit more coverage --- trio/tests/test_subprocess.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 776510d9d5..5666ded8e8 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -294,6 +294,10 @@ async def test_run(): await run_process(CAT, stdin="oh no, it's text") with pytest.raises(ValueError): await run_process(CAT, stdin=subprocess.PIPE) + with pytest.raises(ValueError): + await run_process(CAT, stdout=subprocess.PIPE) + with pytest.raises(ValueError): + await run_process(CAT, stderr=subprocess.PIPE) with pytest.raises(ValueError): await run_process(CAT, capture_stdout=True, stdout=subprocess.DEVNULL) with pytest.raises(ValueError): From a9c54747b90129776f6d2d384e0f64efaac324ee Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 17:34:26 -0700 Subject: [PATCH 0756/1498] Export IP_RECVTOS, newly added in python 3.10 --- trio/socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/socket.py b/trio/socket.py index 5402f5bc73..afbcefa1d3 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -117,7 +117,7 @@ J1939_PGN_MAX, J1939_PGN_PDU1_MAX, J1939_PGN_REQUEST, SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, SCM_J1939_PRIO, SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, - SO_J1939_SEND_PRIO, UDPLITE_RECV_CSCOV, UDPLITE_SEND_CSCOV + SO_J1939_SEND_PRIO, UDPLITE_RECV_CSCOV, UDPLITE_SEND_CSCOV, IP_RECVTOS ) # fmt: on except ImportError: From 6f98761f826427777cebdc4d00288a5eb7039307 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 18:47:56 -0700 Subject: [PATCH 0757/1498] =?UTF-8?q?Rename=20trio.open=5Fprocess=20?= =?UTF-8?q?=E2=86=92=20trio.lowlevel.open=5Fprocess?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/history.rst | 6 +++--- docs/source/reference-io.rst | 7 +++---- docs/source/reference-lowlevel.rst | 6 ++++++ newsfragments/1104.removal.rst | 9 +++++---- trio/__init__.py | 11 ++++++++++- trio/_subprocess.py | 8 ++++++-- trio/_subprocess_platform/__init__.py | 7 +++---- trio/lowlevel.py | 2 ++ trio/tests/test_subprocess.py | 2 +- 9 files changed, 39 insertions(+), 19 deletions(-) diff --git a/docs/source/history.rst b/docs/source/history.rst index 0579d4420d..3936df867e 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -213,7 +213,7 @@ Features worry: you can now pass a custom ``deliver_cancel=`` argument to define your own process killing policy. (`#1104 `__) - It turns out that creating a subprocess can block the parent process - for a surprisingly long time. So `trio.open_process` now uses a worker + for a surprisingly long time. So ``trio.open_process`` now uses a worker thread to avoid blocking the event loop. (`#1109 `__) - We've added FreeBSD to the list of platforms we support and test on. (`#1118 `__) - On Linux kernels v5.3 or newer, `trio.Process.wait` now uses `the @@ -267,7 +267,7 @@ Deprecations and Removals alternatives or make a case for why some particular class should be designed to support subclassing. (`#1044 `__) - If you want to create a `trio.Process` object, you now have to call - `trio.open_process`; calling ``trio.Process()`` directly was + ``trio.open_process``; calling ``trio.Process()`` directly was deprecated in v0.12.0 and has now been removed. (`#1109 `__) - Remove ``clear`` method on `trio.Event`: it was deprecated in 0.12.0. (`#1498 `__) @@ -495,7 +495,7 @@ Deprecations and Removals deprecated. (`#878 `__) - It turns out that it's better to treat subprocess spawning as an async operation. Therefore, direct construction of `Process` objects has - been deprecated. Use `trio.open_process` instead. (`#1109 `__) + been deprecated. Use ``trio.open_process`` instead. (`#1109 `__) Miscellaneous internal changes diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index 77ae4cbd88..3a082982e0 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -697,10 +697,9 @@ task and interact with it while it's running: and we wanted to provide an interface with fewer surprises. If `trio.run_process` is too limiting, we also offer a low-level API, -`trio.open_process`. For example, use `open_process` if you want to -spawn a child process that outlives the parent process: - -.. autofunction:: trio.open_process +`trio.lowlevel.open_process`. For example, use +`~trio.lowlevel.open_process` if you want to spawn a child process +that outlives the parent process: .. _subprocess-options: diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 2fe773cbb7..e0a604bebc 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -106,6 +106,12 @@ The tutorial has a :ref:`fully-worked example Trio's internal scheduling decisions. +Low-level process spawning +========================== + +.. autofunction:: trio.lowlevel.open_process + + Low-level I/O primitives ======================== diff --git a/newsfragments/1104.removal.rst b/newsfragments/1104.removal.rst index 406230ed08..f2a5a0a997 100644 --- a/newsfragments/1104.removal.rst +++ b/newsfragments/1104.removal.rst @@ -1,4 +1,5 @@ -The ``aclose`` method on `Process` has been deprecated, and ``async -with process_obj`` has also been deprecated. We recommend switching to -the new ``nursery.start(trio.run_process, ...)`` API instead, which is -more powerful and predictable. +``trio.open_process`` has been renamed to +`trio.lowlevel.open_process`, and the ``aclose`` method on `Process` +has been deprecated, along with ``async with process_obj``. We +recommend most users switch to the new +``nursery.start(trio.run_process, ...)`` API instead. diff --git a/trio/__init__.py b/trio/__init__.py index d66ffceea9..6a43c42d2a 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -70,7 +70,7 @@ from ._path import Path -from ._subprocess import Process, open_process, run_process +from ._subprocess import Process, run_process from ._ssl import SSLStream, SSLListener, NeedHandshakeError @@ -106,6 +106,15 @@ _deprecate.enable_attribute_deprecations(__name__) +__deprecated_attributes__ = { + "open_process": _deprecate.DeprecatedAttribute( + value=_subprocess.open_process, + version="0.20.0", + issue=1104, + instead="trio.lowlevel.open_process", + ), +} + # Having the public path in .__module__ attributes is important for: # - exception names in printed tracebacks # - sphinx :show-inheritance: diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 44ebc0997c..ec623f19c7 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -78,9 +78,9 @@ class Process(AsyncResource, metaclass=NoPublicConstructor): If you need more control – for example, because you want to spawn a child process that outlives your program – then another option is to use - `open_process`:: + `trio.lowlevel.open_process`:: - process_object = await trio.open_process(...) + process_object = await trio.lowlevel.open_process(...) Attributes: args (str or list): The ``command`` passed at construction time, @@ -304,6 +304,10 @@ async def open_process( can write to the `~Process.stdin` stream, else `~Process.stdin` will be ``None``. + Unlike `trio.run_process`, this function doesn't do any kind of automatic + management of the child process. It's up to you to implement whatever semantics you + want. + Args: command (list or str): The command to run. Typically this is a sequence of strings such as ``['ls', '-l', 'directory with spaces']``, diff --git a/trio/_subprocess_platform/__init__.py b/trio/_subprocess_platform/__init__.py index 37797424ab..4caa27b47c 100644 --- a/trio/_subprocess_platform/__init__.py +++ b/trio/_subprocess_platform/__init__.py @@ -4,6 +4,7 @@ import sys from typing import Optional, Tuple, TYPE_CHECKING +import trio from .. import _core, _subprocess from .._abc import SendStream, ReceiveStream @@ -72,15 +73,13 @@ def create_pipe_from_child_output() -> Tuple[ReceiveStream, int]: pass elif os.name == "posix": - from ..lowlevel import FdStream - def create_pipe_to_child_stdin(): # noqa: F811 rfd, wfd = os.pipe() - return FdStream(wfd), rfd + return trio.lowlevel.FdStream(wfd), rfd def create_pipe_from_child_output(): # noqa: F811 rfd, wfd = os.pipe() - return FdStream(rfd), wfd + return trio.lowlevel.FdStream(rfd), wfd elif os.name == "nt": from .._windows_pipes import PipeSendStream, PipeReceiveStream diff --git a/trio/lowlevel.py b/trio/lowlevel.py index 8e6dfc5ee4..f30fdcc4bd 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -46,6 +46,8 @@ start_guest_run, ) +from ._subprocess import open_process + if sys.platform == "win32": # Windows symbols from ._core import ( diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 5666ded8e8..28783303a9 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -14,11 +14,11 @@ sleep, sleep_forever, Process, - open_process, run_process, TrioDeprecationWarning, ClosedResourceError, ) +from ..lowlevel import open_process from .._core.tests.tutil import slow, skip_if_fbsd_pipes_broken from ..testing import wait_all_tasks_blocked From 35aa94b217f4c3606f2c5d2c1fcfca98e12e1181 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 18:52:36 -0700 Subject: [PATCH 0758/1498] blacken --- trio/_subprocess_platform/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trio/_subprocess_platform/__init__.py b/trio/_subprocess_platform/__init__.py index 4caa27b47c..418808a8ac 100644 --- a/trio/_subprocess_platform/__init__.py +++ b/trio/_subprocess_platform/__init__.py @@ -73,6 +73,7 @@ def create_pipe_from_child_output() -> Tuple[ReceiveStream, int]: pass elif os.name == "posix": + def create_pipe_to_child_stdin(): # noqa: F811 rfd, wfd = os.pipe() return trio.lowlevel.FdStream(wfd), rfd From 69238c43f34747b03edb0a76365a8c741734c8e7 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 18:55:53 -0700 Subject: [PATCH 0759/1498] Try to make mypy happy --- trio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/__init__.py b/trio/__init__.py index 6a43c42d2a..a50ec33310 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -108,7 +108,7 @@ __deprecated_attributes__ = { "open_process": _deprecate.DeprecatedAttribute( - value=_subprocess.open_process, + value=lowlevel.open_process, version="0.20.0", issue=1104, instead="trio.lowlevel.open_process", From ce42d72b8bb8f378cc8e738793f204983cb76eb2 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 13 Jul 2021 18:57:05 -0700 Subject: [PATCH 0760/1498] Fix sphinx crossrefs --- trio/_subprocess.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index ec623f19c7..25d95aea24 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -292,17 +292,16 @@ async def open_process( ) -> Process: r"""Execute a child program in a new process. - After construction, you can interact with the child process by writing - data to its `~Process.stdin` stream (a `~trio.abc.SendStream`), reading - data from its `~Process.stdout` and/or `~Process.stderr` streams (both - `~trio.abc.ReceiveStream`\s), sending it signals using - `~Process.terminate`, `~Process.kill`, or `~Process.send_signal`, and - waiting for it to exit using `~Process.wait`. See `Process` for details. - - Each standard stream is only available if you specify that a pipe should - be created for it. For example, if you pass ``stdin=subprocess.PIPE``, you - can write to the `~Process.stdin` stream, else `~Process.stdin` will be - ``None``. + After construction, you can interact with the child process by writing data to its + `~trio.Process.stdin` stream (a `~trio.abc.SendStream`), reading data from its + `~trio.Process.stdout` and/or `~trio.Process.stderr` streams (both + `~trio.abc.ReceiveStream`\s), sending it signals using `~trio.Process.terminate`, + `~trio.Process.kill`, or `~trio.Process.send_signal`, and waiting for it to exit + using `~trio.Process.wait`. See `trio.Process` for details. + + Each standard stream is only available if you specify that a pipe should be created + for it. For example, if you pass ``stdin=subprocess.PIPE``, you can write to the + `~trio.Process.stdin` stream, else `~trio.Process.stdin` will be ``None``. Unlike `trio.run_process`, this function doesn't do any kind of automatic management of the child process. It's up to you to implement whatever semantics you @@ -334,7 +333,7 @@ async def open_process( are also accepted. Returns: - A new `Process` object. + A new `trio.Process` object. Raises: OSError: if the process spawning fails, for example because the From d4f1d808c92958091f4ffb4f383a0f5ea7d153e2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Jul 2021 05:32:46 +0000 Subject: [PATCH 0761/1498] Bump requests from 2.25.1 to 2.26.0 Bumps [requests](https://github.com/psf/requests) from 2.25.1 to 2.26.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.25.1...v2.26.0) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index ad5f9348ef..3e226afccf 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.5.30 # via requests -chardet==4.0.0 +charset-normalizer==2.0.1 # via requests click-default-group==1.2.2 # via towncrier @@ -54,7 +54,7 @@ pyparsing==2.4.7 # via packaging pytz==2021.1 # via babel -requests==2.25.1 +requests==2.26.0 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in From 3f4e1a23e567825926b225f87c64db56d95f96fb Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sun, 18 Jul 2021 19:24:44 -0400 Subject: [PATCH 0762/1498] test and eliminate additional sources of cyclic garbage (#2063) from MultiErrorCatcher.__exit__, _multierror.copy_tb, MultiError.filter, CancelScope.__exit__, and NurseryManager.__aexit__ methods. This was nearly impossible to catch until #1864 landed so it wasn't cleaned up in #1805 (or during the live stream: https://www.youtube.com/watch?v=E_jvJVYXUAk). --- newsfragments/2063.bugfix.rst | 3 ++ trio/_core/_multierror.py | 10 +++- trio/_core/_run.py | 15 +++++- trio/_core/tests/test_multierror.py | 33 +++++++++++++ trio/_core/tests/test_run.py | 74 +++++++++++++++++++++++++---- 5 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 newsfragments/2063.bugfix.rst diff --git a/newsfragments/2063.bugfix.rst b/newsfragments/2063.bugfix.rst new file mode 100644 index 0000000000..85fe7aa52b --- /dev/null +++ b/newsfragments/2063.bugfix.rst @@ -0,0 +1,3 @@ +Trio now avoids creating cyclic garbage when a `MultiError` is generated and filtered, +including invisibly within the cancellation system. This means errors raised +through nurseries and cancel scopes should result in less GC latency. \ No newline at end of file diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index ed926c029f..b6bf2cf727 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -153,6 +153,9 @@ def __exit__(self, etype, exc, tb): _, value, _ = sys.exc_info() assert value is filtered_exc value.__context__ = old_context + # delete references from locals to avoid creating cycles + # see test_MultiError_catch_doesnt_create_cyclic_garbage + del _, filtered_exc, value class MultiError(BaseException): @@ -343,7 +346,12 @@ def copy_tb(base_tb, tb_next): c_new_tb.tb_lasti = base_tb.tb_lasti c_new_tb.tb_lineno = base_tb.tb_lineno - return new_tb + try: + return new_tb + finally: + # delete references from locals to avoid creating cycles + # see test_MultiError_catch_doesnt_create_cyclic_garbage + del new_tb, old_tb_frame def concat_tb(head, tail): diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 3174261fa9..a40a7a29ae 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -528,12 +528,15 @@ def _close(self, exc): self._cancel_status = None return exc - @enable_ki_protection def __exit__(self, etype, exc, tb): # NB: NurseryManager calls _close() directly rather than __exit__(), # so __exit__() must be just _close() plus this logic for adapting # the exception-filtering result to the context manager API. + # This inlines the enable_ki_protection decorator so we can fix + # f_locals *locally* below to avoid reference cycles + locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True + # Tracebacks show the 'raise' line below out of context, so let's give # this variable a name that makes sense out of context. remaining_error_after_cancel_scope = self._close(exc) @@ -551,6 +554,13 @@ def __exit__(self, etype, exc, tb): _, value, _ = sys.exc_info() assert value is remaining_error_after_cancel_scope value.__context__ = old_context + # delete references from locals to avoid creating cycles + # see test_cancel_scope_exit_doesnt_create_cyclic_garbage + del remaining_error_after_cancel_scope, value, _, exc + # deep magic to remove refs via f_locals + locals() + # TODO: check if PEP558 changes the need for this call + # https://github.com/python/cpython/pull/3640 def __repr__(self): if self._cancel_status is not None: @@ -817,6 +827,9 @@ async def __aexit__(self, etype, exc, tb): _, value, _ = sys.exc_info() assert value is combined_error_from_nursery value.__context__ = old_context + # delete references from locals to avoid creating cycles + # see test_simple_cancel_scope_usage_doesnt_create_cyclic_garbage + del _, combined_error_from_nursery, value, new_exc def __enter__(self): raise RuntimeError( diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 14eab22df7..70d763e652 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -1,3 +1,4 @@ +import gc import logging import pytest @@ -369,6 +370,38 @@ def catch_RuntimeError(exc): assert excinfo.value.__suppress_context__ == suppress_context +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" +) +def test_MultiError_catch_doesnt_create_cyclic_garbage(): + # https://github.com/python-trio/trio/pull/2063 + gc.collect() + old_flags = gc.get_debug() + + def make_multi(): + # make_tree creates cycles itself, so a simple + raise MultiError([get_exc(raiser1), get_exc(raiser2)]) + + def simple_filter(exc): + if isinstance(exc, ValueError): + return Exception() + if isinstance(exc, KeyError): + return RuntimeError() + assert False, "only ValueError and KeyError should exist" # pragma: no cover + + try: + gc.set_debug(gc.DEBUG_SAVEALL) + with pytest.raises(MultiError): + # covers MultiErrorCatcher.__exit__ and _multierror.copy_tb + with MultiError.catch(simple_filter): + raise make_multi() + gc.collect() + assert not gc.garbage + finally: + gc.set_debug(old_flags) + gc.garbage.clear() + + def assert_match_in_seq(pattern_list, string): offset = 0 print("looking for pattern matches...") diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index c56aaaa092..a559ac11ed 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2214,16 +2214,25 @@ async def do_a_cancel(): cscope.cancel() await sleep_forever() + async def crasher(): + raise ValueError + old_flags = gc.get_debug() try: gc.collect() gc.set_debug(gc.DEBUG_SAVEALL) + # cover outcome.Error.unwrap + # (See https://github.com/python-trio/outcome/pull/29) await do_a_cancel() + # cover outcome.Error.unwrap if unrolled_run hangs on to exception refs + # (See https://github.com/python-trio/trio/pull/1864) await do_a_cancel() - async with _core.open_nursery() as nursery: - nursery.start_soon(do_a_cancel) + with pytest.raises(ValueError): + async with _core.open_nursery() as nursery: + # cover MultiError.filter and NurseryManager.__aexit__ + nursery.start_soon(crasher) gc.collect() assert not gc.garbage @@ -2235,21 +2244,66 @@ async def do_a_cancel(): @pytest.mark.skipif( sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" ) -async def test_nursery_cancel_doesnt_create_cyclic_garbage(): - # https://github.com/python-trio/trio/issues/1770#issuecomment-730229423 +async def test_cancel_scope_exit_doesnt_create_cyclic_garbage(): + # https://github.com/python-trio/trio/pull/2063 gc.collect() + async def crasher(): + raise ValueError + old_flags = gc.get_debug() try: - for i in range(3): + + with pytest.raises(ValueError), _core.CancelScope() as outer: async with _core.open_nursery() as nursery: gc.collect() - gc.set_debug(gc.DEBUG_LEAK) - nursery.cancel_scope.cancel() + gc.set_debug(gc.DEBUG_SAVEALL) + # One child that gets cancelled by the outer scope + nursery.start_soon(sleep_forever) + outer.cancel() + # And one that raises a different error + nursery.start_soon(crasher) + # so that outer filters a Cancelled from the MultiError and + # covers CancelScope.__exit__ (and NurseryManager.__aexit__) + # (See https://github.com/python-trio/trio/pull/2063) + + gc.collect() + assert not gc.garbage + finally: + gc.set_debug(old_flags) + gc.garbage.clear() - gc.collect() - gc.set_debug(0) - assert not gc.garbage + +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" +) +async def test_nursery_cancel_doesnt_create_cyclic_garbage(): + # https://github.com/python-trio/trio/issues/1770#issuecomment-730229423 + def toggle_collected(): + nonlocal collected + collected = True + + collected = False + gc.collect() + old_flags = gc.get_debug() + try: + gc.set_debug(0) + gc.collect() + gc.set_debug(gc.DEBUG_SAVEALL) + + # cover Nursery._nested_child_finished + async with _core.open_nursery() as nursery: + nursery.cancel_scope.cancel() + + weakref.finalize(nursery, toggle_collected) + del nursery + # a checkpoint clears the nursery from the internals, apparently + # TODO: stop event loop from hanging on to the nursery at this point + await _core.checkpoint() + + assert collected + gc.collect() + assert not gc.garbage finally: gc.set_debug(old_flags) gc.garbage.clear() From 3c8eac6015cd2c17a3777dda50a4a1e5b3047e7a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Jul 2021 05:28:39 +0000 Subject: [PATCH 0763/1498] Bump black from 21.6b0 to 21.7b0 Bumps [black](https://github.com/psf/black) from 21.6b0 to 21.7b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 765e87e025..a86adb83f4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ attrs==20.3.0 # pytest backcall==0.2.0 # via ipython -black==21.6b0 ; implementation_name == "cpython" +black==21.7b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.5 # via cryptography @@ -118,11 +118,12 @@ sortedcontainers==2.4.0 # via -r test-requirements.in toml==0.10.2 # via - # black # mypy # pylint # pytest # pytest-cov +tomli==1.0.4 + # via black traitlets==5.0.5 # via # ipython From 8eb160094ca5edd74e4cc6e03feb73ccec7433d7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Jul 2021 05:29:24 +0000 Subject: [PATCH 0764/1498] Bump charset-normalizer from 2.0.1 to 2.0.3 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.1 to 2.0.3. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.1...2.0.3) Signed-off-by: dependabot-preview[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 3e226afccf..6284ff5919 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.5.30 # via requests -charset-normalizer==2.0.1 +charset-normalizer==2.0.3 # via requests click-default-group==1.2.2 # via towncrier From 5890a22d881f62ff90a90c46cde478b3d40ee56f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Jul 2021 05:30:08 +0000 Subject: [PATCH 0765/1498] Bump pathspec from 0.8.1 to 0.9.0 Bumps [pathspec](https://github.com/cpburnz/python-path-specification) from 0.8.1 to 0.9.0. - [Release notes](https://github.com/cpburnz/python-path-specification/releases) - [Changelog](https://github.com/cpburnz/python-path-specification/blob/master/CHANGES.rst) - [Commits](https://github.com/cpburnz/python-path-specification/compare/v0.8.1...v0.9.0) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 765e87e025..3ea2f9d036 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -74,7 +74,7 @@ packaging==21.0 # via pytest parso==0.8.2 # via jedi -pathspec==0.8.1 +pathspec==0.9.0 # via black pexpect==4.8.0 # via ipython From 46610b1dccab19af2115c5e7b4ccb405718a52d8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Jul 2021 05:38:36 +0000 Subject: [PATCH 0766/1498] Bump cffi from 1.14.5 to 1.14.6 Bumps [cffi](http://cffi.readthedocs.org) from 1.14.5 to 1.14.6. Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a86adb83f4..5fcae3c1d0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ backcall==0.2.0 # via ipython black==21.7b0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.5 +cffi==1.14.6 # via cryptography click==8.0.1 # via black From 4510523a7ab05751dfa7640e92d09566437ac69f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Jul 2021 05:26:10 +0000 Subject: [PATCH 0767/1498] Bump astroid from 2.6.2 to 2.6.4 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.6.2 to 2.6.4. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.6.2...v2.6.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 280ad3f4e6..16c83adebd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.6.2 +astroid==2.6.4 # via pylint async-generator==1.10 # via -r test-requirements.in From dd01e5854671258732ed680615d3e75a52e9aee8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 21 Jul 2021 05:25:35 +0000 Subject: [PATCH 0768/1498] Bump pylint from 2.9.3 to 2.9.4 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.9.3 to 2.9.4. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.9.3...v2.9.4) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 16c83adebd..c088666f1e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -96,7 +96,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.9.3 +pylint==2.9.4 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From f103342d07b09ff485581ed7e6aa33ab6ad1abd4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 22 Jul 2021 05:23:38 +0000 Subject: [PATCH 0769/1498] Bump astroid from 2.6.4 to 2.6.5 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.6.4 to 2.6.5. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.6.4...v2.6.5) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c088666f1e..27455a692a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.6.4 +astroid==2.6.5 # via pylint async-generator==1.10 # via -r test-requirements.in From 5d175e92b66e9a5a3a7664dfe4dcd01df40ebab1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 22 Jul 2021 05:33:47 +0000 Subject: [PATCH 0770/1498] Bump pylint from 2.9.4 to 2.9.5 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.9.4 to 2.9.5. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.9.4...v2.9.5) Signed-off-by: dependabot-preview[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 27455a692a..82096b95bb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -96,7 +96,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.9.4 +pylint==2.9.5 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 7a8911700f5b410455560b9ad325de67a95a13b4 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 22 Jul 2021 10:59:23 +0100 Subject: [PATCH 0771/1498] add copyright notice to LICENCE.MIT https://github.com/cdfoundation/foundation/blob/master/copyright.md#copyright-notices Fixes #1972 --- LICENSE.MIT | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LICENSE.MIT b/LICENSE.MIT index b8bb971859..c26b9f32ae 100644 --- a/LICENSE.MIT +++ b/LICENSE.MIT @@ -1,3 +1,5 @@ +Copyright Contributors to the Trio project. + The MIT License (MIT) Permission is hereby granted, free of charge, to any person obtaining From c71288838224a471b2abcec1da080ef47346bbb4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Jul 2021 15:17:36 +0000 Subject: [PATCH 0772/1498] Bump attrs from 20.3.0 to 21.2.0 Bumps [attrs](https://github.com/python-attrs/attrs) from 20.3.0 to 21.2.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/20.3.0...21.2.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 10 +++++----- test-requirements.txt | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 6284ff5919..0ce3f147d6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in -attrs==20.3.0 +attrs==21.2.0 # via # -r docs-requirements.in # outcome @@ -18,12 +18,12 @@ certifi==2021.5.30 # via requests charset-normalizer==2.0.3 # via requests -click-default-group==1.2.2 - # via towncrier click==8.0.1 # via # click-default-group # towncrier +click-default-group==1.2.2 + # via towncrier docutils==0.16 # via # sphinx @@ -62,13 +62,13 @@ snowballstemmer==2.1.0 # via sphinx sortedcontainers==2.4.0 # via -r docs-requirements.in -sphinx-rtd-theme==0.5.2 - # via -r docs-requirements.in sphinx==3.3.1 # via # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio +sphinx-rtd-theme==0.5.2 + # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 diff --git a/test-requirements.txt b/test-requirements.txt index 82096b95bb..49be245d91 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,7 +12,7 @@ astroid==2.6.5 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==20.3.0 +attrs==21.2.0 # via # -r test-requirements.in # outcome @@ -43,10 +43,10 @@ immutables==0.15 # via -r test-requirements.in iniconfig==1.1.1 # via pytest -ipython-genutils==0.2.0 - # via traitlets ipython==7.25.0 # via -r test-requirements.in +ipython-genutils==0.2.0 + # via traitlets isort==5.9.2 # via pylint jedi==0.18.0 @@ -61,13 +61,13 @@ mccabe==0.6.1 # via # flake8 # pylint +mypy==0.910 ; implementation_name == "cpython" + # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via # -r test-requirements.in # black # mypy -mypy==0.910 ; implementation_name == "cpython" - # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in packaging==21.0 @@ -102,12 +102,12 @@ pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.12.1 - # via -r test-requirements.in pytest==6.2.4 # via # -r test-requirements.in # pytest-cov +pytest-cov==2.12.1 + # via -r test-requirements.in regex==2021.7.6 # via black six==1.16.0 From e92d296ad4fb6ea9f6c8d19331c56812aabbadd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Jul 2021 15:18:40 +0000 Subject: [PATCH 0773/1498] Bump idna from 2.10 to 3.2 Bumps [idna](https://github.com/kjd/idna) from 2.10 to 3.2. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v2.10...v3.2) --- updated-dependencies: - dependency-name: idna dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 10 +++++----- test-requirements.txt | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 6284ff5919..dcb913860b 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,17 +18,17 @@ certifi==2021.5.30 # via requests charset-normalizer==2.0.3 # via requests -click-default-group==1.2.2 - # via towncrier click==8.0.1 # via # click-default-group # towncrier +click-default-group==1.2.2 + # via towncrier docutils==0.16 # via # sphinx # sphinx-rtd-theme -idna==2.10 +idna==3.2 # via # -r docs-requirements.in # requests @@ -62,13 +62,13 @@ snowballstemmer==2.1.0 # via sphinx sortedcontainers==2.4.0 # via -r docs-requirements.in -sphinx-rtd-theme==0.5.2 - # via -r docs-requirements.in sphinx==3.3.1 # via # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio +sphinx-rtd-theme==0.5.2 + # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 diff --git a/test-requirements.txt b/test-requirements.txt index 82096b95bb..47dd4a5767 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -35,7 +35,7 @@ decorator==5.0.9 # via ipython flake8==3.9.2 # via -r test-requirements.in -idna==2.10 +idna==3.2 # via # -r test-requirements.in # trustme @@ -43,10 +43,10 @@ immutables==0.15 # via -r test-requirements.in iniconfig==1.1.1 # via pytest -ipython-genutils==0.2.0 - # via traitlets ipython==7.25.0 # via -r test-requirements.in +ipython-genutils==0.2.0 + # via traitlets isort==5.9.2 # via pylint jedi==0.18.0 @@ -61,13 +61,13 @@ mccabe==0.6.1 # via # flake8 # pylint +mypy==0.910 ; implementation_name == "cpython" + # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via # -r test-requirements.in # black # mypy -mypy==0.910 ; implementation_name == "cpython" - # via -r test-requirements.in outcome==1.1.0 # via -r test-requirements.in packaging==21.0 @@ -102,12 +102,12 @@ pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest-cov==2.12.1 - # via -r test-requirements.in pytest==6.2.4 # via # -r test-requirements.in # pytest-cov +pytest-cov==2.12.1 + # via -r test-requirements.in regex==2021.7.6 # via black six==1.16.0 From 7c6b9c0f1693b974ddd71993807ec3af0e80f141 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Jul 2021 10:20:11 +0000 Subject: [PATCH 0774/1498] Bump tomli from 1.0.4 to 1.1.0 Bumps [tomli](https://github.com/hukkin/tomli) from 1.0.4 to 1.1.0. - [Release notes](https://github.com/hukkin/tomli/releases) - [Changelog](https://github.com/hukkin/tomli/blob/master/CHANGELOG.md) - [Commits](https://github.com/hukkin/tomli/compare/1.0.4...1.1.0) --- updated-dependencies: - dependency-name: tomli dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2b92c9100c..91cb6b7963 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -122,7 +122,7 @@ toml==0.10.2 # pylint # pytest # pytest-cov -tomli==1.0.4 +tomli==1.1.0 # via black traitlets==5.0.5 # via From 009fa8c415285ca4fbf92924db3ed26f30638652 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Jul 2021 10:21:17 +0000 Subject: [PATCH 0775/1498] Bump isort from 5.9.2 to 5.9.3 Bumps [isort](https://github.com/pycqa/isort) from 5.9.2 to 5.9.3. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.9.2...5.9.3) --- updated-dependencies: - dependency-name: isort dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 91cb6b7963..6d6433c7d3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -47,7 +47,7 @@ ipython==7.25.0 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets -isort==5.9.2 +isort==5.9.3 # via pylint jedi==0.18.0 # via From ac8f9c51278e4fc7b48b0d6aea5beda5b426dff9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Aug 2021 10:20:56 +0000 Subject: [PATCH 0776/1498] Bump ipython from 7.25.0 to 7.26.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.25.0 to 7.26.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.25.0...7.26.0) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 91cb6b7963..2a462ca9e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,7 +43,7 @@ immutables==0.15 # via -r test-requirements.in iniconfig==1.1.1 # via pytest -ipython==7.25.0 +ipython==7.26.0 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets From 672d1f5e004f6fc5db766c72c150379a86ba9215 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Aug 2021 10:21:33 +0000 Subject: [PATCH 0777/1498] Bump charset-normalizer from 2.0.3 to 2.0.4 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.3 to 2.0.4. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.3...2.0.4) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index a2106f73b8..5ace01f69f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.5.30 # via requests -charset-normalizer==2.0.3 +charset-normalizer==2.0.4 # via requests click==8.0.1 # via From 0a29b67ee5c620fde528c94c6917bda2f9567a70 Mon Sep 17 00:00:00 2001 From: "Anders E. Andersen" Date: Wed, 4 Aug 2021 21:01:13 +0200 Subject: [PATCH 0778/1498] Update awesome-trio-libraries.rst Add new section for larger libraries and frameworks with new entry for Slurry. --- docs/source/awesome-trio-libraries.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 895742a0c7..774f9dd2d3 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -90,6 +90,11 @@ Testing * `pytest-aio `_ - Pytest plugin with support for trio, curio, asyncio +Libraries and Frameworks +------------------------ +* `Slurry `__ - Slurry is a microframework for building reactive, data processing applications with Trio. + + Tools and Utilities ------------------- * `trio-typing `__ - Type hints for Trio and related projects. From 0238abe25ef3c102d8e3cad3b06ebc1acc199d55 Mon Sep 17 00:00:00 2001 From: "Anders E. Andersen" Date: Thu, 5 Aug 2021 21:30:27 +0200 Subject: [PATCH 0779/1498] Update awesome-trio-libraries.rst --- docs/source/awesome-trio-libraries.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 774f9dd2d3..2195ec1480 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -76,6 +76,11 @@ Multi-Core/Multiprocessing * `trio-parallel `__ - CPU parallelism for Trio +Stream Processing +----------------- +* `Slurry `__ - Slurry is a microframework for building reactive, data processing applications with Trio. + + RPC --- * `purepc `__ - Asynchronous pure Python gRPC client and server implementation using anyio. @@ -90,11 +95,6 @@ Testing * `pytest-aio `_ - Pytest plugin with support for trio, curio, asyncio -Libraries and Frameworks ------------------------- -* `Slurry `__ - Slurry is a microframework for building reactive, data processing applications with Trio. - - Tools and Utilities ------------------- * `trio-typing `__ - Type hints for Trio and related projects. From 2f43a658a01e7fc78de8d93a5ddcc266e2b0e0ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 10:21:47 +0000 Subject: [PATCH 0780/1498] Bump immutables from 0.15 to 0.16 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.15 to 0.16. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.15...v0.16) --- updated-dependencies: - dependency-name: immutables dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index a2106f73b8..1afc3391ba 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -34,7 +34,7 @@ idna==3.2 # requests imagesize==1.2.0 # via sphinx -immutables==0.15 +immutables==0.16 # via -r docs-requirements.in incremental==21.3.0 # via towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 91cb6b7963..f2c5210d1e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,7 +39,7 @@ idna==3.2 # via # -r test-requirements.in # trustme -immutables==0.15 +immutables==0.16 # via -r test-requirements.in iniconfig==1.1.1 # via pytest From 19c20a9f2cfedd0d4457742fe1bdfd3276693b2b Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 11 Aug 2021 10:31:11 +0400 Subject: [PATCH 0781/1498] Add new TCP_KEEPALIVE trio.socket symbol --- trio/socket.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/socket.py b/trio/socket.py index afbcefa1d3..27e75c8dbc 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -117,7 +117,8 @@ J1939_PGN_MAX, J1939_PGN_PDU1_MAX, J1939_PGN_REQUEST, SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, SCM_J1939_PRIO, SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, - SO_J1939_SEND_PRIO, UDPLITE_RECV_CSCOV, UDPLITE_SEND_CSCOV, IP_RECVTOS + SO_J1939_SEND_PRIO, UDPLITE_RECV_CSCOV, UDPLITE_SEND_CSCOV, IP_RECVTOS, + TCP_KEEPALIVE ) # fmt: on except ImportError: From 152c2377765fd9a4fc7873d60fa7c13e773523a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 07:17:13 +0000 Subject: [PATCH 0782/1498] Bump pylint from 2.9.5 to 2.9.6 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.9.5 to 2.9.6. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.9.5...v2.9.6) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b2082bf14c..dca77fa723 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -96,7 +96,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.7.4 # via ipython -pylint==2.9.5 +pylint==2.9.6 # via -r test-requirements.in pyopenssl==20.0.1 # via -r test-requirements.in From 022b2b1b1b879369296b8dbc885caed14c41d464 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 07:17:39 +0000 Subject: [PATCH 0783/1498] Bump tomli from 1.1.0 to 1.2.1 Bumps [tomli](https://github.com/hukkin/tomli) from 1.1.0 to 1.2.1. - [Release notes](https://github.com/hukkin/tomli/releases) - [Changelog](https://github.com/hukkin/tomli/blob/master/CHANGELOG.md) - [Commits](https://github.com/hukkin/tomli/compare/1.1.0...1.2.1) --- updated-dependencies: - dependency-name: tomli dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b2082bf14c..272be4caa2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -122,7 +122,7 @@ toml==0.10.2 # pylint # pytest # pytest-cov -tomli==1.1.0 +tomli==1.2.1 # via black traitlets==5.0.5 # via From 63b9ac0c96263a5d591829749d21f429a313d69a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 07:20:19 +0000 Subject: [PATCH 0784/1498] Bump regex from 2021.7.6 to 2021.8.3 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2021.7.6 to 2021.8.3. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) --- updated-dependencies: - dependency-name: regex dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b2082bf14c..f5eeb8ccb9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -108,7 +108,7 @@ pytest==6.2.4 # pytest-cov pytest-cov==2.12.1 # via -r test-requirements.in -regex==2021.7.6 +regex==2021.8.3 # via black six==1.16.0 # via pyopenssl From 78357d94c2a38b6d4f9b577162abba3aae1fb0a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 07:20:42 +0000 Subject: [PATCH 0785/1498] Bump astroid from 2.6.5 to 2.6.6 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.6.5 to 2.6.6. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.6.5...v2.6.6) --- updated-dependencies: - dependency-name: astroid dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b2082bf14c..2e390708aa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via black astor==0.8.1 # via -r test-requirements.in -astroid==2.6.5 +astroid==2.6.6 # via pylint async-generator==1.10 # via -r test-requirements.in From 852968f75a271970f121077c8cf0aaf80e21c8ea Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 13 Aug 2021 00:00:53 +0400 Subject: [PATCH 0786/1498] Bump pygments from 2.7.4 to 2.9.0 --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 9df74fe78b..604f47d5a0 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -48,7 +48,7 @@ outcome==1.1.0 # via -r docs-requirements.in packaging==21.0 # via sphinx -pygments==2.7.4 +pygments==2.9.0 # via sphinx pyparsing==2.4.7 # via packaging diff --git a/test-requirements.txt b/test-requirements.txt index c9934ce5f7..c0df0a1b35 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -94,7 +94,7 @@ pycparser==2.20 # via cffi pyflakes==2.3.1 # via flake8 -pygments==2.7.4 +pygments==2.9.0 # via ipython pylint==2.9.6 # via -r test-requirements.in From 927a561973ccbf8b00af23e90772d052b6322d27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:21:41 +0000 Subject: [PATCH 0787/1498] Bump trustme from 0.8.0 to 0.9.0 Bumps [trustme](https://github.com/python-trio/trustme) from 0.8.0 to 0.9.0. - [Release notes](https://github.com/python-trio/trustme/releases) - [Commits](https://github.com/python-trio/trustme/compare/v0.8.0...v0.9.0) --- updated-dependencies: - dependency-name: trustme dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c0df0a1b35..efe9b4b868 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -128,7 +128,7 @@ traitlets==5.0.5 # via # ipython # matplotlib-inline -trustme==0.8.0 +trustme==0.9.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" # via -r test-requirements.in From 1ad5550d56d3ac93af2ecea2feadead000ca64a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:22:44 +0000 Subject: [PATCH 0788/1498] Bump regex from 2021.8.3 to 2021.8.28 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2021.8.3 to 2021.8.28. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) --- updated-dependencies: - dependency-name: regex dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index efe9b4b868..701c701dfa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -108,7 +108,7 @@ pytest==6.2.4 # pytest-cov pytest-cov==2.12.1 # via -r test-requirements.in -regex==2021.8.3 +regex==2021.8.28 # via black six==1.16.0 # via pyopenssl From 5383ec70f5f4172baa042802feb4e907a103f6ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:24:06 +0000 Subject: [PATCH 0789/1498] Bump black from 21.7b0 to 21.8b0 Bumps [black](https://github.com/psf/black) from 21.7b0 to 21.8b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index efe9b4b868..e578b0443d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --output-file test-requirements.txt test-requirements.in # -appdirs==1.4.4 - # via black astor==0.8.1 # via -r test-requirements.in astroid==2.6.6 @@ -19,7 +17,7 @@ attrs==21.2.0 # pytest backcall==0.2.0 # via ipython -black==21.7b0 ; implementation_name == "cpython" +black==21.8b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.6 # via cryptography @@ -80,6 +78,8 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython +platformdirs==2.2.0 + # via black pluggy==0.13.1 # via pytest prompt-toolkit==3.0.19 @@ -135,6 +135,7 @@ typed-ast==1.4.3 ; implementation_name == "cpython" typing-extensions==3.10.0.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # black # mypy wcwidth==0.2.5 # via prompt-toolkit From 1f403522cabfaa4e18ce0887ebbc3f95055cbaa7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:26:06 +0000 Subject: [PATCH 0790/1498] Bump typing-extensions from 3.10.0.0 to 3.10.0.1 Bumps [typing-extensions](https://github.com/python/typing) from 3.10.0.0 to 3.10.0.1. - [Release notes](https://github.com/python/typing/releases) - [Commits](https://github.com/python/typing/compare/3.10.0.0...3.10.0.1) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index efe9b4b868..8ad0cc80cd 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -132,7 +132,7 @@ trustme==0.9.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" # via -r test-requirements.in -typing-extensions==3.10.0.0 ; implementation_name == "cpython" +typing-extensions==3.10.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in # mypy From 7ecb99bfcc3db73d7944657b52bdd1b8684db797 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:28:07 +0000 Subject: [PATCH 0791/1498] Bump ipython from 7.26.0 to 7.27.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.26.0 to 7.27.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.26.0...7.27.0) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index efe9b4b868..b819a500d2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,7 +43,7 @@ immutables==0.16 # via -r test-requirements.in iniconfig==1.1.1 # via pytest -ipython==7.26.0 +ipython==7.27.0 # via -r test-requirements.in ipython-genutils==0.2.0 # via traitlets From 5644f569fd47f4376058fad39cce783e1c7b460a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 31 Aug 2021 14:06:19 +0400 Subject: [PATCH 0792/1498] Fix Python 3.10 build (#2108) black 21.8b0 forbids typing-extensions 3.10.0.1 on Python 3.10. --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1b1d8bb913..fc98db8389 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -132,7 +132,7 @@ trustme==0.9.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" # via -r test-requirements.in -typing-extensions==3.10.0.1 ; implementation_name == "cpython" +typing-extensions==3.10.0.0 ; implementation_name == "cpython" # via # -r test-requirements.in # black From b706c7c59d75c53a2dfc55f0b9ef8f052d8172d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 10:24:06 +0000 Subject: [PATCH 0793/1498] Bump traitlets from 5.0.5 to 5.1.0 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.0.5 to 5.1.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.0.5...5.1.0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index fc98db8389..0db87869f6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -43,8 +43,6 @@ iniconfig==1.1.1 # via pytest ipython==7.27.0 # via -r test-requirements.in -ipython-genutils==0.2.0 - # via traitlets isort==5.9.3 # via pylint jedi==0.18.0 @@ -124,7 +122,7 @@ toml==0.10.2 # pytest-cov tomli==1.2.1 # via black -traitlets==5.0.5 +traitlets==5.1.0 # via # ipython # matplotlib-inline From 7320acacda47bcd877791216875e7185ab1abe57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 10:25:11 +0000 Subject: [PATCH 0794/1498] Bump platformdirs from 2.2.0 to 2.3.0 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.2.0...2.3.0) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fc98db8389..fa3faac9a9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -78,7 +78,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.2.0 +platformdirs==2.3.0 # via black pluggy==0.13.1 # via pytest From c7479dfe7609f521a531a78b6801dbb9af692e71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 10:26:06 +0000 Subject: [PATCH 0795/1498] Bump typing-extensions from 3.10.0.0 to 3.10.0.2 Bumps [typing-extensions](https://github.com/python/typing) from 3.10.0.0 to 3.10.0.2. - [Release notes](https://github.com/python/typing/releases) - [Commits](https://github.com/python/typing/compare/3.10.0.0...3.10.0.2) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fc98db8389..fbfaa478a3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -132,7 +132,7 @@ trustme==0.9.0 # via -r test-requirements.in typed-ast==1.4.3 ; implementation_name == "cpython" # via -r test-requirements.in -typing-extensions==3.10.0.0 ; implementation_name == "cpython" +typing-extensions==3.10.0.2 ; implementation_name == "cpython" # via # -r test-requirements.in # black From d015e90ba9d3cd39f3fc62506e527f1b6dd724d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 11:09:22 +0000 Subject: [PATCH 0796/1498] Bump pytest from 6.2.4 to 6.2.5 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.4 to 6.2.5. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.4...6.2.5) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2f78c96465..4921a1ced8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -100,7 +100,7 @@ pyopenssl==20.0.1 # via -r test-requirements.in pyparsing==2.4.7 # via packaging -pytest==6.2.4 +pytest==6.2.5 # via # -r test-requirements.in # pytest-cov From 9e028b3d955d86e1f043e845462a7ec7a5d39346 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 06:38:09 +0000 Subject: [PATCH 0797/1498] Bump cryptography from 3.4.7 to 3.4.8 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.7 to 3.4.8. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.7...3.4.8) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4921a1ced8..78884d2eba 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.0.1 # via black coverage==5.5 # via pytest-cov -cryptography==3.4.7 +cryptography==3.4.8 # via # pyopenssl # trustme From bb4f4234e7fd6cff50d139acccacce2ce7e0054c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 06:38:41 +0000 Subject: [PATCH 0798/1498] Bump prompt-toolkit from 3.0.19 to 3.0.20 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.19 to 3.0.20. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.19...3.0.20) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4921a1ced8..a988f22743 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -80,7 +80,7 @@ platformdirs==2.3.0 # via black pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.19 +prompt-toolkit==3.0.20 # via ipython ptyprocess==0.7.0 # via pexpect From d02369efde776cc6c06a9a2d348c5776e63569ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 10:17:50 +0000 Subject: [PATCH 0799/1498] Bump pluggy from 0.13.1 to 1.0.0 Bumps [pluggy](https://github.com/pytest-dev/pluggy) from 0.13.1 to 1.0.0. - [Release notes](https://github.com/pytest-dev/pluggy/releases) - [Changelog](https://github.com/pytest-dev/pluggy/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pluggy/compare/0.13.1...1.0.0) --- updated-dependencies: - dependency-name: pluggy dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4921a1ced8..52ae2c3647 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -78,7 +78,7 @@ pickleshare==0.7.5 # via ipython platformdirs==2.3.0 # via black -pluggy==0.13.1 +pluggy==1.0.0 # via pytest prompt-toolkit==3.0.19 # via ipython From 40d17e953be67ef70bbd615c680548640fd52ce0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Sep 2021 10:15:34 +0000 Subject: [PATCH 0800/1498] Bump matplotlib-inline from 0.1.2 to 0.1.3 Bumps [matplotlib-inline](https://github.com/ipython/matplotlib-inline) from 0.1.2 to 0.1.3. - [Release notes](https://github.com/ipython/matplotlib-inline/releases) - [Commits](https://github.com/ipython/matplotlib-inline/compare/0.1.2...0.1.3) --- updated-dependencies: - dependency-name: matplotlib-inline dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index af5ac46cf7..e1832ea728 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -51,7 +51,7 @@ jedi==0.18.0 # ipython lazy-object-proxy==1.6.0 # via astroid -matplotlib-inline==0.1.2 +matplotlib-inline==0.1.3 # via ipython mccabe==0.6.1 # via From c670d41c640c5c279f90887a255130976d73f171 Mon Sep 17 00:00:00 2001 From: Kirill Klenov Date: Fri, 10 Sep 2021 14:56:30 +0300 Subject: [PATCH 0801/1498] Update awesome-trio-libraries.rst --- docs/source/awesome-trio-libraries.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 2195ec1480..2de49327df 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -48,6 +48,8 @@ Database * `redio `__ - Redis client, pure Python and Trio. * `trio_redis `__ - A Redis client for Trio. Depends on hiredis-py. * `asyncakumuli `__ - Client for the `Akumuli `__ time series database. +* `aio-databases Date: Fri, 10 Sep 2021 14:56:54 +0300 Subject: [PATCH 0802/1498] Update awesome-trio-libraries.rst --- docs/source/awesome-trio-libraries.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 2de49327df..4b50772c1d 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -48,8 +48,8 @@ Database * `redio `__ - Redis client, pure Python and Trio. * `trio_redis `__ - A Redis client for Trio. Depends on hiredis-py. * `asyncakumuli `__ - Client for the `Akumuli `__ time series database. -* `aio-databases `_ - Async Support for various databases (triopg, trio-mysql) +* `peewee-aio `_ - Peewee Async ORM with trio support (triopg, trio-mysql). IOT From 81ced3cea14e8fdd8906287b223ed66b845bc637 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 10:23:00 +0000 Subject: [PATCH 0803/1498] Bump decorator from 5.0.9 to 5.1.0 Bumps [decorator](https://github.com/micheles/decorator) from 5.0.9 to 5.1.0. - [Release notes](https://github.com/micheles/decorator/releases) - [Changelog](https://github.com/micheles/decorator/blob/master/CHANGES.md) - [Commits](https://github.com/micheles/decorator/commits) --- updated-dependencies: - dependency-name: decorator dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index af5ac46cf7..9f8296d4e7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -29,7 +29,7 @@ cryptography==3.4.8 # via # pyopenssl # trustme -decorator==5.0.9 +decorator==5.1.0 # via ipython flake8==3.9.2 # via -r test-requirements.in From 76034fab5be24176730a93eb2e46302b43418761 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 13 Sep 2021 19:43:38 -0700 Subject: [PATCH 0804/1498] Update docs in response to reviews --- docs/source/reference-io.rst | 7 +++--- trio/_subprocess.py | 47 +++++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index 3a082982e0..26a25f0500 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -697,9 +697,10 @@ task and interact with it while it's running: and we wanted to provide an interface with fewer surprises. If `trio.run_process` is too limiting, we also offer a low-level API, -`trio.lowlevel.open_process`. For example, use -`~trio.lowlevel.open_process` if you want to spawn a child process -that outlives the parent process: +`trio.lowlevel.open_process`. For example, if you want to spawn a +child process that will and outlive the parent process and be +orphaned, then `~trio.run_process` can't do that, but +`~trio.lowlevel.open_process` can. .. _subprocess-options: diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 25d95aea24..f836e45e95 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -180,7 +180,7 @@ def returncode(self): return result @deprecated( - "0.16.0", + "0.20.0", thing="using trio.Process as an async context manager", issue=1104, instead="run_process or nursery.start(run_process, ...)", @@ -189,7 +189,7 @@ async def __aenter__(self): return self @deprecated( - "0.16.0", issue=1104, instead="run_process or nursery.start(run_process, ...)" + "0.20.0", issue=1104, instead="run_process or nursery.start(run_process, ...)" ) async def aclose(self): """Close any pipes we have to the process (both input and output) @@ -477,15 +477,16 @@ async def run_process( **Output:** By default, any output produced by the subprocess is passed through to the standard output and error streams of the - parent Trio process. If you would like to capture this output and - do something with it, you can pass ``capture_stdout=True`` to - capture the subprocess's standard output, and/or - ``capture_stderr=True`` to capture its standard error. Captured - data is provided as the + parent Trio process. + + When calling `run_process` directly, you can capture the subprocess's output by + passing ``capture_stdout=True`` to capture the subprocess's standard output, and/or + ``capture_stderr=True`` to capture its standard error. Captured data is collected up + by Trio into an in-memory buffer, and then provided as the :attr:`~subprocess.CompletedProcess.stdout` and/or - :attr:`~subprocess.CompletedProcess.stderr` attributes of the - returned :class:`~subprocess.CompletedProcess` object. The value - for any stream that was not captured will be ``None``. + :attr:`~subprocess.CompletedProcess.stderr` attributes of the returned + :class:`~subprocess.CompletedProcess` object. The value for any stream that was not + captured will be ``None``. If you want to capture both stdout and stderr while keeping them separate, pass ``capture_stdout=True, capture_stderr=True``. @@ -496,6 +497,13 @@ async def run_process( output will be available in the `~subprocess.CompletedProcess.stdout` attribute. + If you're using ``await nursery.start(trio.run_process, ...)`` and want to capture + the subprocess's output for further processing, then use ``stdout=subprocess.PIPE`` + and then make sure to read the data out of the `Process.stdout` stream. If you want + to capture stderr separately, use ``stderr=subprocess.PIPE``. If you want to capture + both, but mixed together in the correct order, use ``stdout=subproces.PIPE, + stderr=subprocess.STDOUT``. + **Error checking:** If the subprocess exits with a nonzero status code, indicating failure, :func:`run_process` raises a :exc:`subprocess.CalledProcessError` exception rather than @@ -541,8 +549,9 @@ async def run_process( same place as the parent Trio process's standard input. As is the case with the :mod:`subprocess` module, you can also pass a file descriptor or an object with a ``fileno()`` method, in which case - the subprocess's standard input will come from that file. And when - starting `run_process` as a background task, you can use + the subprocess's standard input will come from that file. + + When starting `run_process` as a background task, you can also use ``stdin=subprocess.PIPE``, in which case `Process.stdin` will be a `~trio.abc.SendStream` that you can use to send data to the child. @@ -629,19 +638,19 @@ async def my_deliver_cancel(process): if task_status is trio.TASK_STATUS_IGNORED: if stdin is subprocess.PIPE: raise ValueError( - "stdin=subprocess.PIPE doesn't make sense without " - "nursery.start, since there's no way to access the " - "pipe; pass the data you want to send or use nursery.start" + "stdout=subprocess.PIPE is only valid with nursery.start, " + "since that's the only way to access the pipe; use nursery.start " + "or pass the data you want to write directly" ) if options.get("stdout") is subprocess.PIPE: raise ValueError( - "stdout=subprocess.PIPE doesn't make sense without " - "nursery.start, since there's no way to access the pipe" + "stdout=subprocess.PIPE is only valid with nursery.start, " + "since that's the only way to access the pipe" ) if options.get("stderr") is subprocess.PIPE: raise ValueError( - "stderr=subprocess.PIPE doesn't make sense without " - "nursery.start, since there's no way to access the pipe" + "stderr=subprocess.PIPE is only valid with nursery.start, " + "since that's the only way to access the pipe" ) if isinstance(stdin, (bytes, bytearray, memoryview)): input = stdin From 4cd986fe4d6f1dd2abd5775860fa8e93e6356b12 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Sep 2021 10:18:55 +0000 Subject: [PATCH 0805/1498] Bump sphinx-rtd-theme from 0.5.2 to 1.0.0 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 0.5.2 to 1.0.0. - [Release notes](https://github.com/readthedocs/sphinx_rtd_theme/releases) - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/0.5.2...1.0.0) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 604f47d5a0..972d49778c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -67,7 +67,7 @@ sphinx==3.3.1 # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio -sphinx-rtd-theme==0.5.2 +sphinx_rtd_theme==1.0.0 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx From be938a7e3524cba58616abb21ec31764b4888f86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Sep 2021 10:20:35 +0000 Subject: [PATCH 0806/1498] Bump black from 21.8b0 to 21.9b0 Bumps [black](https://github.com/psf/black) from 21.8b0 to 21.9b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index af5ac46cf7..0fef8c4418 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==21.2.0 # pytest backcall==0.2.0 # via ipython -black==21.8b0 ; implementation_name == "cpython" +black==21.9b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.14.6 # via cryptography From d6cabd09c8062ecc65f94e2959bfcf7c25ac0f0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Sep 2021 10:19:36 +0000 Subject: [PATCH 0807/1498] Bump docutils from 0.16 to 0.17.1 Bumps [docutils](http://docutils.sourceforge.net/) from 0.16 to 0.17.1. --- updated-dependencies: - dependency-name: docutils dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 972d49778c..36f852741e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -24,7 +24,7 @@ click==8.0.1 # towncrier click-default-group==1.2.2 # via towncrier -docutils==0.16 +docutils==0.17.1 # via # sphinx # sphinx-rtd-theme @@ -67,7 +67,7 @@ sphinx==3.3.1 # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio -sphinx_rtd_theme==1.0.0 +sphinx-rtd-theme==1.0.0 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx From 823101d268c9c2a890e93671d255a75d281f69c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Sep 2021 10:21:14 +0000 Subject: [PATCH 0808/1498] Bump charset-normalizer from 2.0.4 to 2.0.5 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.4 to 2.0.5. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.4...2.0.5) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 972d49778c..bc10e4fede 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.5.30 # via requests -charset-normalizer==2.0.4 +charset-normalizer==2.0.5 # via requests click==8.0.1 # via @@ -67,7 +67,7 @@ sphinx==3.3.1 # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio -sphinx_rtd_theme==1.0.0 +sphinx-rtd-theme==1.0.0 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx From 1c4c99c3f01049f242f11f98722a3675122bd3c5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 19 Sep 2021 00:26:06 -0700 Subject: [PATCH 0809/1498] Remove stray 'and' --- docs/source/reference-io.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index 26a25f0500..81a615e039 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -698,7 +698,7 @@ task and interact with it while it's running: If `trio.run_process` is too limiting, we also offer a low-level API, `trio.lowlevel.open_process`. For example, if you want to spawn a -child process that will and outlive the parent process and be +child process that will outlive the parent process and be orphaned, then `~trio.run_process` can't do that, but `~trio.lowlevel.open_process` can. From 0fdb9be8ea3ea1ed01ab6bb1fbf12301bb394070 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sun, 10 Oct 2021 14:45:34 -0400 Subject: [PATCH 0810/1498] fix stale function name in example --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index a40a7a29ae..fcb36c32c5 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -2358,7 +2358,7 @@ async def checkpoint_if_cancelled(): Equivalent to (but potentially more efficient than):: - if trio.current_deadline() == -inf: + if trio.current_effective_deadline() == -inf: await trio.lowlevel.checkpoint() This is either a no-op, or else it allow other tasks to be scheduled and From e8f8bf10c5b4515827c0afba680426794d259895 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Thu, 21 Oct 2021 09:11:06 +0900 Subject: [PATCH 0811/1498] Add PyPy 3.8 (and nightly) to the build matrix Additionally, start testing PyPy on Windows, as that has broke before. Adding to that, also drop PyPy 3.7 nightly tests, as PyPy 3.8 nightly covers that case for the most part. --- .github/workflows/ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1e0cd46c5..26b62707b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,14 @@ jobs: # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' # lsp_extract_file: '' # extra_name: ', with non-IFS LSP' + + # trio on pypy on windows has failed before: + # https://github.com/python-trio/trio/issues/1361 + - python: '3.8' + arch: 'x64' + pypy_nightly_branch: 'py3.8' + extra_name: ', pypy 3.8 nightly' + steps: - name: Checkout uses: actions/checkout@v2 @@ -67,7 +75,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.6', 'pypy-3.7', '3.6', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] + python: ['pypy-3.6', 'pypy-3.7', 'pypy-3.8', '3.6', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] @@ -75,9 +83,9 @@ jobs: - python: '3.8' check_formatting: '1' extra_name: ', check formatting' - - python: '3.7' # <- not actually used - pypy_nightly_branch: 'py3.7' - extra_name: ', pypy 3.7 nightly' + - python: '3.8' # <- not actually used + pypy_nightly_branch: 'py3.8' + extra_name: ', pypy 3.8 nightly' steps: - name: Checkout uses: actions/checkout@v2 From 4d60c27210637adee00d68b369be55d9d05b1401 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 23 Oct 2021 09:46:22 +0900 Subject: [PATCH 0812/1498] Skip failing IPython tests (we know the cause and it's not us) --- trio/_core/tests/test_multierror.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 70d763e652..4afbee9864 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -715,6 +715,10 @@ def test_custom_excepthook(): @slow @need_ipython +@pytest.mark.skipif( + sys.version_info >= (3, 9), + reason="ipython --quick currently breaks on latest dev python" +) def test_ipython_exc_handler(): completed = run_script("simple_excepthook.py", use_ipython=True) check_simple_excepthook(completed) @@ -738,6 +742,10 @@ def test_partial_imported_but_unused(): @slow @need_ipython +@pytest.mark.skipif( + sys.version_info >= (3, 9), + reason="ipython --quick currently breaks on latest dev python" +) def test_ipython_custom_exc_handler(): # Check we get a nice warning (but only one!) if the user is using IPython # and already has some other set_custom_exc handler installed. From 0936509a68607f72fa45fe7e441cfd62269761f4 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 23 Oct 2021 11:08:38 +0900 Subject: [PATCH 0813/1498] Placating black --- trio/_core/tests/test_multierror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 4afbee9864..3a0debaee9 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -717,7 +717,7 @@ def test_custom_excepthook(): @need_ipython @pytest.mark.skipif( sys.version_info >= (3, 9), - reason="ipython --quick currently breaks on latest dev python" + reason="ipython --quick currently breaks on latest dev python", ) def test_ipython_exc_handler(): completed = run_script("simple_excepthook.py", use_ipython=True) @@ -744,7 +744,7 @@ def test_partial_imported_but_unused(): @need_ipython @pytest.mark.skipif( sys.version_info >= (3, 9), - reason="ipython --quick currently breaks on latest dev python" + reason="ipython --quick currently breaks on latest dev python", ) def test_ipython_custom_exc_handler(): # Check we get a nice warning (but only one!) if the user is using IPython From c5266eb42b3d96256a618257124f3db42fc4d9c1 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Tue, 26 Oct 2021 10:15:02 +0900 Subject: [PATCH 0814/1498] Add tests for macOS Prior I added something for Windows, which has broken previously: https://github.com/python-trio/trio/issues/1361 --- .github/workflows/ci.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26b62707b9..9b3af0d643 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,9 +35,6 @@ jobs: # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' # lsp_extract_file: '' # extra_name: ', with non-IFS LSP' - - # trio on pypy on windows has failed before: - # https://github.com/python-trio/trio/issues/1361 - python: '3.8' arch: 'x64' pypy_nightly_branch: 'py3.8' @@ -83,7 +80,10 @@ jobs: - python: '3.8' check_formatting: '1' extra_name: ', check formatting' - - python: '3.8' # <- not actually used + - python: '3.7' # <- not actually used + pypy_nightly_branch: 'py3.7' + extra_name: ', pypy 3.7 nightly' + - python: '3.8' pypy_nightly_branch: 'py3.8' extra_name: ', pypy 3.8 nightly' steps: @@ -115,6 +115,11 @@ jobs: fail-fast: false matrix: python: ['3.6', '3.7', '3.8', '3.9', '3.10'] + include: + - python: '3.8' + arch: 'x64' + pypy_nightly_branch: 'py3.8' + extra_name: ', pypy 3.8 nightly' steps: - name: Checkout uses: actions/checkout@v2 From 378c8779e92aa73584e25b04e9b9eac24b5a6a0a Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Tue, 26 Oct 2021 10:26:03 +0900 Subject: [PATCH 0815/1498] Fix the yaml structure --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b3af0d643..6f3235c1a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: fail-fast: false matrix: python: ['3.6', '3.7', '3.8', '3.9', '3.10'] - include: + include: - python: '3.8' arch: 'x64' pypy_nightly_branch: 'py3.8' From a1e2ab9a5ffe99bf22d553775fbea2c21c04fb13 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 26 Oct 2021 09:46:42 +0400 Subject: [PATCH 0816/1498] Clarify PyPy CI setup --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f3235c1a7..606e64977b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: # lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe' # lsp_extract_file: '' # extra_name: ', with non-IFS LSP' - - python: '3.8' + - python: '3.8' # <- not actually used arch: 'x64' pypy_nightly_branch: 'py3.8' extra_name: ', pypy 3.8 nightly' @@ -83,7 +83,7 @@ jobs: - python: '3.7' # <- not actually used pypy_nightly_branch: 'py3.7' extra_name: ', pypy 3.7 nightly' - - python: '3.8' + - python: '3.8' # <- not actually used pypy_nightly_branch: 'py3.8' extra_name: ', pypy 3.8 nightly' steps: @@ -116,7 +116,7 @@ jobs: matrix: python: ['3.6', '3.7', '3.8', '3.9', '3.10'] include: - - python: '3.8' + - python: '3.8' # <- not actually used arch: 'x64' pypy_nightly_branch: 'py3.8' extra_name: ', pypy 3.8 nightly' From 92d6a45ea4c5a646f58dc8410586b542959d3ab5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 05:53:20 +0000 Subject: [PATCH 0817/1498] Bump ipython from 7.27.0 to 7.28.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.27.0 to 7.28.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.27.0...7.28.0) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4be0ba190a..2524266fb5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ immutables==0.16 # via -r test-requirements.in iniconfig==1.1.1 # via pytest -ipython==7.27.0 +ipython==7.28.0 # via -r test-requirements.in isort==5.9.3 # via pylint From dd6c43a6e66df806e351692fbba5f842d258e068 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 05:53:35 +0000 Subject: [PATCH 0818/1498] Bump urllib3 from 1.26.6 to 1.26.7 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.6 to 1.26.7. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.7/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.6...1.26.7) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index eeadac9b91..a649064a5d 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -87,5 +87,5 @@ toml==0.10.2 # via towncrier towncrier==21.3.0 # via -r docs-requirements.in -urllib3==1.26.6 +urllib3==1.26.7 # via requests From 92493f22a01cebd704d8a189d8bff902c79ec615 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 05:54:04 +0000 Subject: [PATCH 0819/1498] Bump pyopenssl from 20.0.1 to 21.0.0 Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 20.0.1 to 21.0.0. - [Release notes](https://github.com/pyca/pyopenssl/releases) - [Changelog](https://github.com/pyca/pyopenssl/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/pyopenssl/compare/20.0.1...21.0.0) --- updated-dependencies: - dependency-name: pyopenssl dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4be0ba190a..dd849bf89e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -96,7 +96,7 @@ pygments==2.9.0 # via ipython pylint==2.9.6 # via -r test-requirements.in -pyopenssl==20.0.1 +pyopenssl==21.0.0 # via -r test-requirements.in pyparsing==2.4.7 # via packaging From a0bbfda42c045595dcf8eb905180a3cd8dd33830 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 05:54:16 +0000 Subject: [PATCH 0820/1498] Bump cryptography from 3.4.8 to 35.0.0 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.8 to 35.0.0. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.8...35.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4be0ba190a..a9c6ff300f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.0.1 # via black coverage==5.5 # via pytest-cov -cryptography==3.4.8 +cryptography==35.0.0 # via # pyopenssl # trustme From 891efbf6175af405b93546b7e3d740567ecf0ce4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 07:34:47 +0000 Subject: [PATCH 0821/1498] Bump pygments from 2.9.0 to 2.10.0 Bumps [pygments](https://github.com/pygments/pygments) from 2.9.0 to 2.10.0. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.9.0...2.10.0) --- updated-dependencies: - dependency-name: pygments dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index a649064a5d..879ff095bf 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -48,7 +48,7 @@ outcome==1.1.0 # via -r docs-requirements.in packaging==21.0 # via sphinx -pygments==2.9.0 +pygments==2.10.0 # via sphinx pyparsing==2.4.7 # via packaging diff --git a/test-requirements.txt b/test-requirements.txt index dd663cbd59..4386a30f68 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -92,7 +92,7 @@ pycparser==2.20 # via cffi pyflakes==2.3.1 # via flake8 -pygments==2.9.0 +pygments==2.10.0 # via ipython pylint==2.9.6 # via -r test-requirements.in From ed95cd9a4cb65a1764c9549056d13a1ae2282039 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 07:54:10 +0000 Subject: [PATCH 0822/1498] Bump idna from 3.2 to 3.3 Bumps [idna](https://github.com/kjd/idna) from 3.2 to 3.3. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.2...v3.3) --- updated-dependencies: - dependency-name: idna dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- test-requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index eeadac9b91..9a91d29127 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -28,7 +28,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -idna==3.2 +idna==3.3 # via # -r docs-requirements.in # requests @@ -87,5 +87,5 @@ toml==0.10.2 # via towncrier towncrier==21.3.0 # via -r docs-requirements.in -urllib3==1.26.6 +urllib3==1.26.7 # via requests diff --git a/test-requirements.txt b/test-requirements.txt index dd849bf89e..fec48b9dff 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.0.1 # via black coverage==5.5 # via pytest-cov -cryptography==3.4.8 +cryptography==35.0.0 # via # pyopenssl # trustme @@ -33,7 +33,7 @@ decorator==5.1.0 # via ipython flake8==3.9.2 # via -r test-requirements.in -idna==3.2 +idna==3.3 # via # -r test-requirements.in # trustme @@ -41,7 +41,7 @@ immutables==0.16 # via -r test-requirements.in iniconfig==1.1.1 # via pytest -ipython==7.27.0 +ipython==7.28.0 # via -r test-requirements.in isort==5.9.3 # via pylint From b9bb203ecfdb7b78ce9571e163c16c9a245a028d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 09:46:45 +0000 Subject: [PATCH 0823/1498] Bump pytz from 2021.1 to 2021.3 Bumps [pytz](https://github.com/stub42/pytz) from 2021.1 to 2021.3. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2021.1...release_2021.3) --- updated-dependencies: - dependency-name: pytz dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index a649064a5d..a9e9aef06d 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -48,11 +48,11 @@ outcome==1.1.0 # via -r docs-requirements.in packaging==21.0 # via sphinx -pygments==2.9.0 +pygments==2.10.0 # via sphinx pyparsing==2.4.7 # via packaging -pytz==2021.1 +pytz==2021.3 # via babel requests==2.26.0 # via sphinx From 91d0b3864403f7dcf229d3c5bb0af4c501a5740b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 09:46:54 +0000 Subject: [PATCH 0824/1498] Bump flake8 from 3.9.2 to 4.0.1 Bumps [flake8](https://github.com/pycqa/flake8) from 3.9.2 to 4.0.1. - [Release notes](https://github.com/pycqa/flake8/releases) - [Commits](https://github.com/pycqa/flake8/compare/3.9.2...4.0.1) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..d2b1de769c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -31,7 +31,7 @@ cryptography==35.0.0 # trustme decorator==5.1.0 # via ipython -flake8==3.9.2 +flake8==4.0.1 # via -r test-requirements.in idna==3.3 # via @@ -86,11 +86,11 @@ ptyprocess==0.7.0 # via pexpect py==1.10.0 # via pytest -pycodestyle==2.7.0 +pycodestyle==2.8.0 # via flake8 pycparser==2.20 # via cffi -pyflakes==2.3.1 +pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython From cb23f9d51eed63164fa7df4607724bde0c3abc38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 09:47:14 +0000 Subject: [PATCH 0825/1498] Bump cffi from 1.14.6 to 1.15.0 Bumps [cffi](http://cffi.readthedocs.org) from 1.14.6 to 1.15.0. --- updated-dependencies: - dependency-name: cffi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..ead068efa9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ backcall==0.2.0 # via ipython black==21.9b0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.6 +cffi==1.15.0 # via cryptography click==8.0.1 # via black From c15d83da36b3b5228463a33aa123b261d0301b36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 09:50:21 +0000 Subject: [PATCH 0826/1498] Bump pylint from 2.9.6 to 2.11.1 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.9.6 to 2.11.1. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.9.6...v2.11.1) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..9d1f6e86fc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.6.6 +astroid==2.8.4 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -77,7 +77,9 @@ pexpect==4.8.0 pickleshare==0.7.5 # via ipython platformdirs==2.3.0 - # via black + # via + # black + # pylint pluggy==1.0.0 # via pytest prompt-toolkit==3.0.20 @@ -94,7 +96,7 @@ pyflakes==2.3.1 # via flake8 pygments==2.10.0 # via ipython -pylint==2.9.6 +pylint==2.11.1 # via -r test-requirements.in pyopenssl==21.0.0 # via -r test-requirements.in @@ -133,8 +135,10 @@ typed-ast==1.4.3 ; implementation_name == "cpython" typing-extensions==3.10.0.2 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid # black # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.12.1 From 7c9210f76ea85df7a4ba74d299e1dd8d3ca8a752 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 10:28:36 +0000 Subject: [PATCH 0827/1498] Bump tomli from 1.2.1 to 1.2.2 Bumps [tomli](https://github.com/hukkin/tomli) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/hukkin/tomli/releases) - [Changelog](https://github.com/hukkin/tomli/blob/master/CHANGELOG.md) - [Commits](https://github.com/hukkin/tomli/compare/1.2.1...1.2.2) --- updated-dependencies: - dependency-name: tomli dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..f8b5b84f3c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -120,7 +120,7 @@ toml==0.10.2 # pylint # pytest # pytest-cov -tomli==1.2.1 +tomli==1.2.2 # via black traitlets==5.1.0 # via From 8eed3c532384be741d415b5ec450b49a0c2a0bcb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 10:29:31 +0000 Subject: [PATCH 0828/1498] Bump pyparsing from 2.4.7 to 3.0.1 Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 2.4.7 to 3.0.1. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_2.4.7...pyparsing_3.0.1) --- updated-dependencies: - dependency-name: pyparsing dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 621e7476ae..b363481a08 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -50,7 +50,7 @@ packaging==21.0 # via sphinx pygments==2.10.0 # via sphinx -pyparsing==2.4.7 +pyparsing==3.0.1 # via packaging pytz==2021.1 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..5957cfd657 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -98,7 +98,7 @@ pylint==2.9.6 # via -r test-requirements.in pyopenssl==21.0.0 # via -r test-requirements.in -pyparsing==2.4.7 +pyparsing==3.0.1 # via packaging pytest==6.2.5 # via From 5e898b4b9ecce830709ee9232db1ab11244f6fee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 10:33:06 +0000 Subject: [PATCH 0829/1498] Bump charset-normalizer from 2.0.5 to 2.0.7 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.5 to 2.0.7. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.5...2.0.7) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 62cfb41147..d9cd615de1 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.5.30 # via requests -charset-normalizer==2.0.5 +charset-normalizer==2.0.7 # via requests click==8.0.1 # via From 4239db285f166777c07034ada885f4c5ce5e804a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:19:20 +0000 Subject: [PATCH 0830/1498] Bump traitlets from 5.1.0 to 5.1.1 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.1.0...5.1.1) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..f91a83aa95 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ backcall==0.2.0 # via ipython black==21.9b0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.6 +cffi==1.15.0 # via cryptography click==8.0.1 # via black @@ -122,7 +122,7 @@ toml==0.10.2 # pytest-cov tomli==1.2.1 # via black -traitlets==5.1.0 +traitlets==5.1.1 # via # ipython # matplotlib-inline From 204cb80935f286b7d2a5e9216cfbbb6c0c124735 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:22:28 +0000 Subject: [PATCH 0831/1498] Bump certifi from 2021.5.30 to 2021.10.8 Bumps [certifi](https://github.com/certifi/python-certifi) from 2021.5.30 to 2021.10.8. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2021.05.30...2021.10.08) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 8a7272de80..3bf346b995 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==21.2.0 # outcome babel==2.9.1 # via sphinx -certifi==2021.5.30 +certifi==2021.10.8 # via requests charset-normalizer==2.0.7 # via requests From 36836beecfec0ef592363123b1e4a27f57f2ca23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:22:33 +0000 Subject: [PATCH 0832/1498] Bump regex from 2021.8.28 to 2021.10.23 Bumps [regex](https://bitbucket.org/mrabarnett/mrab-regex) from 2021.8.28 to 2021.10.23. - [Commits](https://bitbucket.org/mrabarnett/mrab-regex/commits) --- updated-dependencies: - dependency-name: regex dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 73242073df..d8aa90b408 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -108,7 +108,7 @@ pytest==6.2.5 # pytest-cov pytest-cov==2.12.1 # via -r test-requirements.in -regex==2021.8.28 +regex==2021.10.23 # via black six==1.16.0 # via pyopenssl From 507b2c23dd4bfe17d614c18d689362b78aa978c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:23:34 +0000 Subject: [PATCH 0833/1498] Bump prompt-toolkit from 3.0.20 to 3.0.21 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.20 to 3.0.21. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/commits) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..6d527fb91b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ backcall==0.2.0 # via ipython black==21.9b0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.6 +cffi==1.15.0 # via cryptography click==8.0.1 # via black @@ -80,7 +80,7 @@ platformdirs==2.3.0 # via black pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.20 +prompt-toolkit==3.0.21 # via ipython ptyprocess==0.7.0 # via pexpect From 3fb9ddc339f3e9c8ef2d3fc08c45b3884999c53c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:24:22 +0000 Subject: [PATCH 0834/1498] Bump jinja2 from 3.0.1 to 3.0.2 Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.1...3.0.2) --- updated-dependencies: - dependency-name: jinja2 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 621e7476ae..938aade6fa 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.5.30 # via requests -charset-normalizer==2.0.5 +charset-normalizer==2.0.7 # via requests click==8.0.1 # via @@ -38,7 +38,7 @@ immutables==0.16 # via -r docs-requirements.in incremental==21.3.0 # via towncrier -jinja2==3.0.1 +jinja2==3.0.2 # via # sphinx # towncrier @@ -52,7 +52,7 @@ pygments==2.10.0 # via sphinx pyparsing==2.4.7 # via packaging -pytz==2021.1 +pytz==2021.3 # via babel requests==2.26.0 # via sphinx From c4903f46aaf4850ce2b42ebe567f14810fba8c93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:24:46 +0000 Subject: [PATCH 0835/1498] Bump platformdirs from 2.3.0 to 2.4.0 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.3.0...2.4.0) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 66837d5257..80a1478456 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -76,7 +76,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.3.0 +platformdirs==2.4.0 # via # black # pylint From 37f275541f751623b47c70356e4e56bb3f93e105 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 11:25:10 +0000 Subject: [PATCH 0836/1498] Bump coverage from 5.5 to 6.0.2 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.5 to 6.0.2. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.5...6.0.2) --- updated-dependencies: - dependency-name: coverage dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 57fac44abd..4f4ecd6138 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,11 +19,11 @@ backcall==0.2.0 # via ipython black==21.9b0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.14.6 +cffi==1.15.0 # via cryptography click==8.0.1 # via black -coverage==5.5 +coverage==6.0.2 # via pytest-cov cryptography==35.0.0 # via From 3c1bd5cc20684d1d2ca32776f1d41cbbdee0231b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Oct 2021 17:23:49 +0000 Subject: [PATCH 0837/1498] Bump click from 8.0.1 to 8.0.3 Bumps [click](https://github.com/pallets/click) from 8.0.1 to 8.0.3. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.0.1...8.0.3) --- updated-dependencies: - dependency-name: click dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 8a7272de80..6144c84d09 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.5.30 # via requests charset-normalizer==2.0.7 # via requests -click==8.0.1 +click==8.0.3 # via # click-default-group # towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 604f635ba5..f4b5511f6a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ black==21.9b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography -click==8.0.1 +click==8.0.3 # via black coverage==6.0.2 # via pytest-cov From 97e14a74d88bfef9b6f452c52893a2bead894648 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Wed, 27 Oct 2021 08:42:57 +0900 Subject: [PATCH 0838/1498] Remove unnecessary skips in ipython tests --- trio/_core/tests/test_multierror.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 3a0debaee9..70d763e652 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -715,10 +715,6 @@ def test_custom_excepthook(): @slow @need_ipython -@pytest.mark.skipif( - sys.version_info >= (3, 9), - reason="ipython --quick currently breaks on latest dev python", -) def test_ipython_exc_handler(): completed = run_script("simple_excepthook.py", use_ipython=True) check_simple_excepthook(completed) @@ -742,10 +738,6 @@ def test_partial_imported_but_unused(): @slow @need_ipython -@pytest.mark.skipif( - sys.version_info >= (3, 9), - reason="ipython --quick currently breaks on latest dev python", -) def test_ipython_custom_exc_handler(): # Check we get a nice warning (but only one!) if the user is using IPython # and already has some other set_custom_exc handler installed. From b1db4e6d2f45294c7a203b127dbc69cea12cf876 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Oct 2021 04:38:41 +0000 Subject: [PATCH 0839/1498] Bump pytest-cov from 2.12.1 to 3.0.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.12.1 to 3.0.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.12.1...v3.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index c7ca1444d1..6c80fe90cc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,7 +23,7 @@ cffi==1.15.0 # via cryptography click==8.0.1 # via black -coverage==6.0.2 +coverage[toml]==6.0.2 # via pytest-cov cryptography==35.0.0 # via @@ -106,7 +106,7 @@ pytest==6.2.5 # via # -r test-requirements.in # pytest-cov -pytest-cov==2.12.1 +pytest-cov==3.0.0 # via -r test-requirements.in regex==2021.8.28 # via black @@ -121,9 +121,10 @@ toml==0.10.2 # mypy # pylint # pytest - # pytest-cov tomli==1.2.2 - # via black + # via + # black + # coverage traitlets==5.1.1 # via # ipython From 907ad1872d24f52709f0f555c855b12f67ecb722 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Oct 2021 10:25:20 +0000 Subject: [PATCH 0840/1498] Bump wrapt from 1.12.1 to 1.13.2 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.12.1 to 1.13.2. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.12.1...1.13.2) --- updated-dependencies: - dependency-name: wrapt dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5860fabc38..2c6f6869b8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -142,5 +142,5 @@ typing-extensions==3.10.0.2 ; implementation_name == "cpython" # pylint wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.12.1 +wrapt==1.13.2 # via astroid From bb24a6bacbb8f8ec1f72d07b6221880621347981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Wed, 27 Oct 2021 21:41:54 +0200 Subject: [PATCH 0841/1498] =?UTF-8?q?=E2=9C=A8=20Add=20support=20for=20pas?= =?UTF-8?q?sing=20contextvars=20to=20thread=20workers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_core/_generated_run.py | 8 ++++++-- trio/_core/_run.py | 31 +++++++++++++++++++++++------- trio/_threads.py | 37 ++++++++++++++++++++++++++++++------ 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/trio/_core/_generated_run.py b/trio/_core/_generated_run.py index 1272b4c73c..d20891c55e 100644 --- a/trio/_core/_generated_run.py +++ b/trio/_core/_generated_run.py @@ -102,7 +102,7 @@ def reschedule(task, next_send=_NO_SEND): raise RuntimeError("must be called from async context") -def spawn_system_task(async_fn, *args, name=None): +def spawn_system_task(async_fn, *args, name=None, context=None): """Spawn a "system" task. System tasks have a few differences from regular tasks: @@ -145,6 +145,10 @@ def spawn_system_task(async_fn, *args, name=None): case is if you're wrapping a function before spawning a new task, you might pass the original function as the ``name=`` to make debugging easier. + context: An optional ``contextvars.Context`` object with context variables + to use for this task. You would normally get a copy of the current + context with ``context = contextvars.copy_context()`` and then you would + pass that ``context`` object here. Returns: Task: the newly spawned task @@ -152,7 +156,7 @@ def spawn_system_task(async_fn, *args, name=None): """ locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True try: - return GLOBAL_RUN_CONTEXT.runner.spawn_system_task(async_fn, *args, name=name) + return GLOBAL_RUN_CONTEXT.runner.spawn_system_task(async_fn, *args, name=name, context=context) except AttributeError: raise RuntimeError("must be called from async context") diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 3174261fa9..f2db0f8df9 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1416,7 +1416,9 @@ def reschedule(self, task, next_send=_NO_SEND): if "task_scheduled" in self.instruments: self.instruments.call("task_scheduled", task) - def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): + def spawn_impl( + self, async_fn, args, nursery, name, *, system_task=False, context=None + ): ###### # Make sure the nursery is in working order @@ -1446,10 +1448,11 @@ def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): except AttributeError: name = repr(name) - if system_task: - context = self.system_context.copy() - else: - context = copy_context() + if context is None: + if system_task: + context = self.system_context.copy() + else: + context = copy_context() if not hasattr(coro, "cr_frame"): # This async function is implemented in C or Cython @@ -1527,7 +1530,7 @@ def task_exited(self, task, outcome): ################ @_public - def spawn_system_task(self, async_fn, *args, name=None): + def spawn_system_task(self, async_fn, *args, name=None, context=None): """Spawn a "system" task. System tasks have a few differences from regular tasks: @@ -1570,13 +1573,27 @@ def spawn_system_task(self, async_fn, *args, name=None): case is if you're wrapping a function before spawning a new task, you might pass the original function as the ``name=`` to make debugging easier. + context: An optional ``contextvars.Context`` object with context variables + to use for this task. You would normally get a copy of the current + context with ``context = contextvars.copy_context()`` and then you would + pass that ``context`` object here. Returns: Task: the newly spawned task """ + + async def async_fn_wrapper(): + current_async_library_cvar.set("trio") + return await async_fn() + return self.spawn_impl( - async_fn, args, self.system_nursery, name, system_task=True + async_fn_wrapper, + args, + self.system_nursery, + name, + system_task=True, + context=context, ) async def init(self, async_fn, args): diff --git a/trio/_threads.py b/trio/_threads.py index 356eb0d2d1..9a15bb77f4 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -1,12 +1,15 @@ # coding: utf-8 +import contextvars import threading import queue as stdlib_queue +import functools from itertools import count import attr import inspect import outcome +from sniffio import current_async_library_cvar import trio @@ -165,6 +168,7 @@ def do_release_then_return_result(): current_trio_token = trio.lowlevel.current_trio_token() def worker_fn(): + current_async_library_cvar.set(None) TOKEN_LOCAL.token = current_trio_token try: ret = sync_fn(*args) @@ -181,6 +185,9 @@ def worker_fn(): finally: del TOKEN_LOCAL.token + context = contextvars.copy_context() + contextvars_aware_worker_fn = functools.partial(context.run, worker_fn) + def deliver_worker_fn_result(result): try: current_trio_token.run_sync_soon(report_back_in_trio_thread_fn, result) @@ -192,7 +199,7 @@ def deliver_worker_fn_result(result): await limiter.acquire_on_behalf_of(placeholder) try: - start_thread_soon(worker_fn, deliver_worker_fn_result) + start_thread_soon(contextvars_aware_worker_fn, deliver_worker_fn_result) except: limiter.release_on_behalf_of(placeholder) raise @@ -207,7 +214,7 @@ def abort(_): return await trio.lowlevel.wait_task_rescheduled(abort) -def _run_fn_as_system_task(cb, fn, *args, trio_token=None): +def _run_fn_as_system_task(cb, fn, *args, trio_token=None, context=None): """Helper function for from_thread.run and from_thread.run_sync. Since this internally uses TrioToken.run_sync_soon, all warnings about @@ -234,7 +241,13 @@ def _run_fn_as_system_task(cb, fn, *args, trio_token=None): raise RuntimeError("this is a blocking function; call it from a thread") q = stdlib_queue.Queue() - trio_token.run_sync_soon(cb, q, fn, args) + if context is not None: + func = context.run + func_args = (cb, q, fn, args) + else: + func = cb + func_args = (q, fn, args) + trio_token.run_sync_soon(func, *func_args) return q.get().unwrap() @@ -282,14 +295,20 @@ async def unprotected_afn(): async def await_in_trio_thread_task(): q.put_nowait(await outcome.acapture(unprotected_afn)) + context = contextvars.copy_context() try: - trio.lowlevel.spawn_system_task(await_in_trio_thread_task, name=afn) + trio.lowlevel.spawn_system_task( + await_in_trio_thread_task, name=afn, context=context + ) except RuntimeError: # system nursery is closed q.put_nowait( outcome.Error(trio.RunFinishedError("system nursery is closed")) ) - return _run_fn_as_system_task(callback, afn, *args, trio_token=trio_token) + context = contextvars.copy_context() + return _run_fn_as_system_task( + callback, afn, *args, trio_token=trio_token, context=context + ) def from_thread_run_sync(fn, *args, trio_token=None): @@ -324,6 +343,8 @@ def from_thread_run_sync(fn, *args, trio_token=None): """ def callback(q, fn, args): + current_async_library_cvar.set("trio") + @disable_ki_protection def unprotected_fn(): ret = fn(*args) @@ -341,4 +362,8 @@ def unprotected_fn(): res = outcome.capture(unprotected_fn) q.put_nowait(res) - return _run_fn_as_system_task(callback, fn, *args, trio_token=trio_token) + context = contextvars.copy_context() + + return _run_fn_as_system_task( + callback, fn, *args, trio_token=trio_token, context=context + ) From 9a149d6f1c32f9cc48b5c9bd009978284422e737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Wed, 27 Oct 2021 21:42:24 +0200 Subject: [PATCH 0842/1498] =?UTF-8?q?=E2=9C=85=20Add=20tests=20for=20passi?= =?UTF-8?q?ng=20contextvars=20to=20threads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/tests/test_threads.py | 105 +++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 47727817e6..1c0bbe3fd9 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -1,3 +1,4 @@ +import contextvars import threading import queue as stdlib_queue import time @@ -468,6 +469,36 @@ async def async_fn(): # pragma: no cover await to_thread_run_sync(async_fn) +trio_test_contextvar = contextvars.ContextVar("trio_test_contextvar") + + +async def test_trio_to_thread_run_sync_contextvars(): + trio_thread = threading.current_thread() + trio_test_contextvar.set("main") + + def f(): + value = trio_test_contextvar.get() + return (value, threading.current_thread()) + + value, child_thread = await to_thread_run_sync(f) + assert value == "main" + assert child_thread != trio_thread + + def g(): + parent_value = trio_test_contextvar.get() + trio_test_contextvar.set("worker") + inner_value = trio_test_contextvar.get() + return (parent_value, inner_value, threading.current_thread()) + + parent_value, inner_value, child_thread = await to_thread_run_sync(g) + current_value = trio_test_contextvar.get() + assert parent_value == "main" + assert inner_value == "worker" + assert ( + current_value == "main" + ), "The contextvar value set on the worker would not propagate back to the main thread" + + async def test_trio_from_thread_run_sync(): # Test that to_thread_run_sync correctly "hands off" the trio token to # trio.from_thread.run_sync() @@ -545,6 +576,80 @@ async def test_from_thread_no_token(): from_thread_run_sync(_core.current_time) +async def test_trio_from_thread_run_sync_contextvars(): + trio_test_contextvar.set("main") + + def thread_fn(): + thread_parent_value = trio_test_contextvar.get() + trio_test_contextvar.set("worker") + thread_current_value = trio_test_contextvar.get() + + def back_in_main(): + back_parent_value = trio_test_contextvar.get() + trio_test_contextvar.set("back_in_main") + back_current_value = trio_test_contextvar.get() + return back_parent_value, back_current_value + + back_parent_value, back_current_value = from_thread_run_sync(back_in_main) + thread_after_value = trio_test_contextvar.get() + return ( + thread_parent_value, + thread_current_value, + thread_after_value, + back_parent_value, + back_current_value, + ) + + ( + thread_parent_value, + thread_current_value, + thread_after_value, + back_parent_value, + back_current_value, + ) = await to_thread_run_sync(thread_fn) + current_value = trio_test_contextvar.get() + assert current_value == thread_parent_value == "main" + assert thread_current_value == back_parent_value == thread_after_value == "worker" + assert back_current_value == "back_in_main" + + +async def test_trio_from_thread_run_contextvars(): + trio_test_contextvar.set("main") + + def thread_fn(): + thread_parent_value = trio_test_contextvar.get() + trio_test_contextvar.set("worker") + thread_current_value = trio_test_contextvar.get() + + async def async_back_in_main(): + back_parent_value = trio_test_contextvar.get() + trio_test_contextvar.set("back_in_main") + back_current_value = trio_test_contextvar.get() + return back_parent_value, back_current_value + + back_parent_value, back_current_value = from_thread_run(async_back_in_main) + thread_after_value = trio_test_contextvar.get() + return ( + thread_parent_value, + thread_current_value, + thread_after_value, + back_parent_value, + back_current_value, + ) + + ( + thread_parent_value, + thread_current_value, + thread_after_value, + back_parent_value, + back_current_value, + ) = await to_thread_run_sync(thread_fn) + current_value = trio_test_contextvar.get() + assert current_value == thread_parent_value == "main" + assert thread_current_value == back_parent_value == thread_after_value == "worker" + assert back_current_value == "back_in_main" + + def test_run_fn_as_system_task_catched_badly_typed_token(): with pytest.raises(RuntimeError): from_thread_run_sync(_core.current_time, trio_token="Not TrioTokentype") From ac329076fb994e6bbd21a429df965a0ea6669e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Wed, 27 Oct 2021 22:32:10 +0200 Subject: [PATCH 0843/1498] =?UTF-8?q?=E2=9C=A8=20Add=20support=20for=20cus?= =?UTF-8?q?tom=20contextvars=20context=20for=20start=5Fsoon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_core/_run.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index f2db0f8df9..7361d618c0 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -944,7 +944,7 @@ def aborted(raise_cancel): # (see test_nursery_cancel_doesnt_create_cyclic_garbage) del self._pending_excs - def start_soon(self, async_fn, *args, name=None): + def start_soon(self, async_fn, *args, name=None, context=None): """Creates a child task, scheduling ``await async_fn(*args)``. This and :meth:`start` are the two fundamental methods for @@ -979,13 +979,20 @@ def start_soon(self, async_fn, *args, name=None): before spawning a new task, you might pass the original function as the ``name=`` to make debugging easier. + context: An optional ``contextvars.Context`` object with context + variables to use for this task. You would normally get a copy + of the current context with + ``context = contextvars.copy_context()`` and then you would + pass that ``context`` object here. Raises: RuntimeError: If this nursery is no longer open (i.e. its ``async with`` block has exited). """ - GLOBAL_RUN_CONTEXT.runner.spawn_impl(async_fn, args, self, name) + GLOBAL_RUN_CONTEXT.runner.spawn_impl( + async_fn, args, self, name, context=context + ) async def start(self, async_fn, *args, name=None): r"""Creates and initializes a child task. From d678e7bc35aa2975fa9aea3f02ac37da5088fc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Wed, 27 Oct 2021 22:32:43 +0200 Subject: [PATCH 0844/1498] =?UTF-8?q?=E2=9C=85=20Add=20tests=20for=20conte?= =?UTF-8?q?xtvars=20with=20start=5Fsoon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/tests/test_contextvars.py | 52 ++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 trio/tests/test_contextvars.py diff --git a/trio/tests/test_contextvars.py b/trio/tests/test_contextvars.py new file mode 100644 index 0000000000..f166888640 --- /dev/null +++ b/trio/tests/test_contextvars.py @@ -0,0 +1,52 @@ +import contextvars + +from .. import _core + +trio_testing_contextvar = contextvars.ContextVar("trio_testing_contextvar") + + +async def test_contextvars_default(): + trio_testing_contextvar.set("main") + record = [] + + async def child(): + value = trio_testing_contextvar.get() + record.append(value) + + async with _core.open_nursery() as nursery: + nursery.start_soon(child) + assert record == ["main"] + + +async def test_contextvars_set(): + trio_testing_contextvar.set("main") + record = [] + + async def child(): + trio_testing_contextvar.set("child") + value = trio_testing_contextvar.get() + record.append(value) + + async with _core.open_nursery() as nursery: + nursery.start_soon(child) + value = trio_testing_contextvar.get() + assert record == ["child"] + assert value == "main" + + +async def test_contextvars_copy(): + trio_testing_contextvar.set("main") + context = contextvars.copy_context() + trio_testing_contextvar.set("second_main") + record = [] + + async def child(): + value = trio_testing_contextvar.get() + record.append(value) + + async with _core.open_nursery() as nursery: + nursery.start_soon(child, context=context) + nursery.start_soon(child) + value = trio_testing_contextvar.get() + assert record == ["main", "second_main"] + assert value == "second_main" From 5c2b5d06375dad8b7237a9046c510a9bcc1b4b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Wed, 27 Oct 2021 23:34:10 +0200 Subject: [PATCH 0845/1498] =?UTF-8?q?=F0=9F=90=9B=20Fix=20unnecessary=20wr?= =?UTF-8?q?apper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_core/_run.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 7361d618c0..476eaa263a 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1589,13 +1589,9 @@ def spawn_system_task(self, async_fn, *args, name=None, context=None): Task: the newly spawned task """ - - async def async_fn_wrapper(): - current_async_library_cvar.set("trio") - return await async_fn() - + current_async_library_cvar.set("trio") return self.spawn_impl( - async_fn_wrapper, + async_fn, args, self.system_nursery, name, From 2bfde9629ce8e0154f1503b61b8d1d8126761484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Wed, 27 Oct 2021 23:34:29 +0200 Subject: [PATCH 0846/1498] =?UTF-8?q?=E2=9C=85=20Fix=20tests=20with=20unor?= =?UTF-8?q?dered=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/tests/test_contextvars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_contextvars.py b/trio/tests/test_contextvars.py index f166888640..375e155394 100644 --- a/trio/tests/test_contextvars.py +++ b/trio/tests/test_contextvars.py @@ -48,5 +48,5 @@ async def child(): nursery.start_soon(child, context=context) nursery.start_soon(child) value = trio_testing_contextvar.get() - assert record == ["main", "second_main"] + assert set(record) == {"main", "second_main"} assert value == "second_main" From fa6d256314f230d286e4a2d8b745590d46d73d0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Oct 2021 10:31:36 +0000 Subject: [PATCH 0847/1498] Bump pyparsing from 3.0.1 to 3.0.3 Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 3.0.1 to 3.0.3. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_3.0.1...pyparsing_3.0.3) --- updated-dependencies: - dependency-name: pyparsing dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 0526cfdd95..d9c9be5c2f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -50,7 +50,7 @@ packaging==21.0 # via sphinx pygments==2.10.0 # via sphinx -pyparsing==3.0.1 +pyparsing==3.0.3 # via packaging pytz==2021.3 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index 2c6f6869b8..2b706488fe 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -100,7 +100,7 @@ pylint==2.11.1 # via -r test-requirements.in pyopenssl==21.0.0 # via -r test-requirements.in -pyparsing==3.0.1 +pyparsing==3.0.3 # via packaging pytest==6.2.5 # via From 6e9935b3f4d705838bafc418ac77b739fc5ebea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 12:28:40 +0200 Subject: [PATCH 0848/1498] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Move=20contextva?= =?UTF-8?q?rs=20context=20out=20of=20partial=20and=20into=20a=20closure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_threads.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index 9a15bb77f4..4408fbcaca 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -167,11 +167,13 @@ def do_release_then_return_result(): current_trio_token = trio.lowlevel.current_trio_token() + outer_context = contextvars.copy_context() + def worker_fn(): current_async_library_cvar.set(None) TOKEN_LOCAL.token = current_trio_token try: - ret = sync_fn(*args) + ret = outer_context.run(sync_fn, *args) if inspect.iscoroutine(ret): # Manually close coroutine to avoid RuntimeWarnings @@ -185,9 +187,6 @@ def worker_fn(): finally: del TOKEN_LOCAL.token - context = contextvars.copy_context() - contextvars_aware_worker_fn = functools.partial(context.run, worker_fn) - def deliver_worker_fn_result(result): try: current_trio_token.run_sync_soon(report_back_in_trio_thread_fn, result) @@ -199,7 +198,7 @@ def deliver_worker_fn_result(result): await limiter.acquire_on_behalf_of(placeholder) try: - start_thread_soon(contextvars_aware_worker_fn, deliver_worker_fn_result) + start_thread_soon(worker_fn, deliver_worker_fn_result) except: limiter.release_on_behalf_of(placeholder) raise From 7ddd95c6372b1f49c7ba39217b75b40ea0233f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 12:30:27 +0200 Subject: [PATCH 0849/1498] =?UTF-8?q?=F0=9F=94=A5=20Remove=20unused=20impo?= =?UTF-8?q?rt=20functools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_threads.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trio/_threads.py b/trio/_threads.py index 4408fbcaca..615249e084 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -3,7 +3,6 @@ import contextvars import threading import queue as stdlib_queue -import functools from itertools import count import attr From 0a9fc4122ce133de662c22257f8ce820215561d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 12:43:50 +0200 Subject: [PATCH 0850/1498] =?UTF-8?q?=E2=9C=85=20Add=20test=20for=20sniffi?= =?UTF-8?q?o=20current=5Fasync=5Flibrary=5Fcvar=20unsetting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/tests/test_threads.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 1c0bbe3fd9..4c795534e3 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -5,6 +5,7 @@ import weakref import pytest +from sniffio import current_async_library_cvar from trio._core import TrioToken, current_trio_token from .. import _core @@ -478,25 +479,32 @@ async def test_trio_to_thread_run_sync_contextvars(): def f(): value = trio_test_contextvar.get() - return (value, threading.current_thread()) + sniffio_cvar_value = current_async_library_cvar.get() + return (value, sniffio_cvar_value, threading.current_thread()) - value, child_thread = await to_thread_run_sync(f) + value, sniffio_cvar_value, child_thread = await to_thread_run_sync(f) assert value == "main" + assert sniffio_cvar_value == None assert child_thread != trio_thread def g(): parent_value = trio_test_contextvar.get() trio_test_contextvar.set("worker") inner_value = trio_test_contextvar.get() - return (parent_value, inner_value, threading.current_thread()) + sniffio_cvar_value = current_async_library_cvar.get() + return (parent_value, inner_value, sniffio_cvar_value, threading.current_thread()) - parent_value, inner_value, child_thread = await to_thread_run_sync(g) + parent_value, inner_value, sniffio_cvar_value, child_thread = await to_thread_run_sync(g) current_value = trio_test_contextvar.get() + sniffio_outer_value = current_async_library_cvar.get() assert parent_value == "main" assert inner_value == "worker" assert ( current_value == "main" ), "The contextvar value set on the worker would not propagate back to the main thread" + assert sniffio_cvar_value is None + assert sniffio_outer_value == "trio" + async def test_trio_from_thread_run_sync(): From ac46161d0d4012d3a13c3e064c579e7125d9f769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 12:44:29 +0200 Subject: [PATCH 0851/1498] =?UTF-8?q?=E2=8F=AA=20Revert=20moving=20context?= =?UTF-8?q?=20copy=20out=20of=20partial=20as=20that=20breaks=20sniffio=20c?= =?UTF-8?q?urrent=5Fasync=5Flibrary=5Fcvar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_threads.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index 615249e084..9a15bb77f4 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -3,6 +3,7 @@ import contextvars import threading import queue as stdlib_queue +import functools from itertools import count import attr @@ -166,13 +167,11 @@ def do_release_then_return_result(): current_trio_token = trio.lowlevel.current_trio_token() - outer_context = contextvars.copy_context() - def worker_fn(): current_async_library_cvar.set(None) TOKEN_LOCAL.token = current_trio_token try: - ret = outer_context.run(sync_fn, *args) + ret = sync_fn(*args) if inspect.iscoroutine(ret): # Manually close coroutine to avoid RuntimeWarnings @@ -186,6 +185,9 @@ def worker_fn(): finally: del TOKEN_LOCAL.token + context = contextvars.copy_context() + contextvars_aware_worker_fn = functools.partial(context.run, worker_fn) + def deliver_worker_fn_result(result): try: current_trio_token.run_sync_soon(report_back_in_trio_thread_fn, result) @@ -197,7 +199,7 @@ def deliver_worker_fn_result(result): await limiter.acquire_on_behalf_of(placeholder) try: - start_thread_soon(worker_fn, deliver_worker_fn_result) + start_thread_soon(contextvars_aware_worker_fn, deliver_worker_fn_result) except: limiter.release_on_behalf_of(placeholder) raise From 4891d28784134f1d5770a03c3949d52cefb25692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 13:05:35 +0200 Subject: [PATCH 0852/1498] =?UTF-8?q?=E2=9C=85=20Add=20tests=20for=20sniff?= =?UTF-8?q?io=20contextvar=20when=20using=20worker=20threads=20and=20conte?= =?UTF-8?q?xtvars?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/tests/test_threads.py | 47 ++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 4c795534e3..e481619846 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -492,9 +492,19 @@ def g(): trio_test_contextvar.set("worker") inner_value = trio_test_contextvar.get() sniffio_cvar_value = current_async_library_cvar.get() - return (parent_value, inner_value, sniffio_cvar_value, threading.current_thread()) + return ( + parent_value, + inner_value, + sniffio_cvar_value, + threading.current_thread(), + ) - parent_value, inner_value, sniffio_cvar_value, child_thread = await to_thread_run_sync(g) + ( + parent_value, + inner_value, + sniffio_cvar_value, + child_thread, + ) = await to_thread_run_sync(g) current_value = trio_test_contextvar.get() sniffio_outer_value = current_async_library_cvar.get() assert parent_value == "main" @@ -506,7 +516,6 @@ def g(): assert sniffio_outer_value == "trio" - async def test_trio_from_thread_run_sync(): # Test that to_thread_run_sync correctly "hands off" the trio token to # trio.from_thread.run_sync() @@ -591,34 +600,47 @@ def thread_fn(): thread_parent_value = trio_test_contextvar.get() trio_test_contextvar.set("worker") thread_current_value = trio_test_contextvar.get() + sniffio_cvar_thread_pre_value = current_async_library_cvar.get() def back_in_main(): back_parent_value = trio_test_contextvar.get() trio_test_contextvar.set("back_in_main") back_current_value = trio_test_contextvar.get() - return back_parent_value, back_current_value + sniffio_cvar_back_value = current_async_library_cvar.get() + return back_parent_value, back_current_value, sniffio_cvar_back_value - back_parent_value, back_current_value = from_thread_run_sync(back_in_main) + back_parent_value, back_current_value, sniffio_cvar_back_value = from_thread_run_sync(back_in_main) thread_after_value = trio_test_contextvar.get() + sniffio_cvar_thread_after_value = current_async_library_cvar.get() return ( thread_parent_value, thread_current_value, thread_after_value, + sniffio_cvar_thread_pre_value, + sniffio_cvar_thread_after_value, back_parent_value, back_current_value, + sniffio_cvar_back_value ) ( thread_parent_value, thread_current_value, thread_after_value, + sniffio_cvar_thread_pre_value, + sniffio_cvar_thread_after_value, back_parent_value, back_current_value, + sniffio_cvar_back_value, ) = await to_thread_run_sync(thread_fn) current_value = trio_test_contextvar.get() + sniffio_cvar_out_value = current_async_library_cvar.get() assert current_value == thread_parent_value == "main" assert thread_current_value == back_parent_value == thread_after_value == "worker" assert back_current_value == "back_in_main" + assert sniffio_cvar_out_value == sniffio_cvar_back_value == "trio" + assert sniffio_cvar_thread_pre_value == sniffio_cvar_thread_after_value == None + async def test_trio_from_thread_run_contextvars(): @@ -628,34 +650,45 @@ def thread_fn(): thread_parent_value = trio_test_contextvar.get() trio_test_contextvar.set("worker") thread_current_value = trio_test_contextvar.get() + sniffio_cvar_thread_pre_value = current_async_library_cvar.get() async def async_back_in_main(): back_parent_value = trio_test_contextvar.get() trio_test_contextvar.set("back_in_main") back_current_value = trio_test_contextvar.get() - return back_parent_value, back_current_value + sniffio_cvar_back_value = current_async_library_cvar.get() + return back_parent_value, back_current_value, sniffio_cvar_back_value - back_parent_value, back_current_value = from_thread_run(async_back_in_main) + back_parent_value, back_current_value, sniffio_cvar_back_value = from_thread_run(async_back_in_main) thread_after_value = trio_test_contextvar.get() + sniffio_cvar_thread_after_value = current_async_library_cvar.get() return ( thread_parent_value, thread_current_value, thread_after_value, + sniffio_cvar_thread_pre_value, + sniffio_cvar_thread_after_value, back_parent_value, back_current_value, + sniffio_cvar_back_value, ) ( thread_parent_value, thread_current_value, thread_after_value, + sniffio_cvar_thread_pre_value, + sniffio_cvar_thread_after_value, back_parent_value, back_current_value, + sniffio_cvar_back_value, ) = await to_thread_run_sync(thread_fn) current_value = trio_test_contextvar.get() assert current_value == thread_parent_value == "main" assert thread_current_value == back_parent_value == thread_after_value == "worker" assert back_current_value == "back_in_main" + assert sniffio_cvar_thread_pre_value == sniffio_cvar_thread_after_value == None + assert sniffio_cvar_back_value == "trio" def test_run_fn_as_system_task_catched_badly_typed_token(): From 6541f1577a8c5f31066928c8b344a411192f2f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 13:06:27 +0200 Subject: [PATCH 0853/1498] =?UTF-8?q?=F0=9F=90=9B=20Fix=20setting=20the=20?= =?UTF-8?q?sniffio=20contextvar=20in=20from=5Fthread.run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_threads.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trio/_threads.py b/trio/_threads.py index 9a15bb77f4..7bede39c04 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -306,6 +306,7 @@ async def await_in_trio_thread_task(): ) context = contextvars.copy_context() + context.run(current_async_library_cvar.set, "trio") return _run_fn_as_system_task( callback, afn, *args, trio_token=trio_token, context=context ) From ac36a90a574cbc540b53b88fa634bd1e99aa389e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 13:07:11 +0200 Subject: [PATCH 0854/1498] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20utili?= =?UTF-8?q?ty=20function=20to=20require=20context=20(always=20passed)=20an?= =?UTF-8?q?d=20handle=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/_threads.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index 7bede39c04..adad6343d6 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -214,7 +214,7 @@ def abort(_): return await trio.lowlevel.wait_task_rescheduled(abort) -def _run_fn_as_system_task(cb, fn, *args, trio_token=None, context=None): +def _run_fn_as_system_task(cb, fn, *args, context, trio_token=None): """Helper function for from_thread.run and from_thread.run_sync. Since this internally uses TrioToken.run_sync_soon, all warnings about @@ -241,13 +241,7 @@ def _run_fn_as_system_task(cb, fn, *args, trio_token=None, context=None): raise RuntimeError("this is a blocking function; call it from a thread") q = stdlib_queue.Queue() - if context is not None: - func = context.run - func_args = (cb, q, fn, args) - else: - func = cb - func_args = (q, fn, args) - trio_token.run_sync_soon(func, *func_args) + trio_token.run_sync_soon(context.run, cb, q, fn, args) return q.get().unwrap() @@ -308,7 +302,11 @@ async def await_in_trio_thread_task(): context = contextvars.copy_context() context.run(current_async_library_cvar.set, "trio") return _run_fn_as_system_task( - callback, afn, *args, trio_token=trio_token, context=context + callback, + afn, + *args, + context=context, + trio_token=trio_token, ) @@ -366,5 +364,9 @@ def unprotected_fn(): context = contextvars.copy_context() return _run_fn_as_system_task( - callback, fn, *args, trio_token=trio_token, context=context + callback, + fn, + *args, + context=context, + trio_token=trio_token, ) From fe83fd63265241445d77b8a257e4b50e24848d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 13:10:55 +0200 Subject: [PATCH 0855/1498] =?UTF-8?q?=F0=9F=8E=A8=20Fix=20formatting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- trio/tests/test_threads.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index e481619846..3cf8ebd867 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -609,7 +609,11 @@ def back_in_main(): sniffio_cvar_back_value = current_async_library_cvar.get() return back_parent_value, back_current_value, sniffio_cvar_back_value - back_parent_value, back_current_value, sniffio_cvar_back_value = from_thread_run_sync(back_in_main) + ( + back_parent_value, + back_current_value, + sniffio_cvar_back_value, + ) = from_thread_run_sync(back_in_main) thread_after_value = trio_test_contextvar.get() sniffio_cvar_thread_after_value = current_async_library_cvar.get() return ( @@ -620,7 +624,7 @@ def back_in_main(): sniffio_cvar_thread_after_value, back_parent_value, back_current_value, - sniffio_cvar_back_value + sniffio_cvar_back_value, ) ( @@ -642,7 +646,6 @@ def back_in_main(): assert sniffio_cvar_thread_pre_value == sniffio_cvar_thread_after_value == None - async def test_trio_from_thread_run_contextvars(): trio_test_contextvar.set("main") @@ -659,7 +662,11 @@ async def async_back_in_main(): sniffio_cvar_back_value = current_async_library_cvar.get() return back_parent_value, back_current_value, sniffio_cvar_back_value - back_parent_value, back_current_value, sniffio_cvar_back_value = from_thread_run(async_back_in_main) + ( + back_parent_value, + back_current_value, + sniffio_cvar_back_value, + ) = from_thread_run(async_back_in_main) thread_after_value = trio_test_contextvar.get() sniffio_cvar_thread_after_value = current_async_library_cvar.get() return ( From 3250e5316fbfb4fa1411eee1edfad8e819188606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 19:46:51 +0200 Subject: [PATCH 0856/1498] =?UTF-8?q?=F0=9F=93=9D=20Add=20newsfragment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- newsfragments/2160.feature.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 newsfragments/2160.feature.rst diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst new file mode 100644 index 0000000000..0be330a209 --- /dev/null +++ b/newsfragments/2160.feature.rst @@ -0,0 +1,6 @@ +Now context variables set with `contextvars` are preserved when runing functions +in a worker thread with `trio.to_thread.run_sync(sync_fn, ...)`, or when running +functions from the worker thread in the parent Trio thread with +`trio.from_thread.run(async_fn, ...)`, and `trio.from_thread.run_sync(sync_fn, ...)`. +This is done by automatically copying the `contextvars` context. You can also pass a +custom `contextvars` context to `nursery.start_soon(child, context=context)`. From a268435dd6b87d1374de983a860550a6110d94a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 19:48:28 +0200 Subject: [PATCH 0857/1498] =?UTF-8?q?=F0=9F=8D=B1=20Add=20example=20of=20c?= =?UTF-8?q?ontextvars=20with=20threads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../thread-contextvars-example.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/source/reference-core/thread-contextvars-example.py diff --git a/docs/source/reference-core/thread-contextvars-example.py b/docs/source/reference-core/thread-contextvars-example.py new file mode 100644 index 0000000000..d6c062bea4 --- /dev/null +++ b/docs/source/reference-core/thread-contextvars-example.py @@ -0,0 +1,47 @@ +import contextvars +import time + +import trio + +request_state = contextvars.ContextVar("request_state") + +# Blocking function that should be run on a thread +# It could be reading or writing files, communicating with a database +# with a driver not compatible with async / await, etc. +def work_in_thread(msg): + # Only use request_state.get() inside the worker thread + state_value = request_state.get() + current_user_id = state_value["current_user_id"] + time.sleep(3) # this would be some blocking call, like reading a file + print(f"Processed user {current_user_id} with message {msg} in a thread worker") + # Modify/mutate the state object, without setting the entire + # contextvar with request_state.set() + state_value["msg"] = msg + + +# An example "request handler" that does some work itself and also +# spawns some helper tasks in threads to execute blocking code. +async def handle_request(current_user_id): + # Write to task-local storage: + current_state = {"current_user_id": current_user_id, "msg": ""} + request_state.set(current_state) + + # Here the current implicit contextvars context will be automatically copied + # inside the worker thread + await trio.to_thread.run_sync(work_in_thread, f"Hello {current_user_id}") + # Extract the value set inside the thread in the same object stored in a contextvar + new_msg = current_state["msg"] + print( + f"New contextvar value from worker thread for user {current_user_id}: {new_msg}" + ) + + +# Spawn several "request handlers" simultaneously, to simulate a +# busy server handling multiple requests at the same time. +async def main(): + async with trio.open_nursery() as nursery: + for i in range(3): + nursery.start_soon(handle_request, i) + + +trio.run(main) From 83e1b2e7f62995060ea892cdf5ef2f10901d2631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 19:48:48 +0200 Subject: [PATCH 0858/1498] =?UTF-8?q?=F0=9F=93=9D=20Add=20docs=20for=20con?= =?UTF-8?q?textvars=20with=20threads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/reference-core.rst | 72 +++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 514b5072c3..515f236cd0 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1021,8 +1021,18 @@ Example output (yours may differ slightly): request 0: Request received finished For more information, read the -`contextvar docs `__. +`contextvars docs `__. +.. note:: + + In advanced use cases you can also pass a custom `contextvars` context to + ``nursery.start_soon(child, context=context)``. + + That's an advanced use case, and you probably won't need that easily, but now you + know in case you need that in the future. + + You will also learn about how to use these `contextvars` with threads later here + in the docs. .. _synchronization: @@ -1820,6 +1830,66 @@ to spawn a child thread, and then use a :ref:`memory channel .. literalinclude:: reference-core/from-thread-example.py +Threads and task-local storage +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When working with threads, you can use the same `contextvars` we discussed above, +because their values are preserved. + +This is done by automatically copying the `contextvars` context when you use any of: + +* ``trio.to_thread.run_sync(sync_fn, ...)`` +* ``trio.from_thread.run(async_fn, ...)`` +* ``trio.from_thread.run_sync(sync_fn, ...)`` + +That means that the values of the context variables are accessible even in worker +threads, or when sending a function to be run in the main/parent Trio thread using +``trio.from_thread.run(async_fn, ...)`` *from* one of these worker threads. + +But it also means that as the context is not the same but a copy, if you `set` the +context variable value *inside* one of these functions that work in threads, the +new value will only be available in that context (that was copied). So, the new value +will be available for that function and other internal/children tasks, but the value +won't be available in the parent thread. + +If you need to modify values that would live in the context variables and you need to +make those modifications from the children threads, you can instead set an object +(e.g. a dictionary) in the context variable, in the top level/parent Trio thread. And +then in the children, instead of setting the context variable, you can ``get`` the same +object, and modify its values. That way you keep the same object in the context +variable and only mutate it in children threads. + +This way, you can modify the object content in children threads and still access the +new content in the parent thread. + +Here's an example: + +.. literalinclude:: reference-core/thread-contextvars-example.py + +Running that script will result in the output: + +.. code-block:: none + + Processed user 2 with message Hello 2 in a thread worker + Processed user 0 with message Hello 0 in a thread worker + Processed user 1 with message Hello 1 in a thread worker + New contextvar value from worker thread for user 2: Hello 2 + New contextvar value from worker thread for user 1: Hello 1 + New contextvar value from worker thread for user 0: Hello 0 + +If you are using ``contextvars`` or you are using a library that uses them, now you +know how they interact when working with threads in Trio. + +But have in mind that in many cases it might be a lot simpler to *not* use context +variables in your own code and instead pass values in arguments, as it might be more +explicit and might be easier to reason about. + +.. note:: + + The context is automatically copied instead of using the same parent context because + a single context can't be used in more than one thread, it's not supported by + ``contextvars``. + Exceptions and warnings ----------------------- From 4b3dd53b485452b5997e3ae94e6bdee3abf9f640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 20:32:02 +0200 Subject: [PATCH 0859/1498] =?UTF-8?q?=F0=9F=93=9D=20Fix=20docs=20reference?= =?UTF-8?q?s=20to=20internal=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/reference-core.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 515f236cd0..5e7ca9c56b 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1838,13 +1838,13 @@ because their values are preserved. This is done by automatically copying the `contextvars` context when you use any of: -* ``trio.to_thread.run_sync(sync_fn, ...)`` -* ``trio.from_thread.run(async_fn, ...)`` -* ``trio.from_thread.run_sync(sync_fn, ...)`` +* `trio.to_thread.run_sync` +* `trio.from_thread.run` +* `trio.from_thread.run_sync` That means that the values of the context variables are accessible even in worker threads, or when sending a function to be run in the main/parent Trio thread using -``trio.from_thread.run(async_fn, ...)`` *from* one of these worker threads. +`trio.from_thread.run` *from* one of these worker threads. But it also means that as the context is not the same but a copy, if you `set` the context variable value *inside* one of these functions that work in threads, the From 33202e97dface74d7cdf4f396f4806e1740a6882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 29 Oct 2021 20:35:23 +0200 Subject: [PATCH 0860/1498] =?UTF-8?q?=F0=9F=93=9D=20Fix=20internal=20funct?= =?UTF-8?q?ion=20references=20in=20newsfragment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- newsfragments/2160.feature.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst index 0be330a209..9bb4f6ae44 100644 --- a/newsfragments/2160.feature.rst +++ b/newsfragments/2160.feature.rst @@ -1,6 +1,6 @@ Now context variables set with `contextvars` are preserved when runing functions -in a worker thread with `trio.to_thread.run_sync(sync_fn, ...)`, or when running +in a worker thread with `trio.to_thread.run_sync`, or when running functions from the worker thread in the parent Trio thread with -`trio.from_thread.run(async_fn, ...)`, and `trio.from_thread.run_sync(sync_fn, ...)`. +`trio.from_thread.run`, and `trio.from_thread.run_sync`. This is done by automatically copying the `contextvars` context. You can also pass a -custom `contextvars` context to `nursery.start_soon(child, context=context)`. +custom `contextvars` context to ``nursery.start_soon(child, context=context)``. From 1a7671178c46fb5ee83b91462cb4a904eef607ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 10:20:10 +0000 Subject: [PATCH 0861/1498] Bump packaging from 21.0 to 21.2 Bumps [packaging](https://github.com/pypa/packaging) from 21.0 to 21.2. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/21.0...21.2) --- updated-dependencies: - dependency-name: packaging dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- test-requirements.txt | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index d9c9be5c2f..b9c1f3d79f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -46,11 +46,11 @@ markupsafe==2.0.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in -packaging==21.0 +packaging==21.2 # via sphinx pygments==2.10.0 # via sphinx -pyparsing==3.0.3 +pyparsing==2.4.7 # via packaging pytz==2021.3 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index 2b706488fe..f204c60516 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -66,7 +66,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # mypy outcome==1.1.0 # via -r test-requirements.in -packaging==21.0 +packaging==21.2 # via pytest parso==0.8.2 # via jedi @@ -100,7 +100,7 @@ pylint==2.11.1 # via -r test-requirements.in pyopenssl==21.0.0 # via -r test-requirements.in -pyparsing==3.0.3 +pyparsing==2.4.7 # via packaging pytest==6.2.5 # via @@ -136,10 +136,8 @@ typed-ast==1.4.3 ; implementation_name == "cpython" typing-extensions==3.10.0.2 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.2 From 95544e4bba6ae76ec562b5c368eae63e82e0e3fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 10:21:36 +0000 Subject: [PATCH 0862/1498] Bump black from 21.9b0 to 21.10b0 Bumps [black](https://github.com/psf/black) from 21.9b0 to 21.10b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2b706488fe..8e3a844607 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==21.2.0 # pytest backcall==0.2.0 # via ipython -black==21.9b0 ; implementation_name == "cpython" +black==21.10b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography @@ -136,10 +136,8 @@ typed-ast==1.4.3 ; implementation_name == "cpython" typing-extensions==3.10.0.2 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.2 From a7d194d73d19fceb10169e9ab0f5adc484423e4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 13:24:22 +0000 Subject: [PATCH 0863/1498] Bump ipython from 7.28.0 to 7.29.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.28.0 to 7.29.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.28.0...7.29.0) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8e3a844607..63102fdc15 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -41,7 +41,7 @@ immutables==0.16 # via -r test-requirements.in iniconfig==1.1.1 # via pytest -ipython==7.28.0 +ipython==7.29.0 # via -r test-requirements.in isort==5.9.3 # via pylint From 06c26d64334056070b11bf89274bceaa2e8d61e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 13:24:29 +0000 Subject: [PATCH 0864/1498] Bump wrapt from 1.13.2 to 1.13.3 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.13.2 to 1.13.3. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.13.2...1.13.3) --- updated-dependencies: - dependency-name: wrapt dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8e3a844607..1eb9838160 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -140,5 +140,5 @@ typing-extensions==3.10.0.2 ; implementation_name == "cpython" # mypy wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.13.2 +wrapt==1.13.3 # via astroid From b2bacb82e7aa94a96b95b0f88fab41f854b9dddf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Nov 2021 10:20:41 +0000 Subject: [PATCH 0865/1498] Bump regex from 2021.10.23 to 2021.11.1 Bumps [regex](https://github.com/mrabarnett/mrab-regex) from 2021.10.23 to 2021.11.1. - [Release notes](https://github.com/mrabarnett/mrab-regex/releases) - [Commits](https://github.com/mrabarnett/mrab-regex/commits) --- updated-dependencies: - dependency-name: regex dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 61effc01b3..3aaa4c3115 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -108,7 +108,7 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -regex==2021.10.23 +regex==2021.11.1 # via black six==1.16.0 # via pyopenssl From 3cc00290bfacc12f9391441527c96c6192993028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 15 Nov 2021 11:08:15 -0500 Subject: [PATCH 0866/1498] =?UTF-8?q?=E2=9C=8F=20Update=20docs/source/refe?= =?UTF-8?q?rence-core.rst?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: richardsheridan --- docs/source/reference-core.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 5e7ca9c56b..1842cf7732 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1028,8 +1028,8 @@ For more information, read the In advanced use cases you can also pass a custom `contextvars` context to ``nursery.start_soon(child, context=context)``. - That's an advanced use case, and you probably won't need that easily, but now you - know in case you need that in the future. + That's an advanced use case that you probably won't need, but now you + know why that keyword argument exists! You will also learn about how to use these `contextvars` with threads later here in the docs. From 81320fc89714d1c907b19ba10b9f160cd392e6cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Mon, 15 Nov 2021 11:08:58 -0500 Subject: [PATCH 0867/1498] =?UTF-8?q?=E2=9C=8F=20Update=20docs/source/refe?= =?UTF-8?q?rence-core.rst?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: richardsheridan --- docs/source/reference-core.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 1842cf7732..720bfbf590 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1853,13 +1853,13 @@ will be available for that function and other internal/children tasks, but the v won't be available in the parent thread. If you need to modify values that would live in the context variables and you need to -make those modifications from the children threads, you can instead set an object -(e.g. a dictionary) in the context variable, in the top level/parent Trio thread. And -then in the children, instead of setting the context variable, you can ``get`` the same +make those modifications from the child threads, you can instead set a mutable object +(e.g. a dictionary) in the context variable of the top level/parent Trio thread. +Then in the children, instead of setting the context variable, you can ``get`` the same object, and modify its values. That way you keep the same object in the context -variable and only mutate it in children threads. +variable and only mutate it in child threads. -This way, you can modify the object content in children threads and still access the +This way, you can modify the object content in child threads and still access the new content in the parent thread. Here's an example: From af99a1f0840c0056aa9ab9a11d498ff54eb67475 Mon Sep 17 00:00:00 2001 From: David Delassus Date: Fri, 26 Nov 2021 17:04:29 +0100 Subject: [PATCH 0868/1498] Add new awesome trio library "triotp" --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 4b50772c1d..29a70f8a2f 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -105,7 +105,7 @@ Tools and Utilities * `tenacity `__ - Retrying library for Python with async/await support. * `perf-timer `__ - A code timer with Trio async support (see ``TrioPerfTimer``). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers ``trio_perf_counter()`` for low-level timing. * `aiometer `__ - Execute lots of tasks concurrently while controlling concurrency limits - +* `triotp `__ - The OTP framework for Python Trio Trio/Asyncio Interoperability ----------------------------- From c4761d7640c6f134a4818667aa8f9243f06335cd Mon Sep 17 00:00:00 2001 From: David Delassus Date: Mon, 29 Nov 2021 08:17:24 +0100 Subject: [PATCH 0869/1498] Update awesome-trio-libraries (replace "the OTP" with just "OTP") Co-authored-by: Quentin Pradet --- docs/source/awesome-trio-libraries.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 29a70f8a2f..37bf4c08bb 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -105,7 +105,7 @@ Tools and Utilities * `tenacity `__ - Retrying library for Python with async/await support. * `perf-timer `__ - A code timer with Trio async support (see ``TrioPerfTimer``). Collects execution time of a block of code excluding time when the coroutine isn't scheduled, such as during blocking I/O and sleep. Also offers ``trio_perf_counter()`` for low-level timing. * `aiometer `__ - Execute lots of tasks concurrently while controlling concurrency limits -* `triotp `__ - The OTP framework for Python Trio +* `triotp `__ - OTP framework for Python Trio Trio/Asyncio Interoperability ----------------------------- From c835c38763106f3eeec689149b2746f9fd7106b2 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Fri, 10 Dec 2021 09:41:00 -0700 Subject: [PATCH 0870/1498] Fix documentation references to `socket.socket` This is a class in the stdlib, not a function. I didn't really dig into why/when sphinx decided to get fussy about it. I tested this by building the docs and clicking the link on one of the changed sites, and it took me to the right place in the Python stdlib docs --- docs/source/reference-io.rst | 8 ++++---- trio/_socket.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index 81a615e039..4d33601f17 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -301,7 +301,7 @@ library socket into a Trio socket: .. autofunction:: from_stdlib_socket -Unlike :func:`socket.socket`, :func:`trio.socket.socket` is a +Unlike :class:`socket.socket`, :func:`trio.socket.socket` is a function, not a class; if you want to check whether an object is a Trio socket, use ``isinstance(obj, trio.socket.SocketType)``. @@ -380,7 +380,7 @@ Socket objects additional error checking. In addition, the following methods are similar to the equivalents - in :func:`socket.socket`, but have some Trio-specific quirks: + in :class:`socket.socket`, but have some Trio-specific quirks: .. method:: connect :async: @@ -421,7 +421,7 @@ Socket objects False otherwise. The following methods are identical to their equivalents in - :func:`socket.socket`, except async, and the ones that take address + :class:`socket.socket`, except async, and the ones that take address arguments require pre-resolved addresses: * :meth:`~socket.socket.accept` @@ -437,7 +437,7 @@ Socket objects * :meth:`~socket.socket.sendmsg` (if available) All methods and attributes *not* mentioned above are identical to - their equivalents in :func:`socket.socket`: + their equivalents in :class:`socket.socket`: * :attr:`~socket.socket.family` * :attr:`~socket.socket.type` diff --git a/trio/_socket.py b/trio/_socket.py index c805f829ff..4e4e603726 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -231,7 +231,7 @@ async def getprotobyname(name): def from_stdlib_socket(sock): - """Convert a standard library :func:`socket.socket` object into a Trio + """Convert a standard library :class:`socket.socket` object into a Trio socket object. """ @@ -271,7 +271,7 @@ def socket( proto=0, fileno=None, ): - """Create a new Trio socket, like :func:`socket.socket`. + """Create a new Trio socket, like :class:`socket.socket`. This function's behavior can be customized using :func:`set_custom_socket_factory`. From d26bbffbbac407d8a9b58a5634a63a8fa27887c2 Mon Sep 17 00:00:00 2001 From: Lura Skye Date: Thu, 23 Dec 2021 05:09:46 +0000 Subject: [PATCH 0871/1498] Only use typed_ast on 3.7. --- test-requirements.in | 3 ++- test-requirements.txt | 27 +++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 9f5d6ec99f..b060211281 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -14,7 +14,8 @@ flake8 astor # code generation # https://github.com/python-trio/trio/pull/654#issuecomment-420518745 -typed_ast; implementation_name == "cpython" +# typed_ast is deprecatedd as of 3.8, and straight up doesn't compile on 3.10-dev as of 2021-12-13 +typed_ast; implementation_name == "cpython" and python_version < "3.8" mypy-extensions; implementation_name == "cpython" typing-extensions; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index dc19c3131e..a3ea5f58d3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # -# pip-compile --output-file test-requirements.txt test-requirements.in +# pip-compile --output-file=test-requirements.txt test-requirements.in # astor==0.8.1 # via -r test-requirements.in @@ -39,6 +39,12 @@ idna==3.3 # trustme immutables==0.16 # via -r test-requirements.in +importlib-metadata==4.2.0 + # via + # click + # flake8 + # pluggy + # pytest iniconfig==1.1.1 # via pytest ipython==7.29.0 @@ -131,14 +137,27 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" - # via -r test-requirements.in +typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" + # via + # -r test-requirements.in + # astroid + # black + # mypy typing-extensions==3.10.0.2 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid # black + # immutables + # importlib-metadata # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid +zipp==3.6.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 47250b18fbe4dcc8ac8ae0a97e8ad52471d46c29 Mon Sep 17 00:00:00 2001 From: Lura Skye Date: Thu, 23 Dec 2021 05:13:59 +0000 Subject: [PATCH 0872/1498] Pin cryptography to 36.0.0. --- test-requirements.in | 13 +++++++------ test-requirements.txt | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index b060211281..dc7ecdfd8e 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -1,11 +1,12 @@ # For tests -pytest >= 5.0 # for faulthandler in core +pytest >= 5.0 # for faulthandler in core pytest-cov >= 2.6.0 -ipython # for the IPython traceback integration tests -pyOpenSSL # for the ssl tests -trustme # for the ssl tests -pylint # for pylint finding all symbols tests -jedi # for jedi code completion tests +ipython # for the IPython traceback integration tests +pyOpenSSL # for the ssl tests +trustme # for the ssl tests +pylint # for pylint finding all symbols tests +jedi # for jedi code completion tests +cryptography>=36.0.0 # 35.0.0 is transitive but fails # Tools black; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index a3ea5f58d3..f1162efdb0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,8 +25,9 @@ click==8.0.3 # via black coverage[toml]==6.0.2 # via pytest-cov -cryptography==35.0.0 +cryptography==36.0.1 # via + # -r test-requirements.in # pyopenssl # trustme decorator==5.1.0 From 313f17876fc4469dd1fe6bce3afe3d944f872e1c Mon Sep 17 00:00:00 2001 From: Lura Skye Date: Thu, 23 Dec 2021 05:50:34 +0000 Subject: [PATCH 0873/1498] Fix OpenSSL on Python 3.10. This works around a CPython bug whhere the right exception type isn't raised. --- test-requirements.in | 2 +- trio/_highlevel_ssl_helpers.py | 4 ++++ trio/_ssl.py | 21 +++++++++++++++------ trio/tests/test_ssl.py | 12 ++++++++++-- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index dc7ecdfd8e..894fc654d3 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -15,7 +15,7 @@ flake8 astor # code generation # https://github.com/python-trio/trio/pull/654#issuecomment-420518745 -# typed_ast is deprecatedd as of 3.8, and straight up doesn't compile on 3.10-dev as of 2021-12-13 +# typed_ast is deprecated as of 3.8, and straight up doesn't compile on 3.10-dev as of 2021-12-13 typed_ast; implementation_name == "cpython" and python_version < "3.8" mypy-extensions; implementation_name == "cpython" typing-extensions; implementation_name == "cpython" diff --git a/trio/_highlevel_ssl_helpers.py b/trio/_highlevel_ssl_helpers.py index fb21f239e0..a339a3d238 100644 --- a/trio/_highlevel_ssl_helpers.py +++ b/trio/_highlevel_ssl_helpers.py @@ -53,6 +53,10 @@ async def open_ssl_over_tcp_stream( ) if ssl_context is None: ssl_context = ssl.create_default_context() + + if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + ssl_context.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + return trio.SSLStream( tcp_stream, ssl_context, server_hostname=host, https_compatible=https_compatible ) diff --git a/trio/_ssl.py b/trio/_ssl.py index 5f52b6ff3b..f9bb90afd8 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -190,6 +190,16 @@ STARTING_RECEIVE_SIZE = 16384 +def _is_eof(exc): + # There appears to be a bug on Python 3.10, where SSLErrors + # aren't properly translated into SSLEOFErrors. + # This stringly-typed error check is borrowed from the AnyIO + # project. + return isinstance(exc, _stdlib_ssl.SSLEOFError) or ( + hasattr(exc, "strerror") and "UNEXPECTED_EOF_WHILE_READING" in exc.strerror + ) + + class NeedHandshakeError(Exception): """Some :class:`SSLStream` methods can't return any meaningful data until after the handshake. If you call them before the handshake, they raise @@ -658,9 +668,9 @@ async def receive_some(self, max_bytes=None): # For some reason, EOF before handshake sometimes raises # SSLSyscallError instead of SSLEOFError (e.g. on my linux # laptop, but not on appveyor). Thanks openssl. - if self._https_compatible and isinstance( - exc.__cause__, - (_stdlib_ssl.SSLEOFError, _stdlib_ssl.SSLSyscallError), + if self._https_compatible and ( + isinstance(exc.__cause__, _stdlib_ssl.SSLSyscallError) + or _is_eof(exc.__cause__) ): await trio.lowlevel.checkpoint() return b"" @@ -683,9 +693,8 @@ async def receive_some(self, max_bytes=None): # BROKEN. But that's actually fine, because after getting an # EOF on TLS then the only thing you can do is close the # stream, and closing doesn't care about the state. - if self._https_compatible and isinstance( - exc.__cause__, _stdlib_ssl.SSLEOFError - ): + + if self._https_compatible and _is_eof(exc.__cause__): await trio.lowlevel.checkpoint() return b"" else: diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 933e0be470..f3bff42195 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -20,7 +20,7 @@ from .._core import ClosedResourceError, BrokenResourceError from .._highlevel_open_tcp_stream import open_tcp_stream from .. import socket as tsocket -from .._ssl import SSLStream, SSLListener, NeedHandshakeError +from .._ssl import SSLStream, SSLListener, NeedHandshakeError, _is_eof from .._util import ConflictDetector from .._core.tests.tutil import slow @@ -55,6 +55,9 @@ TRIO_TEST_1_CERT = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") SERVER_CTX = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) +if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + SERVER_CTX.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) # TLS 1.3 has a lot of changes from previous versions. So we want to run tests @@ -76,6 +79,10 @@ @pytest.fixture(scope="module", params=client_ctx_params) def client_ctx(request): ctx = ssl.create_default_context() + + if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): + ctx.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF + TRIO_TEST_CA.configure_trust(ctx) if request.param in ["default", "tls13"]: return ctx @@ -1105,7 +1112,8 @@ async def test_ssl_https_compatibility_disagreement(client_ctx): async def receive_and_expect_error(): with pytest.raises(BrokenResourceError) as excinfo: await server.receive_some(10) - assert isinstance(excinfo.value.__cause__, ssl.SSLEOFError) + + assert _is_eof(excinfo.value.__cause__) async with _core.open_nursery() as nursery: nursery.start_soon(client.aclose) From a43b832b7dd215dbe51b5d21bd81710124f81fe2 Mon Sep 17 00:00:00 2001 From: Lura Skye Date: Thu, 23 Dec 2021 18:48:32 +0000 Subject: [PATCH 0874/1498] Add newsfragment for OpenSSL fix. --- newsfragments/2203.bugfix.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/2203.bugfix.rst diff --git a/newsfragments/2203.bugfix.rst b/newsfragments/2203.bugfix.rst new file mode 100644 index 0000000000..55a01d737e --- /dev/null +++ b/newsfragments/2203.bugfix.rst @@ -0,0 +1,2 @@ +Add compatibility with OpenSSL 3.0 on newer Python and PyPy versions by working +around ``SSLEOFError`` not being raised properly. \ No newline at end of file From 65de07a8e27ce706e39009f736b54a32befd5bef Mon Sep 17 00:00:00 2001 From: Tom Pohl Date: Sun, 2 Jan 2022 17:10:21 +0100 Subject: [PATCH 0875/1498] docs: fix minor code example issues Remove redundant f-string. Replace redundant loop variable with underscore. --- docs/source/reference-lowlevel.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index e0a604bebc..4fb9abf299 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -701,8 +701,8 @@ can use as a model:: # A tiny Trio program async def trio_main(): - for i in range(5): - print(f"Hello from Trio!") + for _ in range(5): + print("Hello from Trio!") # This is inside Trio, so we have to use Trio APIs await trio.sleep(1) return "trio done!" From fe59176172eb177963280e153f06b9646b8fed32 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Thu, 9 Dec 2021 11:47:08 -0700 Subject: [PATCH 0876/1498] Fix leaking file descriptors when subprocess creation fails If subprocess spawning fails (i.e. unsuccessful call to `exec`), the created trio resources (i.e. the trio ends of the pipes that will be used to send/receive data to the child) don't get cleaned up until the next garbage collection run (which will call their `__del__`). It would be nicer if this was deterministic, and that resources created during the run of `open_process` were either completely transferred to the `Process` object (which can take over the management of them) or cleaned up (on failure). To implement this, two ExitStacks are used: One will manage the resources that need cleaned up unconditionally (the child ends of the pipe); the other will manage the resources that need cleaned up on failure. Trio currently supports versions of python that lack `contextlib.AsyncExitStack`. While there is a backport for 3.6 available on PyPI, the objects that represent the pipes (_FdHolder and _HandleHolder) don't really need to have an async close. They are internal implementation details anyway, so this refactors their implementation to have the Stream/AsyncResource methods on the actual Stream objects themselves. This allows us to use the regular ExitStack and avoid an extra dep --- trio/_subprocess.py | 66 ++++++++++++++------------- trio/_subprocess_platform/__init__.py | 15 +++++- trio/_unix_pipes.py | 9 ++-- trio/_windows_pipes.py | 20 ++++---- trio/tests/test_subprocess.py | 19 +++++++- 5 files changed, 84 insertions(+), 45 deletions(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index f836e45e95..154a940df8 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -3,6 +3,7 @@ import os import subprocess import sys +from contextlib import ExitStack from typing import Optional from functools import partial import warnings @@ -28,6 +29,7 @@ def pidfd_open(fd: int, flags: int) -> int: ... + from ._subprocess_platform import ClosableReceiveStream, ClosableSendStream else: can_try_pidfd_open = True @@ -359,28 +361,38 @@ async def open_process( "on UNIX systems" ) - trio_stdin = None # type: Optional[SendStream] - trio_stdout = None # type: Optional[ReceiveStream] - trio_stderr = None # type: Optional[ReceiveStream] - - if stdin == subprocess.PIPE: - trio_stdin, stdin = create_pipe_to_child_stdin() - if stdout == subprocess.PIPE: - trio_stdout, stdout = create_pipe_from_child_output() - if stderr == subprocess.STDOUT: - # If we created a pipe for stdout, pass the same pipe for - # stderr. If stdout was some non-pipe thing (DEVNULL or a - # given FD), pass the same thing. If stdout was passed as - # None, keep stderr as STDOUT to allow subprocess to dup - # our stdout. Regardless of which of these is applicable, - # don't create a new Trio stream for stderr -- if stdout - # is piped, stderr will be intermixed on the stdout stream. - if stdout is not None: - stderr = stdout - elif stderr == subprocess.PIPE: - trio_stderr, stderr = create_pipe_from_child_output() + trio_stdin = None # type: Optional[ClosableSendStream] + trio_stdout = None # type: Optional[ClosableReceiveStream] + trio_stderr = None # type: Optional[ClosableReceiveStream] + # Close the parent's handle for each child side of a pipe; we want the child to + # have the only copy, so that when it exits we can read EOF on our side. The + # trio ends of pipes will be transferred to the Process object, which will be + # responsible for their lifetime. If process spawning fails, though, we still + # want to close them before letting the failure bubble out + with ExitStack() as always_cleanup, ExitStack() as cleanup_on_fail: + if stdin == subprocess.PIPE: + trio_stdin, stdin = create_pipe_to_child_stdin() + always_cleanup.callback(os.close, stdin) + cleanup_on_fail.callback(trio_stdin.close) + if stdout == subprocess.PIPE: + trio_stdout, stdout = create_pipe_from_child_output() + always_cleanup.callback(os.close, stdout) + cleanup_on_fail.callback(trio_stdout.close) + if stderr == subprocess.STDOUT: + # If we created a pipe for stdout, pass the same pipe for + # stderr. If stdout was some non-pipe thing (DEVNULL or a + # given FD), pass the same thing. If stdout was passed as + # None, keep stderr as STDOUT to allow subprocess to dup + # our stdout. Regardless of which of these is applicable, + # don't create a new Trio stream for stderr -- if stdout + # is piped, stderr will be intermixed on the stdout stream. + if stdout is not None: + stderr = stdout + elif stderr == subprocess.PIPE: + trio_stderr, stderr = create_pipe_from_child_output() + always_cleanup.callback(os.close, stderr) + cleanup_on_fail.callback(trio_stderr.close) - try: popen = await trio.to_thread.run_sync( partial( subprocess.Popen, @@ -391,16 +403,8 @@ async def open_process( **options, ) ) - finally: - # Close the parent's handle for each child side of a pipe; - # we want the child to have the only copy, so that when - # it exits we can read EOF on our side. - if trio_stdin is not None: - os.close(stdin) - if trio_stdout is not None: - os.close(stdout) - if trio_stderr is not None: - os.close(stderr) + # We did not fail, so dismiss the stack for the trio ends + cleanup_on_fail.pop_all() return Process._create(popen, trio_stdin, trio_stdout, trio_stderr) diff --git a/trio/_subprocess_platform/__init__.py b/trio/_subprocess_platform/__init__.py index 418808a8ac..e03c2e5158 100644 --- a/trio/_subprocess_platform/__init__.py +++ b/trio/_subprocess_platform/__init__.py @@ -13,6 +13,17 @@ _create_child_pipe_error: Optional[ImportError] = None +if TYPE_CHECKING: + # internal types for the pipe representations used in type checking only + class ClosableSendStream(SendStream): + def close(self) -> None: + ... + + + class ClosableReceiveStream(ReceiveStream): + def close(self) -> None: + ... + # Fallback versions of the functions provided -- implementations # per OS are imported atop these at the bottom of the module. async def wait_child_exiting(process: "_subprocess.Process") -> None: @@ -30,7 +41,7 @@ async def wait_child_exiting(process: "_subprocess.Process") -> None: raise NotImplementedError from _wait_child_exiting_error # pragma: no cover -def create_pipe_to_child_stdin() -> Tuple[SendStream, int]: +def create_pipe_to_child_stdin() -> Tuple["ClosableSendStream", int]: """Create a new pipe suitable for sending data from this process to the standard input of a child we're about to spawn. @@ -43,7 +54,7 @@ def create_pipe_to_child_stdin() -> Tuple[SendStream, int]: raise NotImplementedError from _create_child_pipe_error # pragma: no cover -def create_pipe_from_child_output() -> Tuple[ReceiveStream, int]: +def create_pipe_from_child_output() -> Tuple["ClosableReceiveStream", int]: """Create a new pipe suitable for receiving data into this process from the standard output or error stream of a child we're about to spawn. diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index 0d2d11c53c..4b3637dc32 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -67,11 +67,10 @@ def _raw_close(self): def __del__(self): self._raw_close() - async def aclose(self): + def close(self): if not self.closed: trio.lowlevel.notify_closing(self.fd) self._raw_close() - await trio.lowlevel.checkpoint() class FdStream(Stream, metaclass=Final): @@ -180,8 +179,12 @@ async def receive_some(self, max_bytes=None) -> bytes: return data + def close(self): + self._fd_holder.close() + async def aclose(self): - await self._fd_holder.aclose() + self.close() + await trio.lowlevel.checkpoint() def fileno(self): return self._fd_holder.fd diff --git a/trio/_windows_pipes.py b/trio/_windows_pipes.py index fb420535f4..693792ba0e 100644 --- a/trio/_windows_pipes.py +++ b/trio/_windows_pipes.py @@ -25,7 +25,7 @@ def __init__(self, handle: int) -> None: def closed(self): return self.handle == -1 - def _close(self): + def close(self): if self.closed: return handle = self.handle @@ -33,12 +33,8 @@ def _close(self): if not kernel32.CloseHandle(_handle(handle)): raise_winerror() - async def aclose(self): - self._close() - await _core.checkpoint() - def __del__(self): - self._close() + self.close() class PipeSendStream(SendStream, metaclass=Final): @@ -78,8 +74,12 @@ async def wait_send_all_might_not_block(self) -> None: # not implemented yet, and probably not needed await _core.checkpoint() + def close(self): + self._handle_holder.close() + async def aclose(self): - await self._handle_holder.aclose() + self.close() + await _core.checkpoint() class PipeReceiveStream(ReceiveStream, metaclass=Final): @@ -130,5 +130,9 @@ async def receive_some(self, max_bytes=None) -> bytes: del buffer[size:] return buffer + def close(self): + self._handle_holder.close() + async def aclose(self): - await self._handle_holder.aclose() + self.close() + await _core.checkpoint() diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 28783303a9..9dd8c5171e 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -2,6 +2,8 @@ import signal import subprocess import sys +from pathlib import Path as SyncPath + import pytest import random from functools import partial @@ -529,7 +531,7 @@ def broken_terminate(self): nursery.cancel_scope.cancel() -@pytest.mark.skipif(os.name != "posix", reason="posix only") +@pytest.mark.skipif(not posix, reason="posix only") async def test_warn_on_cancel_SIGKILL_escalation(autojump_clock, monkeypatch): monkeypatch.setattr(Process, "terminate", lambda *args: None) @@ -547,3 +549,18 @@ async def test_run_process_background_fail(): async with _core.open_nursery() as nursery: proc = await nursery.start(run_process, EXIT_FALSE) assert proc.returncode == 1 + +@pytest.mark.skipif(not SyncPath("/dev/fd").exists(), + reason="requires a way to iterate through open files") +async def test_for_leaking_fds(): + starting_fds = set(SyncPath("/dev/fd").iterdir()) + await run_process(EXIT_TRUE) + assert set(SyncPath("/dev/fd").iterdir()) == starting_fds + + with pytest.raises(subprocess.CalledProcessError): + await run_process(EXIT_FALSE) + assert set(SyncPath("/dev/fd").iterdir()) == starting_fds + + with pytest.raises(PermissionError): + await run_process(["/dev/fd/0"]) + assert set(SyncPath("/dev/fd").iterdir()) == starting_fds From 45454bc238d074332fa2fc658e6149b3953f0566 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Thu, 9 Dec 2021 13:46:49 -0700 Subject: [PATCH 0877/1498] reformat --- trio/_subprocess_platform/__init__.py | 2 +- trio/tests/test_subprocess.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/trio/_subprocess_platform/__init__.py b/trio/_subprocess_platform/__init__.py index e03c2e5158..7a131e090c 100644 --- a/trio/_subprocess_platform/__init__.py +++ b/trio/_subprocess_platform/__init__.py @@ -19,11 +19,11 @@ class ClosableSendStream(SendStream): def close(self) -> None: ... - class ClosableReceiveStream(ReceiveStream): def close(self) -> None: ... + # Fallback versions of the functions provided -- implementations # per OS are imported atop these at the bottom of the module. async def wait_child_exiting(process: "_subprocess.Process") -> None: diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 9dd8c5171e..60befb606c 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -550,8 +550,11 @@ async def test_run_process_background_fail(): proc = await nursery.start(run_process, EXIT_FALSE) assert proc.returncode == 1 -@pytest.mark.skipif(not SyncPath("/dev/fd").exists(), - reason="requires a way to iterate through open files") + +@pytest.mark.skipif( + not SyncPath("/dev/fd").exists(), + reason="requires a way to iterate through open files", +) async def test_for_leaking_fds(): starting_fds = set(SyncPath("/dev/fd").iterdir()) await run_process(EXIT_TRUE) From d03fd1505956e329a0d03e77210e4c5c8b7d38cf Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Fri, 7 Jan 2022 13:19:37 -0700 Subject: [PATCH 0878/1498] Add towncrier note; restore towncrier README --- newsfragments/2193.bugfix.rst | 3 +++ newsfragments/README.rst | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 newsfragments/2193.bugfix.rst create mode 100644 newsfragments/README.rst diff --git a/newsfragments/2193.bugfix.rst b/newsfragments/2193.bugfix.rst new file mode 100644 index 0000000000..979dec111b --- /dev/null +++ b/newsfragments/2193.bugfix.rst @@ -0,0 +1,3 @@ +Trio now deterministically cleans up file descriptors that were opened before +subprocess creation fails. Previously, they would remain open until the next run of +the garbage collector. diff --git a/newsfragments/README.rst b/newsfragments/README.rst new file mode 100644 index 0000000000..349e67eec0 --- /dev/null +++ b/newsfragments/README.rst @@ -0,0 +1,35 @@ +This directory collects "newsfragments": short files that each contain +a snippet of ReST-formatted text that will be added to the next +release notes. This should be a description of aspects of the change +(if any) that are relevant to users. (This contrasts with your commit +message and PR description, which are a description of the change as +relevant to people working on the code itself.) + +Each file should be named like ``..rst``, where +```` is an issue number, and ```` is one of: + +* ``headline``: a major new feature we want to highlight for users +* ``breaking``: any breaking changes that happen without a proper + deprecation period (note: deprecations, and removal of previously + deprecated features after an appropriate time, go in the + ``deprecated`` category instead) +* ``feature``: any new feature that doesn't qualify for ``headline`` +* ``bugfix`` +* ``doc`` +* ``deprecated`` +* ``misc`` + +So for example: ``123.headline.rst``, ``456.bugfix.rst``, +``789.deprecated.rst`` + +If your PR fixes an issue, use that number here. If there is no issue, +then after you submit the PR and get the PR number you can add a +newsfragment using that instead. + +Your text can use all the same markup that we use in our Sphinx docs. +For example, you can use double-backticks to mark code snippets, or +single-backticks to link to a function/class/module. + +To check how your formatting looks, the easiest way is to make the PR, +and then after the CI checks run, click on the "Read the Docs build" +details link, and navigate to the release history page. From d38a40659c92db8e9220f46e1468e5ee81d4240d Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Mon, 6 Dec 2021 10:52:46 -0500 Subject: [PATCH 0879/1498] Improve stability of tests_subprocess On macOS, there appears to be some sort of race where sending SIGTERM to a process "too early" in its lifetime causes the exit status to appear as if SIGKILL was sent. I can reproduce this with the stdlib subprocess. Additionally, this problem doesn't occur if there is a tiny sleep between spawning a child subprocess and terminating it. My proposal is to use `/bin/sleep` on posix (which reliably does not have this issue on macOS) and continue using the Python form elsewhere. --- trio/tests/test_subprocess.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 60befb606c..45e6c035e4 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -17,7 +17,6 @@ sleep_forever, Process, run_process, - TrioDeprecationWarning, ClosedResourceError, ) from ..lowlevel import open_process @@ -41,7 +40,11 @@ def python(code): EXIT_TRUE = python("sys.exit(0)") EXIT_FALSE = python("sys.exit(1)") CAT = python("sys.stdout.buffer.write(sys.stdin.buffer.read())") -SLEEP = lambda seconds: python("import time; time.sleep({})".format(seconds)) + +if posix: + SLEEP = lambda seconds: ["/bin/sleep", str(seconds)] +else: + SLEEP = lambda seconds: python("import time; time.sleep({})".format(seconds)) def got_signal(proc, sig): From 643684ff30dd264d8496b83fbf46c26645f59aa4 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Wed, 12 Jan 2022 13:35:13 -0700 Subject: [PATCH 0880/1498] subprocess - Fix a bug where trio isn't notified about pidfd closes If you have 1. Task A blocks in `Process.wait` 2. The child process exits 3. Task B calls `Process.returncode` (3) will close the pidfd, but nothing will ever wake up Task A, since trio wasn't notified. I noticed this when I wrote a `deliver_cancel` function that was slow enough that the child actually exited before the call to `Process.returncode`. In the default `_posix_deliver_cancel`, everything is so fast that the usual ordering is that (3) comes before (2), and there isn't an early call to `_pidfd_close`. This fixes the bug by ensuring we notify trio about closing the pidfd when we do it! --- trio/_subprocess.py | 9 +++++++- trio/tests/test_subprocess.py | 41 ++++++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 154a940df8..aef7bcbedb 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING from ._abc import AsyncResource, SendStream, ReceiveStream +from ._core import ClosedResourceError from ._highlevel_generic import StapledStream from ._sync import Lock from ._subprocess_platform import ( @@ -217,6 +218,7 @@ async def aclose(self): def _close_pidfd(self): if self._pidfd is not None: + trio.lowlevel.notify_closing(self._pidfd.fileno()) self._pidfd.close() self._pidfd = None @@ -229,7 +231,12 @@ async def wait(self): async with self._wait_lock: if self.poll() is None: if self._pidfd is not None: - await trio.lowlevel.wait_readable(self._pidfd) + try: + await trio.lowlevel.wait_readable(self._pidfd) + except ClosedResourceError: + # something else (probably a call to poll) already closed the + # pidfd + pass else: await wait_child_exiting(self) # We have to use .wait() here, not .poll(), because on macOS diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 45e6c035e4..c2e17cf0da 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -1,27 +1,27 @@ import os +import random import signal import subprocess import sys +from functools import partial from pathlib import Path as SyncPath import pytest -import random -from functools import partial from async_generator import asynccontextmanager from .. import ( + ClosedResourceError, + Process, _core, - move_on_after, fail_after, + move_on_after, + run_process, sleep, sleep_forever, - Process, - run_process, - ClosedResourceError, ) +from .._core.tests.tutil import skip_if_fbsd_pipes_broken, slow from ..lowlevel import open_process -from .._core.tests.tutil import slow, skip_if_fbsd_pipes_broken -from ..testing import wait_all_tasks_blocked +from ..testing import assert_no_checkpoints, wait_all_tasks_blocked posix = os.name == "posix" if posix: @@ -570,3 +570,28 @@ async def test_for_leaking_fds(): with pytest.raises(PermissionError): await run_process(["/dev/fd/0"]) assert set(SyncPath("/dev/fd").iterdir()) == starting_fds + + +async def test_subprocess_pidfd_unnotified(): + noticed_exit = None + + async def wait_and_tell(proc) -> None: + nonlocal noticed_exit + noticed_exit = False + await proc.wait() + noticed_exit = True + + proc = await open_process(SLEEP(9999)) + async with _core.open_nursery() as nursery: + nursery.start_soon(wait_and_tell, proc) + await wait_all_tasks_blocked() + assert noticed_exit is False + proc.terminate() + # without giving trio a chance to do so, + with assert_no_checkpoints(): + # wait until the process has actually exited; + proc._proc.wait() + # force a call to poll (that closes the pidfd on linux) + proc.poll() + await wait_all_tasks_blocked() + assert noticed_exit, "child task wasn't woken after poll, DEADLOCK" From bb02d15fe9128b4f821c35fbc9f5d1d0b712a2f5 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Wed, 12 Jan 2022 14:29:07 -0700 Subject: [PATCH 0881/1498] Add news and comment --- newsfragments/2209.bugfix.rst | 3 +++ trio/tests/test_subprocess.py | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 newsfragments/2209.bugfix.rst diff --git a/newsfragments/2209.bugfix.rst b/newsfragments/2209.bugfix.rst new file mode 100644 index 0000000000..8ea6094f15 --- /dev/null +++ b/newsfragments/2209.bugfix.rst @@ -0,0 +1,3 @@ +Fix a bug that could cause `Process.wait` to hang on Linux systems using pidfds, if +another task were to access `Process.returncode` after the process exited but before +``wait`` woke up diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index c2e17cf0da..2627521380 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -572,6 +572,8 @@ async def test_for_leaking_fds(): assert set(SyncPath("/dev/fd").iterdir()) == starting_fds +# regression test for #2209 +@pytest.mark.skipif(not posix, reason="Regression test for Linux-only bug") async def test_subprocess_pidfd_unnotified(): noticed_exit = None From 18a850c186e23afa5d9ffdfcf3d4f536bb60ae77 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Wed, 12 Jan 2022 14:37:39 -0700 Subject: [PATCH 0882/1498] Use an event instead --- trio/tests/test_subprocess.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 2627521380..65abe13e50 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -11,6 +11,7 @@ from .. import ( ClosedResourceError, + Event, Process, _core, fail_after, @@ -573,21 +574,20 @@ async def test_for_leaking_fds(): # regression test for #2209 -@pytest.mark.skipif(not posix, reason="Regression test for Linux-only bug") async def test_subprocess_pidfd_unnotified(): noticed_exit = None async def wait_and_tell(proc) -> None: nonlocal noticed_exit - noticed_exit = False + noticed_exit = Event() await proc.wait() - noticed_exit = True + noticed_exit.set() proc = await open_process(SLEEP(9999)) async with _core.open_nursery() as nursery: nursery.start_soon(wait_and_tell, proc) await wait_all_tasks_blocked() - assert noticed_exit is False + assert isinstance(noticed_exit, Event) proc.terminate() # without giving trio a chance to do so, with assert_no_checkpoints(): @@ -595,5 +595,8 @@ async def wait_and_tell(proc) -> None: proc._proc.wait() # force a call to poll (that closes the pidfd on linux) proc.poll() - await wait_all_tasks_blocked() - assert noticed_exit, "child task wasn't woken after poll, DEADLOCK" + with move_on_after(5): + # Some platforms use threads to wait for exit, so it might take a bit + # for everything to notice + await noticed_exit.wait() + assert noticed_exit.is_set(), "child task wasn't woken after poll, DEADLOCK" From 39d01b268b1f354aa0f2290cdddd88fa8c6f6b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 24 Jan 2022 22:03:40 +0200 Subject: [PATCH 0883/1498] Drop Python 3.6 support (#2210) * Remove Python 3.6 IPython hack in CI script * Remove exc_key compatibility hack in MultiError * Remove KI test that is no longer required * Update a number of comments referring to Python 3.6 * Stop checking for warn_on_full_buffer (this is always available now) * Stop installing contextvars since we dropped 3.6 support * Remove TCP_NOTSENT_LOWAT definitions (they always exist in Python 3.7+) * Remove obsolete check for GenericMeta * Remove check for ssl.OP_NO_TLSv1_3 (this flag was introduced in Python 3.7 so it's always there now) Co-authored-by: Quentin Pradet --- .github/workflows/ci.yml | 8 +- README.rst | 6 +- ci.sh | 8 -- docs/source/index.rst | 2 +- docs/source/reference-io.rst | 5 +- docs/source/tutorial.rst | 5 +- pyproject.toml | 2 +- setup.py | 8 +- test-requirements.in | 6 -- trio/_core/_entry_queue.py | 4 +- trio/_core/_multierror.py | 9 +-- trio/_core/_run.py | 18 +---- trio/_core/_wakeup_socketpair.py | 33 +------- trio/_core/tests/test_guest_mode.py | 2 +- trio/_core/tests/test_ki.py | 119 ---------------------------- trio/_core/tests/test_multierror.py | 8 +- trio/_core/tests/test_run.py | 2 +- trio/_highlevel_ssl_helpers.py | 1 - trio/_socket.py | 2 +- trio/_util.py | 16 +--- trio/socket.py | 12 --- trio/tests/test_ssl.py | 26 ++---- trio/tests/test_threads.py | 4 +- 23 files changed, 39 insertions(+), 267 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 606e64977b..65fa899179 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10'] arch: ['x86', 'x64'] lsp: [''] lsp_extract_file: [''] @@ -51,7 +51,7 @@ jobs: # This avoids the need to update for each new alpha, beta, release candidate, # and then finally an actual release version. actions/setup-python doesn't # support this for PyPy presently so we get no help there. - # + # # CPython -> 3.9.0-alpha - 3.9.X # PyPy -> pypy-3.7 python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} @@ -72,7 +72,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.6', 'pypy-3.7', 'pypy-3.8', '3.6', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] + python: ['pypy-3.7', 'pypy-3.8', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] @@ -114,7 +114,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10'] include: - python: '3.8' # <- not actually used arch: 'x64' diff --git a/README.rst b/README.rst index c4b84c1711..4e096eddf3 100644 --- a/README.rst +++ b/README.rst @@ -9,14 +9,14 @@ .. image:: https://img.shields.io/badge/docs-read%20now-blue.svg :target: https://trio.readthedocs.io :alt: Documentation - + .. image:: https://img.shields.io/pypi/v/trio.svg :target: https://pypi.org/project/trio :alt: Latest PyPi version .. image:: https://img.shields.io/conda/vn/conda-forge/trio.svg :target: https://anaconda.org/conda-forge/trio - :alt: Latest conda-forge version + :alt: Latest conda-forge version .. image:: https://codecov.io/gh/python-trio/trio/branch/master/graph/badge.svg :target: https://codecov.io/gh/python-trio/trio @@ -92,7 +92,7 @@ demonstration of implementing the "Happy Eyeballs" algorithm in an older library versus Trio. **Cool, but will it work on my system?** Probably! As long as you have -some kind of Python 3.6-or-better (CPython or the latest PyPy3 are +some kind of Python 3.7-or-better (CPython or the latest PyPy3 are both fine), and are using Linux, macOS, Windows, or FreeBSD, then Trio will work. Other environments might work too, but those are the ones we test on. And all of our dependencies are pure Python, diff --git a/ci.sh b/ci.sh index b4f56f7114..d4f9df3a94 100755 --- a/ci.sh +++ b/ci.sh @@ -72,14 +72,6 @@ python -m pip --version python setup.py sdist --formats=zip python -m pip install dist/*.zip -if python -c 'import sys; sys.exit(sys.version_info >= (3, 7))'; then - # Python < 3.7, select last ipython with 3.6 support - # macOS requires the suffix for --in-place or you get an undefined label error - sed -i'.bak' 's/ipython==[^ ]*/ipython==7.16.1/' test-requirements.txt - sed -i'.bak' 's/traitlets==[^ ]*/traitlets==4.3.3/' test-requirements.txt - git diff test-requirements.txt -fi - if [ "$CHECK_FORMATTING" = "1" ]; then python -m pip install -r test-requirements.txt source check.sh diff --git a/docs/source/index.rst b/docs/source/index.rst index e3adddaee8..84d81880af 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -45,7 +45,7 @@ Vital statistics: * Supported environments: We test on - - Python: 3.6+ (CPython and PyPy) + - Python: 3.7+ (CPython and PyPy) - Windows, macOS, Linux (glibc and musl), FreeBSD Other environments might also work; give it a try and see. diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index 4d33601f17..b18983e272 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -237,8 +237,7 @@ other constants and functions in the :mod:`ssl` module. .. warning:: Avoid instantiating :class:`ssl.SSLContext` directly. A newly constructed :class:`~ssl.SSLContext` has less secure - defaults than one returned by :func:`ssl.create_default_context`, - dramatically so before Python 3.6. + defaults than one returned by :func:`ssl.create_default_context`. Instead of using :meth:`ssl.SSLContext.wrap_socket`, you create a :class:`SSLStream`: @@ -722,7 +721,7 @@ subprocess`` in order to access constants such as ``PIPE`` or Currently, Trio always uses unbuffered byte streams for communicating with a process, so it does not support the ``encoding``, ``errors``, -``universal_newlines`` (alias ``text`` in 3.7+), and ``bufsize`` +``universal_newlines`` (alias ``text``), and ``bufsize`` options. diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index aedba8244a..08206569f5 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -34,9 +34,6 @@ Tutorial print(response) and then again with /delay/10 - (note that asks needs cpython 3.6 though. maybe just for one async - generator?) - value of async/await: show you where the cancellation exceptions can happen -- see pillar re: explicit cancel points @@ -94,7 +91,7 @@ Okay, ready? Let's get started. Before you begin ---------------- -1. Make sure you're using Python 3.6 or newer. +1. Make sure you're using Python 3.7 or newer. 2. ``python3 -m pip install --upgrade trio`` (or on Windows, maybe ``py -3 -m pip install --upgrade trio`` – `details diff --git a/pyproject.toml b/pyproject.toml index 1378e5df7b..944440661f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.black] -target-version = ['py36'] +target-version = ['py37'] [tool.towncrier] diff --git a/setup.py b/setup.py index 11eda8e96f..732fa7acd7 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ Vital statistics: * Supported environments: Linux, macOS, or Windows running some kind of Python - 3.6-or-better (either CPython or PyPy3 is fine). \\*BSD and illumos likely + 3.7-or-better (either CPython or PyPy3 is fine). \\*BSD and illumos likely work too, but are not tested. * Install: ``python3 -m pip install -U trio`` (or on Windows, maybe @@ -89,12 +89,11 @@ # cffi 1.14 fixes memory leak inside ffi.getwinerror() # cffi is required on Windows, except on PyPy where it is built-in "cffi>=1.14; os_name == 'nt' and implementation_name != 'pypy'", - "contextvars>=2.1; python_version < '3.7'", ], # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: include_package_data=True, - python_requires=">=3.6", + python_requires=">=3.7", keywords=["async", "io", "networking", "trio"], classifiers=[ "Development Status :: 3 - Alpha", @@ -108,10 +107,11 @@ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: System :: Networking", "Framework :: Trio", ], diff --git a/test-requirements.in b/test-requirements.in index 894fc654d3..99854204d6 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -22,15 +22,9 @@ typing-extensions; implementation_name == "cpython" # Trio's own dependencies cffi; os_name == "nt" -contextvars; python_version < "3.7" attrs >= 19.2.0 sortedcontainers async_generator >= 1.9 idna outcome sniffio - -# Required by contextvars, but harmless to install everywhere. -# dependabot drops the contextvars dependency because it runs -# on 3.7. -immutables >= 0.6 diff --git a/trio/_core/_entry_queue.py b/trio/_core/_entry_queue.py index 8f6eb05e3b..9f3301b3d2 100644 --- a/trio/_core/_entry_queue.py +++ b/trio/_core/_entry_queue.py @@ -15,8 +15,8 @@ class EntryQueue: # not signal-safe. deque is implemented in C, so each operation is atomic # WRT threads (and this is guaranteed in the docs), AND each operation is # atomic WRT signal delivery (signal handlers can run on either side, but - # not *during* a deque operation). dict makes similar guarantees - and on - # CPython 3.6 and PyPy, it's even ordered! + # not *during* a deque operation). dict makes similar guarantees - and + # it's even ordered! queue = attr.ib(factory=deque) idempotent_queue = attr.ib(factory=dict) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index b6bf2cf727..514f8764c8 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -5,13 +5,6 @@ import attr -# python traceback.TracebackException < 3.6.4 does not support unhashable exceptions -# see https://github.com/python/cpython/pull/4014 for details -if sys.version_info < (3, 6, 4): - exc_key = lambda exc: exc -else: - exc_key = id - ################################################################ # MultiError ################################################################ @@ -419,7 +412,7 @@ def traceback_exception_init( if isinstance(exc_value, MultiError): embedded = [] for exc in exc_value.exceptions: - if exc_key(exc) not in _seen: + if id(exc) not in _seen: embedded.append( traceback.TracebackException.from_exception( exc, diff --git a/trio/_core/_run.py b/trio/_core/_run.py index fcb36c32c5..f3adfc7580 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1,18 +1,12 @@ -# coding: utf-8 - import functools import itertools -import logging -import os import random import select import sys import threading from collections import deque -import collections.abc from contextlib import contextmanager import warnings -import weakref import enum from contextvars import copy_context @@ -47,7 +41,6 @@ from ._thread_cache import start_thread_soon from ._instrumentation import Instruments from .. import _core -from .._deprecate import warn_deprecated from .._util import Final, NoPublicConstructor, coroutine_or_error DEADLINE_HEAP_MIN_PRUNE_THRESHOLD = 1000 @@ -70,13 +63,8 @@ def _public(fn): _r = random.Random() -# On 3.7+, Context.run() is implemented in C and doesn't show up in -# tracebacks. On 3.6, we use the contextvars backport, which is -# currently implemented in Python and adds 1 frame to tracebacks. So this -# function is a super-overkill version of "0 if sys.version_info >= (3, 7) -# else 1". But if Context.run ever changes, we'll be ready! -# -# This can all be removed once we drop support for 3.6. +# On CPython, Context.run() is implemented in C and doesn't show up in +# tracebacks. On PyPy, it is implemented in Python and adds 1 frame to tracebacks. def _count_context_run_tb_frames(): def function_with_unique_name_xyzzy(): 1 / 0 @@ -2172,7 +2160,7 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): try: # We used to unwrap the Outcome object here and send/throw # its contents in directly, but it turns out that .throw() - # is buggy, at least on CPython 3.6: + # is buggy, at least before CPython 3.9: # https://bugs.python.org/issue29587 # https://bugs.python.org/issue29590 # So now we send in the Outcome object and unwrap it on the diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 121cec584e..8d51419ecc 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -1,5 +1,4 @@ import socket -import sys import signal import warnings @@ -7,24 +6,6 @@ from .._util import is_main_thread -def _has_warn_on_full_buffer(): - if sys.version_info < (3, 7): - return False - - if "__pypy__" not in sys.builtin_module_names: - # CPython has warn_on_full_buffer. Don't need to inspect. - # Also, CPython doesn't support inspecting built-in functions. - return True - - import inspect - - args_spec = inspect.getfullargspec(signal.set_wakeup_fd) - return "warn_on_full_buffer" in args_spec.kwonlyargs - - -HAVE_WARN_ON_FULL_BUFFER = _has_warn_on_full_buffer() - - class WakeupSocketpair: def __init__(self): self.wakeup_sock, self.write_sock = socket.socketpair() @@ -38,13 +19,8 @@ def __init__(self): # Windows 10: 525347 # Windows you're weird. (And on Windows setting SNDBUF to 0 makes send # blocking, even on non-blocking sockets, so don't do that.) - # - # But, if we're on an old Python and can't control the signal module's - # warn-on-full-buffer behavior, then we need to leave things alone, so - # the signal module won't spam the console with spurious warnings. - if HAVE_WARN_ON_FULL_BUFFER: - self.wakeup_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1) - self.write_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1) + self.wakeup_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1) + self.write_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1) # On Windows this is a TCP socket so this might matter. On other # platforms this fails b/c AF_UNIX sockets aren't actually TCP. try: @@ -75,10 +51,7 @@ def wakeup_on_signals(self): if not is_main_thread(): return fd = self.write_sock.fileno() - if HAVE_WARN_ON_FULL_BUFFER: - self.old_wakeup_fd = signal.set_wakeup_fd(fd, warn_on_full_buffer=False) - else: - self.old_wakeup_fd = signal.set_wakeup_fd(fd) + self.old_wakeup_fd = signal.set_wakeup_fd(fd, warn_on_full_buffer=False) if self.old_wakeup_fd != -1: warnings.warn( RuntimeWarning( diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index b06849a836..564b65d88a 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -504,7 +504,7 @@ async def trio_main(in_host): @pytest.mark.skipif(buggy_pypy_asyncgens, reason="PyPy 7.2 is buggy") @pytest.mark.xfail( - sys.implementation.name == "pypy" and sys.version_info >= (3, 7), + sys.implementation.name == "pypy", reason="async generator issue under investigation", ) @restore_unraisablehook() diff --git a/trio/_core/tests/test_ki.py b/trio/_core/tests/test_ki.py index 0e4db4af49..430352ab6d 100644 --- a/trio/_core/tests/test_ki.py +++ b/trio/_core/tests/test_ki.py @@ -499,122 +499,3 @@ async def inner(): _core.run(inner) finally: threading._active[thread.ident] = original - - -# For details on why this test is non-trivial, see: -# https://github.com/python-trio/trio/issues/42 -# https://github.com/python-trio/trio/issues/109 -@slow -def test_ki_wakes_us_up(): - assert is_main_thread() - - # This test is flaky due to a race condition on Windows; see: - # https://github.com/python-trio/trio/issues/119 - # https://bugs.python.org/issue30038 - # I think the only fix is to wait for fixed CPython to be released, so in - # the mean time, on affected versions we send two signals (equivalent to - # hitting control-C twice). This works because the problem is that the C - # level signal handler does - # - # write-to-fd -> set-flags - # - # and we need - # - # set-flags -> write-to-fd - # - # so running the C level signal handler twice does - # - # write-to-fd -> set-flags -> write-to-fd -> set-flags - # - # which contains the desired sequence. - # - # Affected version of CPython include 3.6.1 and earlier. - # It's fixed in 3.6.2 and 3.7+ - # - # PyPy was never affected. - # - # The problem technically can occur on Unix as well, if a signal is - # delivered to a non-main thread, though we haven't observed this in - # practice. - # - # There's also this theoretical problem, but hopefully it won't actually - # bite us in practice: - # https://bugs.python.org/issue31119 - # https://bitbucket.org/pypy/pypy/issues/2623 - import platform - - buggy_wakeup_fd = ( - sys.version_info < (3, 6, 2) and platform.python_implementation() == "CPython" - ) - - # lock is only needed to avoid an annoying race condition where the - # *second* ki_self() call arrives *after* the first one woke us up and its - # KeyboardInterrupt was caught, and then generates a second - # KeyboardInterrupt that aborts the test run. The kill_soon thread holds - # the lock while doing the calls to ki_self, which means that it holds it - # while the C-level signal handler is running. Then in the main thread, - # when we're woken up we know that ki_self() has been run at least once; - # if we then take the lock it guaranteeds that ki_self() has been run - # twice, so if a second KeyboardInterrupt is going to arrive it should - # arrive by the time we've acquired the lock. This lets us force it to - # happen inside the pytest.raises block. - # - # It will be very nice when the buggy_wakeup_fd bug is fixed. - lock = threading.Lock() - - def kill_soon(): - # We want the signal to be raised after the main thread has entered - # the IO manager blocking primitive. There really is no way to - # deterministically interlock with that, so we have to use sleep and - # hope it's long enough. - time.sleep(1.1) - with lock: - print("thread doing ki_self()") - ki_self() - if buggy_wakeup_fd: - print("buggy_wakeup_fd =", buggy_wakeup_fd) - ki_self() - - async def main(): - thread = threading.Thread(target=kill_soon) - print("Starting thread") - thread.start() - try: - with pytest.raises(KeyboardInterrupt): - # To limit the damage on CI if this does get broken (as - # compared to sleep_forever()) - print("Going to sleep") - try: - await sleep(20) - print("Woke without raising?!") # pragma: no cover - # The only purpose of this finally: block is to soak up the - # second KeyboardInterrupt that might arrive on - # buggy_wakeup_fd platforms. So it might get aborted at any - # moment randomly on some runs, so pragma: no cover avoids - # coverage flapping: - finally: # pragma: no cover - print("waiting for lock") - with lock: - print("got lock") - # And then we want to force a PyErr_CheckSignals. Which is - # not so easy on Windows. Weird kluge: builtin_repr calls - # PyObject_Repr, which does an unconditional - # PyErr_CheckSignals for some reason. - print(repr(None)) - # And finally, it's possible that the signal was delivered - # but at a moment when we had KI protection enabled, so we - # need to execute a checkpoint to ensure it's delivered - # before we exit main(). - await _core.checkpoint() - finally: - print("joining thread", sys.exc_info()) - thread.join() - - start = time.perf_counter() - try: - _core.run(main) - finally: - end = time.perf_counter() - print("duration", end - start) - print("sys.exc_info", sys.exc_info()) - assert 1.0 <= (end - start) < 2 diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 70d763e652..e4a30e0e69 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -162,13 +162,7 @@ def test_traceback_recursion(): # This could trigger an infinite recursion; the 'seen' set is supposed to prevent # this. exc1.__cause__ = MultiError([exc1, exc2, exc3]) - # python traceback.TracebackException < 3.6.4 does not support unhashable exceptions - # and raises a TypeError exception - if sys.version_info < (3, 6, 4): - with pytest.raises(TypeError): - format_exception(*einfo(exc1)) - else: - format_exception(*einfo(exc1)) + format_exception(*einfo(exc1)) def make_tree(): diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index a559ac11ed..863dad9909 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1056,7 +1056,7 @@ async def child2(): ] -# At least as of CPython 3.6, using .throw() to raise an exception inside a +# Before CPython 3.9, using .throw() to raise an exception inside a # coroutine/generator causes the original exc_info state to be lost, so things # like re-raising and exception chaining are broken. # diff --git a/trio/_highlevel_ssl_helpers.py b/trio/_highlevel_ssl_helpers.py index a339a3d238..19b1ff8777 100644 --- a/trio/_highlevel_ssl_helpers.py +++ b/trio/_highlevel_ssl_helpers.py @@ -19,7 +19,6 @@ async def open_ssl_over_tcp_stream( *, https_compatible=False, ssl_context=None, - # No trailing comma b/c bpo-9232 (fixed in py36) happy_eyeballs_delay=DEFAULT_DELAY, ): """Make a TLS-encrypted Connection to the given host and port over TCP. diff --git a/trio/_socket.py b/trio/_socket.py index 4e4e603726..886f5614f6 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -50,7 +50,7 @@ async def __aexit__(self, etype, value, tb): try: from socket import IPPROTO_IPV6 except ImportError: - # As of at least 3.6, python on Windows is missing IPPROTO_IPV6 + # Before Python 3.8, Windows is missing IPPROTO_IPV6 # https://bugs.python.org/issue29515 if sys.platform == "win32": # pragma: no branch IPPROTO_IPV6 = 41 diff --git a/trio/_util.py b/trio/_util.py index ec0350b305..b5a2ceabdc 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -269,21 +269,7 @@ def __getitem__(self, _): return self -# If a new class inherits from any ABC, then the new class's metaclass has to -# inherit from ABCMeta. If a new class inherits from typing.Generic, and -# you're using Python 3.6, then the new class's metaclass has to -# inherit from typing.GenericMeta. Some of the classes that want to use Final -# or NoPublicConstructor inherit from ABCs and generics, so Final has to -# inherit from these metaclasses. Fortunately, GenericMeta inherits from -# ABCMeta, so inheriting from GenericMeta alone is sufficient (when it -# exists at all). -if not t.TYPE_CHECKING and hasattr(t, "GenericMeta"): - BaseMeta = t.GenericMeta -else: - BaseMeta = ABCMeta - - -class Final(BaseMeta): +class Final(ABCMeta): """Metaclass that enforces a class to be final (i.e., subclass not allowed). If a class uses this metaclass like this:: diff --git a/trio/socket.py b/trio/socket.py index 27e75c8dbc..613375ef41 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -188,18 +188,6 @@ # get names used by Trio that we define on our own from ._socket import IPPROTO_IPV6 -# Not defined in all python versions and platforms but sometimes needed -if not _t.TYPE_CHECKING: - try: - TCP_NOTSENT_LOWAT - except NameError: - # Hopefully will show up in 3.7: - # https://github.com/python/cpython/pull/477 - if sys.platform == "darwin": - TCP_NOTSENT_LOWAT = 0x201 - elif sys.platform == "linux": - TCP_NOTSENT_LOWAT = 25 - if _t.TYPE_CHECKING: IP_BIND_ADDRESS_NO_PORT: int else: diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index f3bff42195..10e6c13d8f 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -60,23 +60,14 @@ TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) + # TLS 1.3 has a lot of changes from previous versions. So we want to run tests # with both TLS 1.3, and TLS 1.2. -if hasattr(ssl, "OP_NO_TLSv1_3"): - # "tls13" means that we're willing to negotiate TLS 1.3. Usually that's - # what will happen, but the renegotiation tests explicitly force a - # downgrade on the server side. "tls12" means we refuse to negotiate TLS - # 1.3, so we'll almost certainly use TLS 1.2. - client_ctx_params = ["tls13", "tls12"] -else: - # We can't control whether we use TLS 1.3, so we just have to accept - # whatever openssl wants to use. This might be TLS 1.2 (if openssl is - # old), or it might be TLS 1.3 (if openssl is new, but our python version - # is too old to expose the configuration knobs). - client_ctx_params = ["default"] - - -@pytest.fixture(scope="module", params=client_ctx_params) +# "tls13" means that we're willing to negotiate TLS 1.3. Usually that's +# what will happen, but the renegotiation tests explicitly force a +# downgrade on the server side. "tls12" means we refuse to negotiate TLS +# 1.3, so we'll almost certainly use TLS 1.2. +@pytest.fixture(scope="module", params=["tls13", "tls12"]) def client_ctx(request): ctx = ssl.create_default_context() @@ -87,10 +78,7 @@ def client_ctx(request): if request.param in ["default", "tls13"]: return ctx elif request.param == "tls12": - if sys.version_info >= (3, 7): - ctx.maximum_version = ssl.TLSVersion.TLSv1_2 - else: - ctx.options |= ssl.OP_NO_TLSv1_3 + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 return ctx else: # pragma: no cover assert False diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 47727817e6..11fc34c131 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -300,8 +300,8 @@ async def test_run_in_worker_thread_limiter(MAX, cancel, use_default_limiter): try: # We used to use regular variables and 'nonlocal' here, but it turns # out that it's not safe to assign to closed-over variables that are - # visible in multiple threads, at least as of CPython 3.6 and PyPy - # 5.8: + # visible in multiple threads, at least as of CPython 3.10 and PyPy + # 7.3: # # https://bugs.python.org/issue30744 # https://bitbucket.org/pypy/pypy/issues/2591/ From 40ee23acba36bf7958af8c6cfa79a95a49c64e51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 20:05:23 +0000 Subject: [PATCH 0884/1498] Bump pylint from 2.11.1 to 2.12.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.11.1 to 2.12.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.11.1...v2.12.2) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index f1162efdb0..c46460852a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.8.4 +astroid==2.9.3 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -38,14 +38,6 @@ idna==3.3 # via # -r test-requirements.in # trustme -immutables==0.16 - # via -r test-requirements.in -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.29.0 @@ -103,7 +95,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.11.1 +pylint==2.12.2 # via -r test-requirements.in pyopenssl==21.0.0 # via -r test-requirements.in @@ -138,27 +130,15 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy typing-extensions==3.10.0.2 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black - # immutables - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -zipp==3.6.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 31706e818ac041781d39c6b51efe65787e915e1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 25 Jan 2022 11:18:07 +0200 Subject: [PATCH 0885/1498] Add news fragment for #2210 --- newsfragments/2210.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2210.removal.rst diff --git a/newsfragments/2210.removal.rst b/newsfragments/2210.removal.rst new file mode 100644 index 0000000000..aa7c74034c --- /dev/null +++ b/newsfragments/2210.removal.rst @@ -0,0 +1 @@ +Remove support for Python 3.6. \ No newline at end of file From 21cd26f91be458566ca16e3f38cefc481e1433da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 25 Jan 2022 11:53:31 +0200 Subject: [PATCH 0886/1498] Add news fragment for #2210 (#2219) --- newsfragments/2210.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2210.removal.rst diff --git a/newsfragments/2210.removal.rst b/newsfragments/2210.removal.rst new file mode 100644 index 0000000000..aa7c74034c --- /dev/null +++ b/newsfragments/2210.removal.rst @@ -0,0 +1 @@ +Remove support for Python 3.6. \ No newline at end of file From 743354b02b60d49b23fbfef3ddef40729cd40fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 16 Jan 2022 00:12:03 +0200 Subject: [PATCH 0887/1498] Replace MultiError with (Base)ExceptionGroup Closes #2211. --- docs/source/design.rst | 3 - docs/source/reference-core.rst | 109 +-- docs/source/tutorial.rst | 8 +- newsfragments/2211.removal.rst | 4 + trio/__init__.py | 1 - trio/_core/__init__.py | 2 - trio/_core/_multierror.py | 517 ------------ trio/_core/_run.py | 27 +- trio/_core/tests/test_multierror.py | 772 ------------------ .../tests/test_multierror_scripts/__init__.py | 2 - .../tests/test_multierror_scripts/_common.py | 7 - .../apport_excepthook.py | 13 - .../custom_excepthook.py | 18 - .../ipython_custom_exc.py | 36 - .../simple_excepthook.py | 21 - .../simple_excepthook_IPython.py | 7 - .../simple_excepthook_partial.py | 13 - trio/_core/tests/test_run.py | 56 +- trio/_highlevel_open_tcp_listeners.py | 7 +- trio/_highlevel_open_tcp_stream.py | 14 +- .../test_highlevel_open_tcp_listeners.py | 7 +- trio/tests/test_highlevel_open_tcp_stream.py | 7 +- 22 files changed, 81 insertions(+), 1570 deletions(-) create mode 100644 newsfragments/2211.removal.rst delete mode 100644 trio/_core/_multierror.py delete mode 100644 trio/_core/tests/test_multierror.py delete mode 100644 trio/_core/tests/test_multierror_scripts/__init__.py delete mode 100644 trio/_core/tests/test_multierror_scripts/_common.py delete mode 100644 trio/_core/tests/test_multierror_scripts/apport_excepthook.py delete mode 100644 trio/_core/tests/test_multierror_scripts/custom_excepthook.py delete mode 100644 trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py delete mode 100644 trio/_core/tests/test_multierror_scripts/simple_excepthook.py delete mode 100644 trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py delete mode 100644 trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py diff --git a/docs/source/design.rst b/docs/source/design.rst index e0a8f939b7..25647fe642 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -465,9 +465,6 @@ There are two notable sub-modules that are largely independent of the rest of Trio, and could (possibly should?) be extracted into their own independent packages: -* ``_multierror.py``: Implements :class:`MultiError` and associated - infrastructure. - * ``_ki.py``: Implements the core infrastructure for safe handling of :class:`KeyboardInterrupt`. diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 514b5072c3..e320088cf8 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -641,7 +641,7 @@ crucial things to keep in mind: * Any unhandled exceptions are re-raised inside the parent task. If there are multiple exceptions, then they're collected up into a - single :exc:`MultiError` exception. + single :exc:`BaseExceptionGroup` or :exc:`ExceptionGroup` exception. Since all tasks are descendents of the initial task, one consequence of this is that :func:`run` can't finish until all tasks have @@ -712,14 +712,9 @@ limitation. Consider code like:: what? In some sense, the answer should be "both of these at once", but in Python there can only be one exception at a time. -Trio's answer is that it raises a :exc:`MultiError` object. This is a +Trio's answer is that it raises a :exc:`BaseExceptionGroup` object. This is a special exception which encapsulates multiple exception objects – -either regular exceptions or nested :exc:`MultiError`\s. To make these -easier to work with, Trio installs a custom `sys.excepthook` that -knows how to print nice tracebacks for unhandled :exc:`MultiError`\s, -and it also provides some helpful utilities like -:meth:`MultiError.catch`, which allows you to catch "part of" a -:exc:`MultiError`. +either regular exceptions or nested :exc:`BaseExceptionGroup`\s. Spawning tasks without becoming a parent @@ -837,104 +832,6 @@ The nursery API See :meth:`~Nursery.start`. -Working with :exc:`MultiError`\s -++++++++++++++++++++++++++++++++ - -.. autoexception:: MultiError - - .. attribute:: exceptions - - The list of exception objects that this :exc:`MultiError` - represents. - - .. automethod:: filter - - .. automethod:: catch - :with: - -Examples: - -Suppose we have a handler function that discards :exc:`ValueError`\s:: - - def handle_ValueError(exc): - if isinstance(exc, ValueError): - return None - else: - return exc - -Then these both raise :exc:`KeyError`:: - - with MultiError.catch(handle_ValueError): - raise MultiError([KeyError(), ValueError()]) - - with MultiError.catch(handle_ValueError): - raise MultiError([ - ValueError(), - MultiError([KeyError(), ValueError()]), - ]) - -And both of these raise nothing at all:: - - with MultiError.catch(handle_ValueError): - raise MultiError([ValueError(), ValueError()]) - - with MultiError.catch(handle_ValueError): - raise MultiError([ - MultiError([ValueError(), ValueError()]), - ValueError(), - ]) - -You can also return a new or modified exception, for example:: - - def convert_ValueError_to_MyCustomError(exc): - if isinstance(exc, ValueError): - # Similar to 'raise MyCustomError from exc' - new_exc = MyCustomError(...) - new_exc.__cause__ = exc - return new_exc - else: - return exc - -In the example above, we set ``__cause__`` as a form of explicit -context chaining. :meth:`MultiError.filter` and -:meth:`MultiError.catch` also perform implicit exception chaining – if -you return a new exception object, then the new object's -``__context__`` attribute will automatically be set to the original -exception. - -We also monkey patch :class:`traceback.TracebackException` to be able -to handle formatting :exc:`MultiError`\s. This means that anything that -formats exception messages like :mod:`logging` will work out of the -box:: - - import logging - - logging.basicConfig() - - try: - raise MultiError([ValueError("foo"), KeyError("bar")]) - except: - logging.exception("Oh no!") - raise - -Will properly log the inner exceptions: - -.. code-block:: none - - ERROR:root:Oh no! - Traceback (most recent call last): - File "", line 2, in - trio.MultiError: ValueError('foo',), KeyError('bar',) - - Details of embedded exception 1: - - ValueError: foo - - Details of embedded exception 2: - - KeyError: 'bar' - - .. _task-local-storage: Task-local storage diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 08206569f5..19289ca991 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -13,14 +13,13 @@ Tutorial still probably read this, because Trio is different.) Trio turns Python into a concurrent language. It takes the core - async/await syntax introduced in 3.5, and uses it to add three + async/await syntax introduced in 3.5, and uses it to add two new pieces of semantics: - cancel scopes: a generic system for managing timeouts and cancellation - nurseries: which let your program do multiple things at the same time - - MultiErrors: for when multiple things go wrong at once Of course it also provides a complete suite of APIs for doing networking, file I/O, using worker threads, @@ -57,8 +56,6 @@ Tutorial and demonstrate start() then point out that you can just use serve_tcp() - exceptions and MultiError - example: catch-all logging in our echo server review of the three (or four) core language extensions @@ -1149,9 +1146,6 @@ TODO: explain :exc:`Cancelled` TODO: explain how cancellation is also used when one child raises an exception -TODO: show an example :exc:`MultiError` traceback and walk through its -structure - TODO: maybe a brief discussion of :exc:`KeyboardInterrupt` handling? .. diff --git a/newsfragments/2211.removal.rst b/newsfragments/2211.removal.rst new file mode 100644 index 0000000000..728ac2f2ff --- /dev/null +++ b/newsfragments/2211.removal.rst @@ -0,0 +1,4 @@ +``trio.MultiError`` has been removed in favor of the built-in ``BaseExceptionGroup`` +(and its derivative ``ExceptionGroup``), falling back to the backport_ on Python < 3.11. + +.. _backport: https://pypi.org/project/exceptiongroup/ diff --git a/trio/__init__.py b/trio/__init__.py index a50ec33310..528ea15f08 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -22,7 +22,6 @@ Cancelled, BusyResourceError, ClosedResourceError, - MultiError, run, open_nursery, CancelScope, diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 2bd0c74e67..8e3e526cfe 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -17,8 +17,6 @@ EndOfChannel, ) -from ._multierror import MultiError - from ._ki import ( enable_ki_protection, disable_ki_protection, diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py deleted file mode 100644 index 514f8764c8..0000000000 --- a/trio/_core/_multierror.py +++ /dev/null @@ -1,517 +0,0 @@ -import sys -import traceback -import textwrap -import warnings - -import attr - -################################################################ -# MultiError -################################################################ - - -def _filter_impl(handler, root_exc): - # We have a tree of MultiError's, like: - # - # MultiError([ - # ValueError, - # MultiError([ - # KeyError, - # ValueError, - # ]), - # ]) - # - # or similar. - # - # We want to - # 1) apply the filter to each of the leaf exceptions -- each leaf - # might stay the same, be replaced (with the original exception - # potentially sticking around as __context__ or __cause__), or - # disappear altogether. - # 2) simplify the resulting tree -- remove empty nodes, and replace - # singleton MultiError's with their contents, e.g.: - # MultiError([KeyError]) -> KeyError - # (This can happen recursively, e.g. if the two ValueErrors above - # get caught then we'll just be left with a bare KeyError.) - # 3) preserve sensible tracebacks - # - # It's the tracebacks that are most confusing. As a MultiError - # propagates through the stack, it accumulates traceback frames, but - # the exceptions inside it don't. Semantically, the traceback for a - # leaf exception is the concatenation the tracebacks of all the - # exceptions you see when traversing the exception tree from the root - # to that leaf. Our correctness invariant is that this concatenated - # traceback should be the same before and after. - # - # The easy way to do that would be to, at the beginning of this - # function, "push" all tracebacks down to the leafs, so all the - # MultiErrors have __traceback__=None, and all the leafs have complete - # tracebacks. But whenever possible, we'd actually prefer to keep - # tracebacks as high up in the tree as possible, because this lets us - # keep only a single copy of the common parts of these exception's - # tracebacks. This is cheaper (in memory + time -- tracebacks are - # unpleasantly quadratic-ish to work with, and this might matter if - # you have thousands of exceptions, which can happen e.g. after - # cancelling a large task pool, and no-one will ever look at their - # tracebacks!), and more importantly, factoring out redundant parts of - # the tracebacks makes them more readable if/when users do see them. - # - # So instead our strategy is: - # - first go through and construct the new tree, preserving any - # unchanged subtrees - # - then go through the original tree (!) and push tracebacks down - # until either we hit a leaf, or we hit a subtree which was - # preserved in the new tree. - - # This used to also support async handler functions. But that runs into: - # https://bugs.python.org/issue29600 - # which is difficult to fix on our end. - - # Filters a subtree, ignoring tracebacks, while keeping a record of - # which MultiErrors were preserved unchanged - def filter_tree(exc, preserved): - if isinstance(exc, MultiError): - new_exceptions = [] - changed = False - for child_exc in exc.exceptions: - new_child_exc = filter_tree(child_exc, preserved) - if new_child_exc is not child_exc: - changed = True - if new_child_exc is not None: - new_exceptions.append(new_child_exc) - if not new_exceptions: - return None - elif changed: - return MultiError(new_exceptions) - else: - preserved.add(id(exc)) - return exc - else: - new_exc = handler(exc) - # Our version of implicit exception chaining - if new_exc is not None and new_exc is not exc: - new_exc.__context__ = exc - return new_exc - - def push_tb_down(tb, exc, preserved): - if id(exc) in preserved: - return - new_tb = concat_tb(tb, exc.__traceback__) - if isinstance(exc, MultiError): - for child_exc in exc.exceptions: - push_tb_down(new_tb, child_exc, preserved) - exc.__traceback__ = None - else: - exc.__traceback__ = new_tb - - preserved = set() - new_root_exc = filter_tree(root_exc, preserved) - push_tb_down(None, root_exc, preserved) - # Delete the local functions to avoid a reference cycle (see - # test_simple_cancel_scope_usage_doesnt_create_cyclic_garbage) - del filter_tree, push_tb_down - return new_root_exc - - -# Normally I'm a big fan of (a)contextmanager, but in this case I found it -# easier to use the raw context manager protocol, because it makes it a lot -# easier to reason about how we're mutating the traceback as we go. (End -# result: if the exception gets modified, then the 'raise' here makes this -# frame show up in the traceback; otherwise, we leave no trace.) -@attr.s(frozen=True) -class MultiErrorCatcher: - _handler = attr.ib() - - def __enter__(self): - pass - - def __exit__(self, etype, exc, tb): - if exc is not None: - filtered_exc = MultiError.filter(self._handler, exc) - - if filtered_exc is exc: - # Let the interpreter re-raise it - return False - if filtered_exc is None: - # Swallow the exception - return True - # When we raise filtered_exc, Python will unconditionally blow - # away its __context__ attribute and replace it with the original - # exc we caught. So after we raise it, we have to pause it while - # it's in flight to put the correct __context__ back. - old_context = filtered_exc.__context__ - try: - raise filtered_exc - finally: - _, value, _ = sys.exc_info() - assert value is filtered_exc - value.__context__ = old_context - # delete references from locals to avoid creating cycles - # see test_MultiError_catch_doesnt_create_cyclic_garbage - del _, filtered_exc, value - - -class MultiError(BaseException): - """An exception that contains other exceptions; also known as an - "inception". - - It's main use is to represent the situation when multiple child tasks all - raise errors "in parallel". - - Args: - exceptions (list): The exceptions - - Returns: - If ``len(exceptions) == 1``, returns that exception. This means that a - call to ``MultiError(...)`` is not guaranteed to return a - :exc:`MultiError` object! - - Otherwise, returns a new :exc:`MultiError` object. - - Raises: - TypeError: if any of the passed in objects are not instances of - :exc:`BaseException`. - - """ - - def __init__(self, exceptions): - # Avoid recursion when exceptions[0] returned by __new__() happens - # to be a MultiError and subsequently __init__() is called. - if hasattr(self, "exceptions"): - # __init__ was already called on this object - assert len(exceptions) == 1 and exceptions[0] is self - return - self.exceptions = exceptions - - def __new__(cls, exceptions): - exceptions = list(exceptions) - for exc in exceptions: - if not isinstance(exc, BaseException): - raise TypeError("Expected an exception object, not {!r}".format(exc)) - if len(exceptions) == 1: - # If this lone object happens to itself be a MultiError, then - # Python will implicitly call our __init__ on it again. See - # special handling in __init__. - return exceptions[0] - else: - # The base class __new__() implicitly invokes our __init__, which - # is what we want. - # - # In an earlier version of the code, we didn't define __init__ and - # simply set the `exceptions` attribute directly on the new object. - # However, linters expect attributes to be initialized in __init__. - return BaseException.__new__(cls, exceptions) - - def __str__(self): - return ", ".join(repr(exc) for exc in self.exceptions) - - def __repr__(self): - return "".format(self) - - @classmethod - def filter(cls, handler, root_exc): - """Apply the given ``handler`` to all the exceptions in ``root_exc``. - - Args: - handler: A callable that takes an atomic (non-MultiError) exception - as input, and returns either a new exception object or None. - root_exc: An exception, often (though not necessarily) a - :exc:`MultiError`. - - Returns: - A new exception object in which each component exception ``exc`` has - been replaced by the result of running ``handler(exc)`` – or, if - ``handler`` returned None for all the inputs, returns None. - - """ - - return _filter_impl(handler, root_exc) - - @classmethod - def catch(cls, handler): - """Return a context manager that catches and re-throws exceptions - after running :meth:`filter` on them. - - Args: - handler: as for :meth:`filter` - - """ - - return MultiErrorCatcher(handler) - - -# Clean up exception printing: -MultiError.__module__ = "trio" - -################################################################ -# concat_tb -################################################################ - -# We need to compute a new traceback that is the concatenation of two existing -# tracebacks. This requires copying the entries in 'head' and then pointing -# the final tb_next to 'tail'. -# -# NB: 'tail' might be None, which requires some special handling in the ctypes -# version. -# -# The complication here is that Python doesn't actually support copying or -# modifying traceback objects, so we have to get creative... -# -# On CPython, we use ctypes. On PyPy, we use "transparent proxies". -# -# Jinja2 is a useful source of inspiration: -# https://github.com/pallets/jinja/blob/master/jinja2/debug.py - -try: - import tputil -except ImportError: - have_tproxy = False -else: - have_tproxy = True - -if have_tproxy: - # http://doc.pypy.org/en/latest/objspace-proxies.html - def copy_tb(base_tb, tb_next): - def controller(operation): - # Rationale for pragma: I looked fairly carefully and tried a few - # things, and AFAICT it's not actually possible to get any - # 'opname' that isn't __getattr__ or __getattribute__. So there's - # no missing test we could add, and no value in coverage nagging - # us about adding one. - if operation.opname in [ - "__getattribute__", - "__getattr__", - ]: # pragma: no cover - if operation.args[0] == "tb_next": - return tb_next - return operation.delegate() - - return tputil.make_proxy(controller, type(base_tb), base_tb) - - -else: - # ctypes it is - import ctypes - - # How to handle refcounting? I don't want to use ctypes.py_object because - # I don't understand or trust it, and I don't want to use - # ctypes.pythonapi.Py_{Inc,Dec}Ref because we might clash with user code - # that also tries to use them but with different types. So private _ctypes - # APIs it is! - import _ctypes - - class CTraceback(ctypes.Structure): - _fields_ = [ - ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), - ("tb_next", ctypes.c_void_p), - ("tb_frame", ctypes.c_void_p), - ("tb_lasti", ctypes.c_int), - ("tb_lineno", ctypes.c_int), - ] - - def copy_tb(base_tb, tb_next): - # TracebackType has no public constructor, so allocate one the hard way - try: - raise ValueError - except ValueError as exc: - new_tb = exc.__traceback__ - c_new_tb = CTraceback.from_address(id(new_tb)) - - # At the C level, tb_next either pointer to the next traceback or is - # NULL. c_void_p and the .tb_next accessor both convert NULL to None, - # but we shouldn't DECREF None just because we assigned to a NULL - # pointer! Here we know that our new traceback has only 1 frame in it, - # so we can assume the tb_next field is NULL. - assert c_new_tb.tb_next is None - # If tb_next is None, then we want to set c_new_tb.tb_next to NULL, - # which it already is, so we're done. Otherwise, we have to actually - # do some work: - if tb_next is not None: - _ctypes.Py_INCREF(tb_next) - c_new_tb.tb_next = id(tb_next) - - assert c_new_tb.tb_frame is not None - _ctypes.Py_INCREF(base_tb.tb_frame) - old_tb_frame = new_tb.tb_frame - c_new_tb.tb_frame = id(base_tb.tb_frame) - _ctypes.Py_DECREF(old_tb_frame) - - c_new_tb.tb_lasti = base_tb.tb_lasti - c_new_tb.tb_lineno = base_tb.tb_lineno - - try: - return new_tb - finally: - # delete references from locals to avoid creating cycles - # see test_MultiError_catch_doesnt_create_cyclic_garbage - del new_tb, old_tb_frame - - -def concat_tb(head, tail): - # We have to use an iterative algorithm here, because in the worst case - # this might be a RecursionError stack that is by definition too deep to - # process by recursion! - head_tbs = [] - pointer = head - while pointer is not None: - head_tbs.append(pointer) - pointer = pointer.tb_next - current_head = tail - for head_tb in reversed(head_tbs): - current_head = copy_tb(head_tb, tb_next=current_head) - return current_head - - -################################################################ -# MultiError traceback formatting -# -# What follows is terrible, terrible monkey patching of -# traceback.TracebackException to add support for handling -# MultiErrors -################################################################ - -traceback_exception_original_init = traceback.TracebackException.__init__ - - -def traceback_exception_init( - self, - exc_type, - exc_value, - exc_traceback, - *, - limit=None, - lookup_lines=True, - capture_locals=False, - compact=False, - _seen=None, -): - if sys.version_info >= (3, 10): - kwargs = {"compact": compact} - else: - kwargs = {} - - # Capture the original exception and its cause and context as TracebackExceptions - traceback_exception_original_init( - self, - exc_type, - exc_value, - exc_traceback, - limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - _seen=_seen, - **kwargs, - ) - - seen_was_none = _seen is None - - if _seen is None: - _seen = set() - - # Capture each of the exceptions in the MultiError along with each of their causes and contexts - if isinstance(exc_value, MultiError): - embedded = [] - for exc in exc_value.exceptions: - if id(exc) not in _seen: - embedded.append( - traceback.TracebackException.from_exception( - exc, - limit=limit, - lookup_lines=lookup_lines, - capture_locals=capture_locals, - # copy the set of _seen exceptions so that duplicates - # shared between sub-exceptions are not omitted - _seen=None if seen_was_none else set(_seen), - ) - ) - self.embedded = embedded - else: - self.embedded = [] - - -traceback.TracebackException.__init__ = traceback_exception_init # type: ignore -traceback_exception_original_format = traceback.TracebackException.format - - -def traceback_exception_format(self, *, chain=True): - yield from traceback_exception_original_format(self, chain=chain) - - for i, exc in enumerate(self.embedded): - yield "\nDetails of embedded exception {}:\n\n".format(i + 1) - yield from (textwrap.indent(line, " " * 2) for line in exc.format(chain=chain)) - - -traceback.TracebackException.format = traceback_exception_format # type: ignore - - -def trio_excepthook(etype, value, tb): - for chunk in traceback.format_exception(etype, value, tb): - sys.stderr.write(chunk) - - -monkeypatched_or_warned = False - -if "IPython" in sys.modules: - import IPython - - ip = IPython.get_ipython() - if ip is not None: - if ip.custom_exceptions != (): - warnings.warn( - "IPython detected, but you already have a custom exception " - "handler installed. I'll skip installing Trio's custom " - "handler, but this means MultiErrors will not show full " - "tracebacks.", - category=RuntimeWarning, - ) - monkeypatched_or_warned = True - else: - - def trio_show_traceback(self, etype, value, tb, tb_offset=None): - # XX it would be better to integrate with IPython's fancy - # exception formatting stuff (and not ignore tb_offset) - trio_excepthook(etype, value, tb) - - ip.set_custom_exc((MultiError,), trio_show_traceback) - monkeypatched_or_warned = True - -if sys.excepthook is sys.__excepthook__: - sys.excepthook = trio_excepthook - monkeypatched_or_warned = True - -# Ubuntu's system Python has a sitecustomize.py file that import -# apport_python_hook and replaces sys.excepthook. -# -# The custom hook captures the error for crash reporting, and then calls -# sys.__excepthook__ to actually print the error. -# -# We don't mind it capturing the error for crash reporting, but we want to -# take over printing the error. So we monkeypatch the apport_python_hook -# module so that instead of calling sys.__excepthook__, it calls our custom -# hook. -# -# More details: https://github.com/python-trio/trio/issues/1065 -if getattr(sys.excepthook, "__name__", None) == "apport_excepthook": - import apport_python_hook - - assert sys.excepthook is apport_python_hook.apport_excepthook - - # Give it a descriptive name as a hint for anyone who's stuck trying to - # debug this mess later. - class TrioFakeSysModuleForApport: - pass - - fake_sys = TrioFakeSysModuleForApport() - fake_sys.__dict__.update(sys.__dict__) - fake_sys.__excepthook__ = trio_excepthook # type: ignore - apport_python_hook.sys = fake_sys - - monkeypatched_or_warned = True - -if not monkeypatched_or_warned: - warnings.warn( - "You seem to already have a custom sys.excepthook handler " - "installed. I'll skip installing Trio's custom handler, but this " - "means MultiErrors will not show full tracebacks.", - category=RuntimeWarning, - ) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index f3adfc7580..fac3a7e122 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -28,7 +28,6 @@ KIManager, enable_ki_protection, ) -from ._multierror import MultiError from ._traps import ( Abort, wait_task_rescheduled, @@ -43,6 +42,9 @@ from .. import _core from .._util import Final, NoPublicConstructor, coroutine_or_error +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + DEADLINE_HEAP_MIN_PRUNE_THRESHOLD = 1000 _NO_SEND = object() @@ -447,12 +449,6 @@ def __enter__(self): task._activate_cancel_status(self._cancel_status) return self - def _exc_filter(self, exc): - if isinstance(exc, Cancelled): - self.cancelled_caught = True - return None - return exc - def _close(self, exc): if self._cancel_status is None: new_exc = RuntimeError( @@ -510,7 +506,16 @@ def _close(self, exc): and self._cancel_status.effectively_cancelled and not self._cancel_status.parent_cancellation_is_visible_to_us ): - exc = MultiError.filter(self._exc_filter, exc) + if isinstance(exc, Cancelled): + self.cancelled_caught = True + exc = None + elif isinstance(exc, BaseExceptionGroup): + matched, exc = exc.split(Cancelled) + if matched: + self.cancelled_caught = True + + while isinstance(exc, BaseExceptionGroup) and len(exc.exceptions) == 1: + exc = exc.exceptions[0] self._cancel_status.close() with self._might_change_registered_deadline(): self._cancel_status = None @@ -910,7 +915,9 @@ def _child_finished(self, task, outcome): self._check_nursery_closed() async def _nested_child_finished(self, nested_child_exc): - """Returns MultiError instance if there are pending exceptions.""" + """ + Returns BaseExceptionGroup instance if there are pending exceptions. + """ if nested_child_exc is not None: self._add_exc(nested_child_exc) self._nested_child_running = False @@ -939,7 +946,7 @@ def aborted(raise_cancel): assert popped is self if self._pending_excs: try: - return MultiError(self._pending_excs) + return BaseExceptionGroup("multiple tasks failed", self._pending_excs) finally: # avoid a garbage cycle # (see test_nursery_cancel_doesnt_create_cyclic_garbage) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py deleted file mode 100644 index e4a30e0e69..0000000000 --- a/trio/_core/tests/test_multierror.py +++ /dev/null @@ -1,772 +0,0 @@ -import gc -import logging -import pytest - -from traceback import ( - extract_tb, - print_exception, - format_exception, -) -from traceback import _cause_message # type: ignore -import sys -import os -import re -from pathlib import Path -import subprocess - -from .tutil import slow - -from .._multierror import MultiError, concat_tb -from ..._core import open_nursery - - -class NotHashableException(Exception): - code = None - - def __init__(self, code): - super().__init__() - self.code = code - - def __eq__(self, other): - if not isinstance(other, NotHashableException): - return False - return self.code == other.code - - -async def raise_nothashable(code): - raise NotHashableException(code) - - -def raiser1(): - raiser1_2() - - -def raiser1_2(): - raiser1_3() - - -def raiser1_3(): - raise ValueError("raiser1_string") - - -def raiser2(): - raiser2_2() - - -def raiser2_2(): - raise KeyError("raiser2_string") - - -def raiser3(): - raise NameError - - -def get_exc(raiser): - try: - raiser() - except Exception as exc: - return exc - - -def get_tb(raiser): - return get_exc(raiser).__traceback__ - - -def einfo(exc): - return (type(exc), exc, exc.__traceback__) - - -def test_concat_tb(): - - tb1 = get_tb(raiser1) - tb2 = get_tb(raiser2) - - # These return a list of (filename, lineno, fn name, text) tuples - # https://docs.python.org/3/library/traceback.html#traceback.extract_tb - entries1 = extract_tb(tb1) - entries2 = extract_tb(tb2) - - tb12 = concat_tb(tb1, tb2) - assert extract_tb(tb12) == entries1 + entries2 - - tb21 = concat_tb(tb2, tb1) - assert extract_tb(tb21) == entries2 + entries1 - - # Check degenerate cases - assert extract_tb(concat_tb(None, tb1)) == entries1 - assert extract_tb(concat_tb(tb1, None)) == entries1 - assert concat_tb(None, None) is None - - # Make sure the original tracebacks didn't get mutated by mistake - assert extract_tb(get_tb(raiser1)) == entries1 - assert extract_tb(get_tb(raiser2)) == entries2 - - -def test_MultiError(): - exc1 = get_exc(raiser1) - exc2 = get_exc(raiser2) - - assert MultiError([exc1]) is exc1 - m = MultiError([exc1, exc2]) - assert m.exceptions == [exc1, exc2] - assert "ValueError" in str(m) - assert "ValueError" in repr(m) - - with pytest.raises(TypeError): - MultiError(object()) - with pytest.raises(TypeError): - MultiError([KeyError(), ValueError]) - - -def test_MultiErrorOfSingleMultiError(): - # For MultiError([MultiError]), ensure there is no bad recursion by the - # constructor where __init__ is called if __new__ returns a bare MultiError. - exceptions = [KeyError(), ValueError()] - a = MultiError(exceptions) - b = MultiError([a]) - assert b == a - assert b.exceptions == exceptions - - -async def test_MultiErrorNotHashable(): - exc1 = NotHashableException(42) - exc2 = NotHashableException(4242) - exc3 = ValueError() - assert exc1 != exc2 - assert exc1 != exc3 - - with pytest.raises(MultiError): - async with open_nursery() as nursery: - nursery.start_soon(raise_nothashable, 42) - nursery.start_soon(raise_nothashable, 4242) - - -def test_MultiError_filter_NotHashable(): - excs = MultiError([NotHashableException(42), ValueError()]) - - def handle_ValueError(exc): - if isinstance(exc, ValueError): - return None - else: - return exc - - filtered_excs = MultiError.filter(handle_ValueError, excs) - assert isinstance(filtered_excs, NotHashableException) - - -def test_traceback_recursion(): - exc1 = RuntimeError() - exc2 = KeyError() - exc3 = NotHashableException(42) - # Note how this creates a loop, where exc1 refers to exc1 - # This could trigger an infinite recursion; the 'seen' set is supposed to prevent - # this. - exc1.__cause__ = MultiError([exc1, exc2, exc3]) - format_exception(*einfo(exc1)) - - -def make_tree(): - # Returns an object like: - # MultiError([ - # MultiError([ - # ValueError, - # KeyError, - # ]), - # NameError, - # ]) - # where all exceptions except the root have a non-trivial traceback. - exc1 = get_exc(raiser1) - exc2 = get_exc(raiser2) - exc3 = get_exc(raiser3) - - # Give m12 a non-trivial traceback - try: - raise MultiError([exc1, exc2]) - except BaseException as m12: - return MultiError([m12, exc3]) - - -def assert_tree_eq(m1, m2): - if m1 is None or m2 is None: - assert m1 is m2 - return - assert type(m1) is type(m2) - assert extract_tb(m1.__traceback__) == extract_tb(m2.__traceback__) - assert_tree_eq(m1.__cause__, m2.__cause__) - assert_tree_eq(m1.__context__, m2.__context__) - if isinstance(m1, MultiError): - assert len(m1.exceptions) == len(m2.exceptions) - for e1, e2 in zip(m1.exceptions, m2.exceptions): - assert_tree_eq(e1, e2) - - -def test_MultiError_filter(): - def null_handler(exc): - return exc - - m = make_tree() - assert_tree_eq(m, m) - assert MultiError.filter(null_handler, m) is m - assert_tree_eq(m, make_tree()) - - # Make sure we don't pick up any detritus if run in a context where - # implicit exception chaining would like to kick in - m = make_tree() - try: - raise ValueError - except ValueError: - assert MultiError.filter(null_handler, m) is m - assert_tree_eq(m, make_tree()) - - def simple_filter(exc): - if isinstance(exc, ValueError): - return None - if isinstance(exc, KeyError): - return RuntimeError() - return exc - - new_m = MultiError.filter(simple_filter, make_tree()) - assert isinstance(new_m, MultiError) - assert len(new_m.exceptions) == 2 - # was: [[ValueError, KeyError], NameError] - # ValueError disappeared & KeyError became RuntimeError, so now: - assert isinstance(new_m.exceptions[0], RuntimeError) - assert isinstance(new_m.exceptions[1], NameError) - - # implicit chaining: - assert isinstance(new_m.exceptions[0].__context__, KeyError) - - # also, the traceback on the KeyError incorporates what used to be the - # traceback on its parent MultiError - orig = make_tree() - # make sure we have the right path - assert isinstance(orig.exceptions[0].exceptions[1], KeyError) - # get original traceback summary - orig_extracted = ( - extract_tb(orig.__traceback__) - + extract_tb(orig.exceptions[0].__traceback__) - + extract_tb(orig.exceptions[0].exceptions[1].__traceback__) - ) - - def p(exc): - print_exception(type(exc), exc, exc.__traceback__) - - p(orig) - p(orig.exceptions[0]) - p(orig.exceptions[0].exceptions[1]) - p(new_m.exceptions[0].__context__) - # compare to the new path - assert new_m.__traceback__ is None - new_extracted = extract_tb(new_m.exceptions[0].__context__.__traceback__) - assert orig_extracted == new_extracted - - # check preserving partial tree - def filter_NameError(exc): - if isinstance(exc, NameError): - return None - return exc - - m = make_tree() - new_m = MultiError.filter(filter_NameError, m) - # with the NameError gone, the other branch gets promoted - assert new_m is m.exceptions[0] - - # check fully handling everything - def filter_all(exc): - return None - - assert MultiError.filter(filter_all, make_tree()) is None - - -def test_MultiError_catch(): - # No exception to catch - - def noop(_): - pass # pragma: no cover - - with MultiError.catch(noop): - pass - - # Simple pass-through of all exceptions - m = make_tree() - with pytest.raises(MultiError) as excinfo: - with MultiError.catch(lambda exc: exc): - raise m - assert excinfo.value is m - # Should be unchanged, except that we added a traceback frame by raising - # it here - assert m.__traceback__ is not None - assert m.__traceback__.tb_frame.f_code.co_name == "test_MultiError_catch" - assert m.__traceback__.tb_next is None - m.__traceback__ = None - assert_tree_eq(m, make_tree()) - - # Swallows everything - with MultiError.catch(lambda _: None): - raise make_tree() - - def simple_filter(exc): - if isinstance(exc, ValueError): - return None - if isinstance(exc, KeyError): - return RuntimeError() - return exc - - with pytest.raises(MultiError) as excinfo: - with MultiError.catch(simple_filter): - raise make_tree() - new_m = excinfo.value - assert isinstance(new_m, MultiError) - assert len(new_m.exceptions) == 2 - # was: [[ValueError, KeyError], NameError] - # ValueError disappeared & KeyError became RuntimeError, so now: - assert isinstance(new_m.exceptions[0], RuntimeError) - assert isinstance(new_m.exceptions[1], NameError) - # Make sure that Python did not successfully attach the old MultiError to - # our new MultiError's __context__ - assert not new_m.__suppress_context__ - assert new_m.__context__ is None - - # check preservation of __cause__ and __context__ - v = ValueError() - v.__cause__ = KeyError() - with pytest.raises(ValueError) as excinfo: - with MultiError.catch(lambda exc: exc): - raise v - assert isinstance(excinfo.value.__cause__, KeyError) - - v = ValueError() - context = KeyError() - v.__context__ = context - with pytest.raises(ValueError) as excinfo: - with MultiError.catch(lambda exc: exc): - raise v - assert excinfo.value.__context__ is context - assert not excinfo.value.__suppress_context__ - - for suppress_context in [True, False]: - v = ValueError() - context = KeyError() - v.__context__ = context - v.__suppress_context__ = suppress_context - distractor = RuntimeError() - with pytest.raises(ValueError) as excinfo: - - def catch_RuntimeError(exc): - if isinstance(exc, RuntimeError): - return None - else: - return exc - - with MultiError.catch(catch_RuntimeError): - raise MultiError([v, distractor]) - assert excinfo.value.__context__ is context - assert excinfo.value.__suppress_context__ == suppress_context - - -@pytest.mark.skipif( - sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" -) -def test_MultiError_catch_doesnt_create_cyclic_garbage(): - # https://github.com/python-trio/trio/pull/2063 - gc.collect() - old_flags = gc.get_debug() - - def make_multi(): - # make_tree creates cycles itself, so a simple - raise MultiError([get_exc(raiser1), get_exc(raiser2)]) - - def simple_filter(exc): - if isinstance(exc, ValueError): - return Exception() - if isinstance(exc, KeyError): - return RuntimeError() - assert False, "only ValueError and KeyError should exist" # pragma: no cover - - try: - gc.set_debug(gc.DEBUG_SAVEALL) - with pytest.raises(MultiError): - # covers MultiErrorCatcher.__exit__ and _multierror.copy_tb - with MultiError.catch(simple_filter): - raise make_multi() - gc.collect() - assert not gc.garbage - finally: - gc.set_debug(old_flags) - gc.garbage.clear() - - -def assert_match_in_seq(pattern_list, string): - offset = 0 - print("looking for pattern matches...") - for pattern in pattern_list: - print("checking pattern:", pattern) - reobj = re.compile(pattern) - match = reobj.search(string, offset) - assert match is not None - offset = match.end() - - -def test_assert_match_in_seq(): - assert_match_in_seq(["a", "b"], "xx a xx b xx") - assert_match_in_seq(["b", "a"], "xx b xx a xx") - with pytest.raises(AssertionError): - assert_match_in_seq(["a", "b"], "xx b xx a xx") - - -def test_format_exception(): - exc = get_exc(raiser1) - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - assert "direct cause" not in formatted - assert "During handling" not in formatted - - exc = get_exc(raiser1) - exc.__cause__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" in formatted - assert "in raiser2_2" in formatted - assert "direct cause" in formatted - assert "During handling" not in formatted - # ensure cause included - assert _cause_message in formatted - - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" in formatted - assert "in raiser2_2" in formatted - assert "direct cause" not in formatted - assert "During handling" in formatted - - exc.__suppress_context__ = True - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - assert "direct cause" not in formatted - assert "During handling" not in formatted - - # chain=False - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc), chain=False)) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - assert "direct cause" not in formatted - assert "During handling" not in formatted - - # limit - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - # get_exc adds a frame that counts against the limit, so limit=2 means we - # get 1 deep into the raiser stack - formatted = "".join(format_exception(*einfo(exc), limit=2)) - print(formatted) - assert "raiser1_string" in formatted - assert "in raiser1" in formatted - assert "in raiser1_2" not in formatted - assert "raiser2_string" in formatted - assert "in raiser2" in formatted - assert "in raiser2_2" not in formatted - - exc = get_exc(raiser1) - exc.__context__ = get_exc(raiser2) - formatted = "".join(format_exception(*einfo(exc), limit=1)) - print(formatted) - assert "raiser1_string" in formatted - assert "in raiser1" not in formatted - assert "raiser2_string" in formatted - assert "in raiser2" not in formatted - - # handles loops - exc = get_exc(raiser1) - exc.__cause__ = exc - formatted = "".join(format_exception(*einfo(exc))) - assert "raiser1_string" in formatted - assert "in raiser1_3" in formatted - assert "raiser2_string" not in formatted - assert "in raiser2_2" not in formatted - # ensure duplicate exception is not included as cause - assert _cause_message not in formatted - - # MultiError - formatted = "".join(format_exception(*einfo(make_tree()))) - print(formatted) - - assert_match_in_seq( - [ - # Outer exception is MultiError - r"MultiError:", - # First embedded exception is the embedded MultiError - r"\nDetails of embedded exception 1", - # Which has a single stack frame from make_tree raising it - r"in make_tree", - # Then it has two embedded exceptions - r" Details of embedded exception 1", - r"in raiser1_2", - # for some reason ValueError has no quotes - r"ValueError: raiser1_string", - r" Details of embedded exception 2", - r"in raiser2_2", - # But KeyError does have quotes - r"KeyError: 'raiser2_string'", - # And finally the NameError, which is a sibling of the embedded - # MultiError - r"\nDetails of embedded exception 2:", - r"in raiser3", - r"NameError", - ], - formatted, - ) - - # Prints duplicate exceptions in sub-exceptions - exc1 = get_exc(raiser1) - - def raise1_raiser1(): - try: - raise exc1 - except: - raise ValueError("foo") - - def raise2_raiser1(): - try: - raise exc1 - except: - raise KeyError("bar") - - exc2 = get_exc(raise1_raiser1) - exc3 = get_exc(raise2_raiser1) - - try: - raise MultiError([exc2, exc3]) - except MultiError as e: - exc = e - - formatted = "".join(format_exception(*einfo(exc))) - print(formatted) - - assert_match_in_seq( - [ - r"Traceback", - # Outer exception is MultiError - r"MultiError:", - # First embedded exception is the embedded ValueError with cause of raiser1 - r"\nDetails of embedded exception 1", - # Print details of exc1 - r" Traceback", - r"in get_exc", - r"in raiser1", - r"ValueError: raiser1_string", - # Print details of exc2 - r"\n During handling of the above exception, another exception occurred:", - r" Traceback", - r"in get_exc", - r"in raise1_raiser1", - r" ValueError: foo", - # Second embedded exception is the embedded KeyError with cause of raiser1 - r"\nDetails of embedded exception 2", - # Print details of exc1 again - r" Traceback", - r"in get_exc", - r"in raiser1", - r"ValueError: raiser1_string", - # Print details of exc3 - r"\n During handling of the above exception, another exception occurred:", - r" Traceback", - r"in get_exc", - r"in raise2_raiser1", - r" KeyError: 'bar'", - ], - formatted, - ) - - -def test_logging(caplog): - exc1 = get_exc(raiser1) - exc2 = get_exc(raiser2) - - m = MultiError([exc1, exc2]) - - message = "test test test" - try: - raise m - except MultiError as exc: - logging.getLogger().exception(message) - # Join lines together - formatted = "".join(format_exception(type(exc), exc, exc.__traceback__)) - assert message in caplog.text - assert formatted in caplog.text - - -def run_script(name, use_ipython=False): - import trio - - trio_path = Path(trio.__file__).parent.parent - script_path = Path(__file__).parent / "test_multierror_scripts" / name - - env = dict(os.environ) - print("parent PYTHONPATH:", env.get("PYTHONPATH")) - if "PYTHONPATH" in env: # pragma: no cover - pp = env["PYTHONPATH"].split(os.pathsep) - else: - pp = [] - pp.insert(0, str(trio_path)) - pp.insert(0, str(script_path.parent)) - env["PYTHONPATH"] = os.pathsep.join(pp) - print("subprocess PYTHONPATH:", env.get("PYTHONPATH")) - - if use_ipython: - lines = [script_path.read_text(), "exit()"] - - cmd = [ - sys.executable, - "-u", - "-m", - "IPython", - # no startup files - "--quick", - "--TerminalIPythonApp.code_to_run=" + "\n".join(lines), - ] - else: - cmd = [sys.executable, "-u", str(script_path)] - print("running:", cmd) - completed = subprocess.run( - cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - print("process output:") - print(completed.stdout.decode("utf-8")) - return completed - - -def check_simple_excepthook(completed): - assert_match_in_seq( - [ - "in ", - "MultiError", - "Details of embedded exception 1", - "in exc1_fn", - "ValueError", - "Details of embedded exception 2", - "in exc2_fn", - "KeyError", - ], - completed.stdout.decode("utf-8"), - ) - - -def test_simple_excepthook(): - completed = run_script("simple_excepthook.py") - check_simple_excepthook(completed) - - -def test_custom_excepthook(): - # Check that user-defined excepthooks aren't overridden - completed = run_script("custom_excepthook.py") - assert_match_in_seq( - [ - # The warning - "RuntimeWarning", - "already have a custom", - # The message printed by the custom hook, proving we didn't - # override it - "custom running!", - # The MultiError - "MultiError:", - ], - completed.stdout.decode("utf-8"), - ) - - -# This warning is triggered by ipython 7.5.0 on python 3.8 -import warnings - -warnings.filterwarnings( - "ignore", - message='.*"@coroutine" decorator is deprecated', - category=DeprecationWarning, - module="IPython.*", -) -try: - import IPython -except ImportError: # pragma: no cover - have_ipython = False -else: - have_ipython = True - -need_ipython = pytest.mark.skipif(not have_ipython, reason="need IPython") - - -@slow -@need_ipython -def test_ipython_exc_handler(): - completed = run_script("simple_excepthook.py", use_ipython=True) - check_simple_excepthook(completed) - - -@slow -@need_ipython -def test_ipython_imported_but_unused(): - completed = run_script("simple_excepthook_IPython.py") - check_simple_excepthook(completed) - - -@slow -def test_partial_imported_but_unused(): - # Check that a functools.partial as sys.excepthook doesn't cause an exception when - # importing trio. This was a problem due to the lack of a .__name__ attribute and - # happens when inside a pytest-qt test case for example. - completed = run_script("simple_excepthook_partial.py") - completed.check_returncode() - - -@slow -@need_ipython -def test_ipython_custom_exc_handler(): - # Check we get a nice warning (but only one!) if the user is using IPython - # and already has some other set_custom_exc handler installed. - completed = run_script("ipython_custom_exc.py", use_ipython=True) - assert_match_in_seq( - [ - # The warning - "RuntimeWarning", - "IPython detected", - "skip installing Trio", - # The MultiError - "MultiError", - "ValueError", - "KeyError", - ], - completed.stdout.decode("utf-8"), - ) - # Make sure our other warning doesn't show up - assert "custom sys.excepthook" not in completed.stdout.decode("utf-8") - - -@slow -@pytest.mark.skipif( - not Path("/usr/lib/python3/dist-packages/apport_python_hook.py").exists(), - reason="need Ubuntu with python3-apport installed", -) -def test_apport_excepthook_monkeypatch_interaction(): - completed = run_script("apport_excepthook.py") - stdout = completed.stdout.decode("utf-8") - - # No warning - assert "custom sys.excepthook" not in stdout - - # Proper traceback - assert_match_in_seq( - ["Details of embedded", "KeyError", "Details of embedded", "ValueError"], - stdout, - ) diff --git a/trio/_core/tests/test_multierror_scripts/__init__.py b/trio/_core/tests/test_multierror_scripts/__init__.py deleted file mode 100644 index a1f6cb598d..0000000000 --- a/trio/_core/tests/test_multierror_scripts/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# This isn't really a package, everything in here is a standalone script. This -# __init__.py is just to fool setup.py into actually installing the things. diff --git a/trio/_core/tests/test_multierror_scripts/_common.py b/trio/_core/tests/test_multierror_scripts/_common.py deleted file mode 100644 index 0c70df1840..0000000000 --- a/trio/_core/tests/test_multierror_scripts/_common.py +++ /dev/null @@ -1,7 +0,0 @@ -# https://coverage.readthedocs.io/en/latest/subprocess.html -try: - import coverage -except ImportError: # pragma: no cover - pass -else: - coverage.process_startup() diff --git a/trio/_core/tests/test_multierror_scripts/apport_excepthook.py b/trio/_core/tests/test_multierror_scripts/apport_excepthook.py deleted file mode 100644 index 12e7fb0851..0000000000 --- a/trio/_core/tests/test_multierror_scripts/apport_excepthook.py +++ /dev/null @@ -1,13 +0,0 @@ -# The apport_python_hook package is only installed as part of Ubuntu's system -# python, and not available in venvs. So before we can import it we have to -# make sure it's on sys.path. -import sys - -sys.path.append("/usr/lib/python3/dist-packages") -import apport_python_hook - -apport_python_hook.install() - -import trio - -raise trio.MultiError([KeyError("key_error"), ValueError("value_error")]) diff --git a/trio/_core/tests/test_multierror_scripts/custom_excepthook.py b/trio/_core/tests/test_multierror_scripts/custom_excepthook.py deleted file mode 100644 index 564c5833b2..0000000000 --- a/trio/_core/tests/test_multierror_scripts/custom_excepthook.py +++ /dev/null @@ -1,18 +0,0 @@ -import _common - -import sys - - -def custom_excepthook(*args): - print("custom running!") - return sys.__excepthook__(*args) - - -sys.excepthook = custom_excepthook - -# Should warn that we'll get kinda-broken tracebacks -import trio - -# The custom excepthook should run, because Trio was polite and didn't -# override it -raise trio.MultiError([ValueError(), KeyError()]) diff --git a/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py b/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py deleted file mode 100644 index b3fd110e50..0000000000 --- a/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py +++ /dev/null @@ -1,36 +0,0 @@ -import _common - -# Override the regular excepthook too -- it doesn't change anything either way -# because ipython doesn't use it, but we want to make sure Trio doesn't warn -# about it. -import sys - - -def custom_excepthook(*args): - print("custom running!") - return sys.__excepthook__(*args) - - -sys.excepthook = custom_excepthook - -import IPython - -ip = IPython.get_ipython() - - -# Set this to some random nonsense -class SomeError(Exception): - pass - - -def custom_exc_hook(etype, value, tb, tb_offset=None): - ip.showtraceback() - - -ip.set_custom_exc((SomeError,), custom_exc_hook) - -import trio - -# The custom excepthook should run, because Trio was polite and didn't -# override it -raise trio.MultiError([ValueError(), KeyError()]) diff --git a/trio/_core/tests/test_multierror_scripts/simple_excepthook.py b/trio/_core/tests/test_multierror_scripts/simple_excepthook.py deleted file mode 100644 index 94004525db..0000000000 --- a/trio/_core/tests/test_multierror_scripts/simple_excepthook.py +++ /dev/null @@ -1,21 +0,0 @@ -import _common - -import trio - - -def exc1_fn(): - try: - raise ValueError - except Exception as exc: - return exc - - -def exc2_fn(): - try: - raise KeyError - except Exception as exc: - return exc - - -# This should be printed nicely, because Trio overrode sys.excepthook -raise trio.MultiError([exc1_fn(), exc2_fn()]) diff --git a/trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py b/trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py deleted file mode 100644 index 6aa12493b0..0000000000 --- a/trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py +++ /dev/null @@ -1,7 +0,0 @@ -import _common - -# To tickle the "is IPython loaded?" logic, make sure that Trio tolerates -# IPython loaded but not actually in use -import IPython - -import simple_excepthook diff --git a/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py b/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py deleted file mode 100644 index e97fc39d57..0000000000 --- a/trio/_core/tests/test_multierror_scripts/simple_excepthook_partial.py +++ /dev/null @@ -1,13 +0,0 @@ -import functools -import sys - -import _common - -# just making sure importing Trio doesn't fail if sys.excepthook doesn't have a -# .__name__ attribute - -sys.excepthook = functools.partial(sys.excepthook) - -assert not hasattr(sys.excepthook, "__name__") - -import trio diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 863dad9909..c9d73f4c9a 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -37,6 +37,9 @@ assert_checkpoints, ) +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup, ExceptionGroup, catch + # slightly different from _timeouts.sleep_forever because it returns the value # its rescheduled with, which is really only useful for tests of @@ -167,8 +170,8 @@ async def main(): def test_main_and_task_both_crash(): - # If main crashes and there's also a task crash, then we get both in a - # MultiError + # If main crashes and there's also a task crash, then we get both in an + # ExceptionGroup async def crasher(): raise ValueError @@ -177,7 +180,7 @@ async def main(): nursery.start_soon(crasher) raise KeyError - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(ExceptionGroup) as excinfo: _core.run(main) print(excinfo.value) assert {type(exc) for exc in excinfo.value.exceptions} == { @@ -195,7 +198,7 @@ async def main(): nursery.start_soon(crasher, KeyError) nursery.start_soon(crasher, ValueError) - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(ExceptionGroup) as excinfo: _core.run(main) assert {type(exc) for exc in excinfo.value.exceptions} == { ValueError, @@ -432,7 +435,7 @@ async def crasher(): # And one that raises a different error nursery.start_soon(crasher) # t4 # and then our __aexit__ also receives an outer Cancelled - except _core.MultiError as multi_exc: + except BaseExceptionGroup as multi_exc: # Since the outer scope became cancelled before the # nursery block exited, all cancellations inside the # nursery block continue propagating to reach the @@ -771,7 +774,7 @@ async def task2(): with pytest.raises(RuntimeError) as exc_info: await nursery_mgr.__aexit__(*sys.exc_info()) assert "which had already been exited" in str(exc_info.value) - assert type(exc_info.value.__context__) is _core.MultiError + assert type(exc_info.value.__context__) is ExceptionGroup assert len(exc_info.value.__context__.exceptions) == 3 cancelled_in_context = False for exc in exc_info.value.__context__.exceptions: @@ -912,7 +915,7 @@ async def main(): _core.run(main) -def test_system_task_crash_MultiError(): +def test_system_task_crash_ExceptionGroup(): async def crasher1(): raise KeyError @@ -932,7 +935,7 @@ async def main(): _core.run(main) me = excinfo.value.__cause__ - assert isinstance(me, _core.MultiError) + assert isinstance(me, ExceptionGroup) assert len(me.exceptions) == 2 for exc in me.exceptions: assert isinstance(exc, (KeyError, ValueError)) @@ -940,7 +943,7 @@ async def main(): def test_system_task_crash_plus_Cancelled(): # Set up a situation where a system task crashes with a - # MultiError([Cancelled, ValueError]) + # BaseExceptionGroup([Cancelled, ValueError]) async def crasher(): try: await sleep_forever() @@ -1111,11 +1114,11 @@ async def test_nursery_exception_chaining_doesnt_make_context_loops(): async def crasher(): raise KeyError - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(ExceptionGroup) as excinfo: async with _core.open_nursery() as nursery: nursery.start_soon(crasher) raise ValueError - # the MultiError should not have the KeyError or ValueError as context + # the ExceptionGroup should not have the KeyError or ValueError as context assert excinfo.value.__context__ is None @@ -1615,7 +1618,7 @@ async def test_trivial_yields(): with _core.CancelScope() as cancel_scope: cancel_scope.cancel() - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(BaseExceptionGroup) as excinfo: async with _core.open_nursery(): raise KeyError assert len(excinfo.value.exceptions) == 2 @@ -1705,7 +1708,7 @@ async def raise_keyerror_after_started(task_status=_core.TASK_STATUS_IGNORED): async with _core.open_nursery() as nursery: with _core.CancelScope() as cs: cs.cancel() - with pytest.raises(_core.MultiError) as excinfo: + with pytest.raises(BaseExceptionGroup) as excinfo: await nursery.start(raise_keyerror_after_started) assert {type(e) for e in excinfo.value.exceptions} == { _core.Cancelled, @@ -1820,7 +1823,7 @@ async def fail(): async with _core.open_nursery() as nursery: nursery.start_soon(fail) raise StopIteration - except _core.MultiError as e: + except ExceptionGroup as e: assert tuple(map(type, e.exceptions)) == (StopIteration, ValueError) @@ -1855,13 +1858,9 @@ async def __anext__(self): def handle(exc): nonlocal got_stop - if isinstance(exc, StopAsyncIteration): - got_stop = True - return None - else: # pragma: no cover - return exc + got_stop = True - with _core.MultiError.catch(handle): + with catch({StopAsyncIteration: handle}): async with _core.open_nursery() as nursery: for i, f in enumerate(nexts): nursery.start_soon(self._accumulate, f, items, i) @@ -1882,14 +1881,14 @@ async def my_child_task(): try: # Trick: For now cancel/nursery scopes still leave a bunch of tb gunk - # behind. But if there's a MultiError, they leave it on the MultiError, - # which lets us get a clean look at the KeyError itself. Someday I - # guess this will always be a MultiError (#611), but for now we can - # force it by raising two exceptions. + # behind. But if there's an ExceptionGroup, they leave it on the + # ExceptionGroup, which lets us get a clean look at the KeyError + # itself. Someday I guess this will always be an ExceptionGroup (#611), + # but for now we can force it by raising two exceptions. async with _core.open_nursery() as nursery: nursery.start_soon(my_child_task) nursery.start_soon(my_child_task) - except _core.MultiError as exc: + except ExceptionGroup as exc: first_exc = exc.exceptions[0] assert isinstance(first_exc, KeyError) # The top frame in the exception traceback should be inside the child @@ -2231,7 +2230,7 @@ async def crasher(): with pytest.raises(ValueError): async with _core.open_nursery() as nursery: - # cover MultiError.filter and NurseryManager.__aexit__ + # cover NurseryManager.__aexit__ nursery.start_soon(crasher) gc.collect() @@ -2263,8 +2262,9 @@ async def crasher(): outer.cancel() # And one that raises a different error nursery.start_soon(crasher) - # so that outer filters a Cancelled from the MultiError and - # covers CancelScope.__exit__ (and NurseryManager.__aexit__) + # so that outer filters a Cancelled from the BaseExceptionGroup + # and covers CancelScope.__exit__ + # (and NurseryManager.__aexit__) # (See https://github.com/python-trio/trio/pull/2063) gc.collect() diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index 80f2c7a180..955b38ffe3 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -5,6 +5,9 @@ import trio from . import socket as tsocket +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + # Default backlog size: # @@ -138,7 +141,9 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): errno.EAFNOSUPPORT, "This system doesn't support any of the kinds of " "socket that that address could use", - ) from trio.MultiError(unsupported_address_families) + ) from ExceptionGroup( + "All socket creation attempts failed", unsupported_address_families + ) return listeners diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 545fac8641..5987a23562 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -1,8 +1,12 @@ +import sys from contextlib import contextmanager import trio from trio.socket import getaddrinfo, SOCK_STREAM, socket +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + # Implementation of RFC 6555 "Happy eyeballs" # https://tools.ietf.org/html/rfc6555 # @@ -114,8 +118,10 @@ def close_all(): sock.close() except BaseException as exc: errs.append(exc) - if errs: - raise trio.MultiError(errs) + if len(errs) == 1: + raise errs[0] + elif errs: + raise BaseExceptionGroup("Multiple close operations failed", errs) def reorder_for_rfc_6555_section_5_4(targets): @@ -364,7 +370,9 @@ async def attempt_connect(socket_args, sockaddr, attempt_failed): msg = "all attempts to connect to {} failed".format( format_host_port(host, port) ) - raise OSError(msg) from trio.MultiError(oserrors) + raise OSError(msg) from BaseExceptionGroup( + "multiple connection attempts failed", oserrors + ) else: stream = trio.SocketStream(winning_socket) open_sockets.remove(winning_socket) diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index d5fc576ec5..d9fe26676b 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -1,3 +1,5 @@ +import sys + import pytest import socket as stdlib_socket @@ -11,6 +13,9 @@ from .. import socket as tsocket from .._core.tests.tutil import slow, creates_ipv6, binds_ipv6 +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + async def test_open_tcp_listeners_basic(): listeners = await open_tcp_listeners(0) @@ -239,7 +244,7 @@ async def test_open_tcp_listeners_some_address_families_unavailable( await open_tcp_listeners(80, host="example.org") assert "This system doesn't support" in str(exc_info.value) - if isinstance(exc_info.value.__cause__, trio.MultiError): + if isinstance(exc_info.value.__cause__, ExceptionGroup): for subexc in exc_info.value.__cause__.exceptions: assert "nope" in str(subexc) else: diff --git a/trio/tests/test_highlevel_open_tcp_stream.py b/trio/tests/test_highlevel_open_tcp_stream.py index eaaff3e17f..0f3b6a0baf 100644 --- a/trio/tests/test_highlevel_open_tcp_stream.py +++ b/trio/tests/test_highlevel_open_tcp_stream.py @@ -13,6 +13,9 @@ format_host_port, ) +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + def test_close_all(): class CloseMe: @@ -436,7 +439,7 @@ async def test_all_fail(autojump_clock): expect_error=OSError, ) assert isinstance(exc, OSError) - assert isinstance(exc.__cause__, trio.MultiError) + assert isinstance(exc.__cause__, BaseExceptionGroup) assert len(exc.__cause__.exceptions) == 4 assert trio.current_time() == (0.1 + 0.2 + 10) assert scenario.connect_times == { @@ -556,7 +559,7 @@ async def test_cancel(autojump_clock): ("3.3.3.3", 10, "success"), ("4.4.4.4", 10, "success"), ], - expect_error=trio.MultiError, + expect_error=BaseExceptionGroup, ) # What comes out should be 1 or more Cancelled errors that all belong # to this cancel_scope; this is the easiest way to check that From 6fdd0fe279518ce11ba0ae554aa36764414219af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 17 Jan 2022 01:15:49 +0200 Subject: [PATCH 0888/1498] Collapse exception groups containing a single exception --- trio/_core/_run.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index fac3a7e122..592280abf2 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -118,6 +118,28 @@ class IdlePrimedTypes(enum.Enum): ################################################################ +def collapse_exception_group(excgroup): + """Recursively collapse any single-exception groups into that single contained + exception. + + """ + exceptions = list(excgroup.exceptions) + modified = False + for i, exc in enumerate(exceptions): + if isinstance(exc, BaseExceptionGroup): + new_exc = collapse_exception_group(exc) + if new_exc is not exc: + modified = True + exceptions[i] = new_exc + + if len(exceptions) == 1: + return exceptions[0] + elif modified: + return excgroup.derive(exceptions) + else: + return excgroup + + @attr.s(eq=False, slots=True) class Deadlines: """A container of deadlined cancel scopes. @@ -514,8 +536,9 @@ def _close(self, exc): if matched: self.cancelled_caught = True - while isinstance(exc, BaseExceptionGroup) and len(exc.exceptions) == 1: - exc = exc.exceptions[0] + if isinstance(exc, BaseExceptionGroup): + exc = collapse_exception_group(exc) + self._cancel_status.close() with self._might_change_registered_deadline(): self._cancel_status = None From 13de11dc80e4654fd92dfe1c82cb4236a624871f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 17 Jan 2022 01:24:48 +0200 Subject: [PATCH 0889/1498] Require exceptiongroup unconditionally for tests --- test-requirements.in | 1 + test-requirements.txt | 2 ++ trio/_core/tests/test_run.py | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test-requirements.in b/test-requirements.in index 99854204d6..75e69f2c9b 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -7,6 +7,7 @@ trustme # for the ssl tests pylint # for pylint finding all symbols tests jedi # for jedi code completion tests cryptography>=36.0.0 # 35.0.0 is transitive but fails +exceptiongroup # for catch() # Tools black; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index f1162efdb0..15f8d58b91 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -32,6 +32,8 @@ cryptography==36.0.1 # trustme decorator==5.1.0 # via ipython +exceptiongroup==1.0.0rc1 + # via -r test-requirements.in flake8==4.0.1 # via -r test-requirements.in idna==3.3 diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index c9d73f4c9a..0c563d5310 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -16,6 +16,7 @@ import outcome import sniffio import pytest +from exceptiongroup import catch from .tutil import ( slow, @@ -38,7 +39,7 @@ ) if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup, ExceptionGroup, catch + from exceptiongroup import BaseExceptionGroup, ExceptionGroup # slightly different from _timeouts.sleep_forever because it returns the value From 9ecca9bac6c99fa342f2334d4a6f9705257edb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 25 Jan 2022 14:29:09 +0200 Subject: [PATCH 0890/1498] Added exceptiongroup to docs requirements --- docs-requirements.in | 1 + docs-requirements.txt | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs-requirements.in b/docs-requirements.in index 23a1b0f652..dc9888ec64 100644 --- a/docs-requirements.in +++ b/docs-requirements.in @@ -13,6 +13,7 @@ async_generator >= 1.9 idna outcome sniffio +exceptiongroup # See note in test-requirements.in immutables >= 0.6 diff --git a/docs-requirements.txt b/docs-requirements.txt index b9c1f3d79f..0f2ca56353 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # -# pip-compile --output-file docs-requirements.txt docs-requirements.in +# pip-compile --output-file=docs-requirements.txt docs-requirements.in # alabaster==0.7.12 # via sphinx @@ -28,6 +28,8 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +exceptiongroup==1.0.0rc1 + # via -r docs-requirements.in idna==3.3 # via # -r docs-requirements.in @@ -36,6 +38,8 @@ imagesize==1.2.0 # via sphinx immutables==0.16 # via -r docs-requirements.in +importlib-metadata==4.10.1 + # via click incremental==21.3.0 # via towncrier jinja2==3.0.2 @@ -87,5 +91,14 @@ toml==0.10.2 # via towncrier towncrier==21.3.0 # via -r docs-requirements.in +typing-extensions==4.0.1 + # via + # immutables + # importlib-metadata urllib3==1.26.7 # via requests +zipp==3.7.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 21757c89904879a89856d87253d5fab11c426b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 25 Jan 2022 14:37:24 +0200 Subject: [PATCH 0891/1498] Fixed dangling references in the documentation --- docs/source/history.rst | 10 +++++----- docs/source/reference-core.rst | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/history.rst b/docs/source/history.rst index 3936df867e..a8fd584675 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -232,9 +232,9 @@ Bugfixes - On Ubuntu systems, the system Python includes a custom unhandled-exception hook to perform `crash reporting `__. Unfortunately, Trio wants to use - the same hook to print nice `MultiError` tracebacks, causing a + the same hook to print nice ``MultiError`` tracebacks, causing a conflict. Previously, Trio would detect the conflict, print a warning, - and you just wouldn't get nice `MultiError` tracebacks. Now, Trio has + and you just wouldn't get nice ``MultiError`` tracebacks. Now, Trio has gotten clever enough to integrate its hook with Ubuntu's, so the two systems should Just Work together. (`#1065 `__) - Fixed an over-strict test that caused failures on Alpine Linux. @@ -436,7 +436,7 @@ Features violated. (One common source of such violations is an async generator that yields within a cancel scope.) The previous behavior was an inscrutable chain of TrioInternalErrors. (`#882 `__) -- MultiError now defines its ``exceptions`` attribute in ``__init__()`` +- ``MultiError`` now defines its ``exceptions`` attribute in ``__init__()`` to better support linters and code autocompletion. (`#1066 `__) - Use ``__slots__`` in more places internally, which should make Trio slightly faster. (`#984 `__) @@ -457,7 +457,7 @@ Bugfixes :meth:`~trio.Path.cwd`, are now async functions. Previously, a bug in the forwarding logic meant :meth:`~trio.Path.cwd` was synchronous and :meth:`~trio.Path.home` didn't work at all. (`#960 `__) -- An exception encapsulated within a :class:`MultiError` doesn't need to be +- An exception encapsulated within a `MultiError` doesn't need to be hashable anymore. .. note:: @@ -1248,7 +1248,7 @@ Other changes interfering with direct use of :func:`~trio.testing.wait_all_tasks_blocked` in the same test. -* :meth:`MultiError.catch` now correctly preserves ``__context__``, +* ``MultiError.catch()`` now correctly preserves ``__context__``, despite Python's best attempts to stop us (`#165 `__) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index e320088cf8..658e14d08e 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -641,7 +641,7 @@ crucial things to keep in mind: * Any unhandled exceptions are re-raised inside the parent task. If there are multiple exceptions, then they're collected up into a - single :exc:`BaseExceptionGroup` or :exc:`ExceptionGroup` exception. + single ``BaseExceptionGroup`` or ``ExceptionGroup`` exception. Since all tasks are descendents of the initial task, one consequence of this is that :func:`run` can't finish until all tasks have @@ -712,9 +712,9 @@ limitation. Consider code like:: what? In some sense, the answer should be "both of these at once", but in Python there can only be one exception at a time. -Trio's answer is that it raises a :exc:`BaseExceptionGroup` object. This is a +Trio's answer is that it raises a ``BaseExceptionGroup`` object. This is a special exception which encapsulates multiple exception objects – -either regular exceptions or nested :exc:`BaseExceptionGroup`\s. +either regular exceptions or nested ``BaseExceptionGroup``\s. Spawning tasks without becoming a parent From 61a1646072e91884005f35ebe472ef635dd04bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 25 Jan 2022 19:12:12 +0200 Subject: [PATCH 0892/1498] Removed obsolete check The __cause__ of the OSError will always be an ExceptionGroup going forward. MultiError's constructor, given a single element exception array, returned just that, but ExceptionGroup does not work the same way. --- trio/tests/test_highlevel_open_tcp_listeners.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index d9fe26676b..e86d108d2b 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -244,12 +244,9 @@ async def test_open_tcp_listeners_some_address_families_unavailable( await open_tcp_listeners(80, host="example.org") assert "This system doesn't support" in str(exc_info.value) - if isinstance(exc_info.value.__cause__, ExceptionGroup): - for subexc in exc_info.value.__cause__.exceptions: - assert "nope" in str(subexc) - else: - assert isinstance(exc_info.value.__cause__, OSError) - assert "nope" in str(exc_info.value.__cause__) + assert isinstance(exc_info.value.__cause__, ExceptionGroup) + for subexc in exc_info.value.__cause__.exceptions: + assert "nope" in str(subexc) else: listeners = await open_tcp_listeners(80) for listener in listeners: From ca05efb127928d6f708fe870dac0b6693cc371e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 25 Jan 2022 19:20:54 +0200 Subject: [PATCH 0893/1498] Removed one last MultiError reference in docs --- docs/source/history.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/history.rst b/docs/source/history.rst index a8fd584675..450817bb77 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -457,7 +457,7 @@ Bugfixes :meth:`~trio.Path.cwd`, are now async functions. Previously, a bug in the forwarding logic meant :meth:`~trio.Path.cwd` was synchronous and :meth:`~trio.Path.home` didn't work at all. (`#960 `__) -- An exception encapsulated within a `MultiError` doesn't need to be +- An exception encapsulated within a ``MultiError`` doesn't need to be hashable anymore. .. note:: From 0b159e8e83ec0ae0ee6299bc11bb1436073c4658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 25 Jan 2022 19:23:51 +0200 Subject: [PATCH 0894/1498] Found one more MultiError, in a news fragment --- newsfragments/2063.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/2063.bugfix.rst b/newsfragments/2063.bugfix.rst index 85fe7aa52b..a0f16db8fb 100644 --- a/newsfragments/2063.bugfix.rst +++ b/newsfragments/2063.bugfix.rst @@ -1,3 +1,3 @@ -Trio now avoids creating cyclic garbage when a `MultiError` is generated and filtered, +Trio now avoids creating cyclic garbage when a ``MultiError`` is generated and filtered, including invisibly within the cancellation system. This means errors raised through nurseries and cancel scopes should result in less GC latency. \ No newline at end of file From e9aaabc55548b5a1c75003a838bb94b660376abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 26 Jan 2022 01:52:52 +0200 Subject: [PATCH 0895/1498] Added a test for exception group collapsing --- trio/_core/tests/test_run.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 0c563d5310..18dd4b2311 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1876,6 +1876,20 @@ def handle(exc): assert result == [[0, 0], [1, 1]] +async def test_nursery_collapse_exceptions(): + # Test that exception groups containing only a single exception are + # recursively collapsed + async def fail(): + raise ExceptionGroup("fail", [ValueError()]) + + try: + async with _core.open_nursery() as nursery: + nursery.start_soon(fail) + raise StopIteration + except ExceptionGroup as e: + assert tuple(map(type, e.exceptions)) == (StopIteration, ValueError) + + async def test_traceback_frame_removal(): async def my_child_task(): raise KeyError() From 0e5b8160d4afb8a452ef8d2083634ca3bae433e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 05:40:48 +0000 Subject: [PATCH 0896/1498] Bump typing-extensions from 3.10.0.2 to 4.0.1 Bumps [typing-extensions](https://github.com/python/typing) from 3.10.0.2 to 4.0.1. - [Release notes](https://github.com/python/typing/releases) - [Changelog](https://github.com/python/typing/blob/master/typing_extensions/CHANGELOG) - [Commits](https://github.com/python/typing/commits) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c46460852a..5d32e278df 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -130,7 +130,7 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typing-extensions==3.10.0.2 ; implementation_name == "cpython" +typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in # black From 13e554d19633c3943b5619dea1dc317b789224b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 05:40:50 +0000 Subject: [PATCH 0897/1498] Bump py from 1.10.0 to 1.11.0 Bumps [py](https://github.com/pytest-dev/py) from 1.10.0 to 1.11.0. - [Release notes](https://github.com/pytest-dev/py/releases) - [Changelog](https://github.com/pytest-dev/py/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/py/compare/1.10.0...1.11.0) --- updated-dependencies: - dependency-name: py dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c46460852a..5e737084b9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -85,7 +85,7 @@ prompt-toolkit==3.0.21 # via ipython ptyprocess==0.7.0 # via pexpect -py==1.10.0 +py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 From aa39085121c9bedf86b8b3eb61f35ecc88a050a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 05:41:01 +0000 Subject: [PATCH 0898/1498] Bump regex from 2021.11.1 to 2022.1.18 Bumps [regex](https://github.com/mrabarnett/mrab-regex) from 2021.11.1 to 2022.1.18. - [Release notes](https://github.com/mrabarnett/mrab-regex/releases) - [Commits](https://github.com/mrabarnett/mrab-regex/commits) --- updated-dependencies: - dependency-name: regex dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c46460852a..f4dd3bd4d9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -107,7 +107,7 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -regex==2021.11.1 +regex==2022.1.18 # via black six==1.16.0 # via pyopenssl From b52504109aa9d06e1653a209f832252da4cb66f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 06:21:26 +0000 Subject: [PATCH 0899/1498] Bump mypy from 0.910 to 0.931 Bumps [mypy](https://github.com/python/mypy) from 0.910 to 0.931. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.910...v0.931) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index c46460852a..f4d992719d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -56,7 +56,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.910 ; implementation_name == "cpython" +mypy==0.931 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via @@ -107,7 +107,7 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -regex==2021.11.1 +regex==2022.1.18 # via black six==1.16.0 # via pyopenssl @@ -117,13 +117,13 @@ sortedcontainers==2.4.0 # via -r test-requirements.in toml==0.10.2 # via - # mypy # pylint # pytest tomli==1.2.2 # via # black # coverage + # mypy traitlets==5.1.1 # via # ipython From 40618a0bda13af12a2d839d3e68b1d2685da5db9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 06:21:51 +0000 Subject: [PATCH 0900/1498] Bump parso from 0.8.2 to 0.8.3 Bumps [parso](https://github.com/davidhalter/parso) from 0.8.2 to 0.8.3. - [Release notes](https://github.com/davidhalter/parso/releases) - [Changelog](https://github.com/davidhalter/parso/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/parso/compare/v0.8.2...v0.8.3) --- updated-dependencies: - dependency-name: parso dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index c46460852a..702f49eca1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ outcome==1.1.0 # via -r test-requirements.in packaging==21.2 # via pytest -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -107,7 +107,7 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -regex==2021.11.1 +regex==2022.1.18 # via black six==1.16.0 # via pyopenssl From 8056b18a08ead1f042288599ee5a852766ce35db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 06:22:47 +0000 Subject: [PATCH 0901/1498] Bump tomli from 1.2.2 to 1.2.3 Bumps [tomli](https://github.com/hukkin/tomli) from 1.2.2 to 1.2.3. - [Release notes](https://github.com/hukkin/tomli/releases) - [Changelog](https://github.com/hukkin/tomli/blob/master/CHANGELOG.md) - [Commits](https://github.com/hukkin/tomli/compare/1.2.2...1.2.3) --- updated-dependencies: - dependency-name: tomli dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index c46460852a..d96d445d24 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -107,7 +107,7 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -regex==2021.11.1 +regex==2022.1.18 # via black six==1.16.0 # via pyopenssl @@ -120,7 +120,7 @@ toml==0.10.2 # mypy # pylint # pytest -tomli==1.2.2 +tomli==1.2.3 # via # black # coverage From ce7b3a984904593a3817a6d94194538718a530df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 06:23:19 +0000 Subject: [PATCH 0902/1498] Bump prompt-toolkit from 3.0.21 to 3.0.24 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.21 to 3.0.24. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.21...3.0.24) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 593ad7e727..9362c10dec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -81,7 +81,7 @@ platformdirs==2.4.0 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.21 +prompt-toolkit==3.0.24 # via ipython ptyprocess==0.7.0 # via pexpect From d891435a72e8db2ba3f78fa21cd2ad83456887da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 07:29:11 +0000 Subject: [PATCH 0903/1498] Bump packaging from 21.2 to 21.3 Bumps [packaging](https://github.com/pypa/packaging) from 21.2 to 21.3. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/21.2...21.3) --- updated-dependencies: - dependency-name: packaging dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index b9c1f3d79f..afcc93f80d 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -46,7 +46,7 @@ markupsafe==2.0.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in -packaging==21.2 +packaging==21.3 # via sphinx pygments==2.10.0 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index f4dd3bd4d9..dd12df8f1f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -65,7 +65,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # mypy outcome==1.1.0 # via -r test-requirements.in -packaging==21.2 +packaging==21.3 # via pytest parso==0.8.2 # via jedi @@ -85,7 +85,7 @@ prompt-toolkit==3.0.21 # via ipython ptyprocess==0.7.0 # via pexpect -py==1.10.0 +py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 @@ -120,7 +120,7 @@ toml==0.10.2 # mypy # pylint # pytest -tomli==1.2.2 +tomli==1.2.3 # via # black # coverage From ab216efcdf490b814e369feebe2efd23c2b7a594 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 07:29:37 +0000 Subject: [PATCH 0904/1498] Bump charset-normalizer from 2.0.7 to 2.0.10 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.7 to 2.0.10. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.7...2.0.10) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index b9c1f3d79f..3a68798624 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.10.8 # via requests -charset-normalizer==2.0.7 +charset-normalizer==2.0.10 # via requests click==8.0.3 # via From 0d2684b006adfb1198a571dbe53412d027023587 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 07:30:48 +0000 Subject: [PATCH 0905/1498] Bump pycparser from 2.20 to 2.21 Bumps [pycparser](https://github.com/eliben/pycparser) from 2.20 to 2.21. - [Release notes](https://github.com/eliben/pycparser/releases) - [Changelog](https://github.com/eliben/pycparser/blob/master/CHANGES) - [Commits](https://github.com/eliben/pycparser/compare/release_v2.20...release_v2.21) --- updated-dependencies: - dependency-name: pycparser dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index c9fe505b1e..f956967881 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -89,7 +89,7 @@ py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 -pycparser==2.20 +pycparser==2.21 # via cffi pyflakes==2.4.0 # via flake8 From 558f2fe25c82a12d8d2cf8b8707fe59a21e22262 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 07:30:52 +0000 Subject: [PATCH 0906/1498] Bump snowballstemmer from 2.1.0 to 2.2.0 Bumps [snowballstemmer](https://github.com/snowballstem/snowball) from 2.1.0 to 2.2.0. - [Release notes](https://github.com/snowballstem/snowball/releases) - [Changelog](https://github.com/snowballstem/snowball/blob/master/NEWS) - [Commits](https://github.com/snowballstem/snowball/compare/v2.1.0...v2.2.0) --- updated-dependencies: - dependency-name: snowballstemmer dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index b9c1f3d79f..83ca0709be 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -58,7 +58,7 @@ requests==2.26.0 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in -snowballstemmer==2.1.0 +snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via -r docs-requirements.in From 5a42fe34a0f6a0949c481aa790874312bdf128a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 08:07:13 +0000 Subject: [PATCH 0907/1498] Bump jinja2 from 3.0.2 to 3.0.3 Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.2...3.0.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index afcc93f80d..efafcd6924 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -38,7 +38,7 @@ immutables==0.16 # via -r docs-requirements.in incremental==21.3.0 # via towncrier -jinja2==3.0.2 +jinja2==3.0.3 # via # sphinx # towncrier From 5206e15b9224b31f9a65119155dfe7bd6fb91d49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 08:07:18 +0000 Subject: [PATCH 0908/1498] Bump jedi from 0.18.0 to 0.18.1 Bumps [jedi](https://github.com/davidhalter/jedi) from 0.18.0 to 0.18.1. - [Release notes](https://github.com/davidhalter/jedi/releases) - [Changelog](https://github.com/davidhalter/jedi/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/jedi/compare/v0.18.0...v0.18.1) --- updated-dependencies: - dependency-name: jedi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2dbe472d96..0dae91a667 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -44,7 +44,7 @@ ipython==7.29.0 # via -r test-requirements.in isort==5.9.3 # via pylint -jedi==0.18.0 +jedi==0.18.1 # via # -r test-requirements.in # ipython From 0e361e203aef49fe5a952c54bbb29c0b5cbd7217 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 08:08:25 +0000 Subject: [PATCH 0909/1498] Bump urllib3 from 1.26.7 to 1.26.8 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.7 to 1.26.8. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.7...1.26.8) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index b9c1f3d79f..98dfac4402 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -46,7 +46,7 @@ markupsafe==2.0.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in -packaging==21.2 +packaging==21.3 # via sphinx pygments==2.10.0 # via sphinx @@ -87,5 +87,5 @@ toml==0.10.2 # via towncrier towncrier==21.3.0 # via -r docs-requirements.in -urllib3==1.26.7 +urllib3==1.26.8 # via requests From 51384db1f22419f707fadcef4bc6d3fdb75d804a Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 26 Jan 2022 15:51:57 +0400 Subject: [PATCH 0910/1498] Bump ipython to 7.31.1 --- test-requirements.in | 3 ++- test-requirements.txt | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 99854204d6..4d15de8e4f 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -1,7 +1,8 @@ # For tests pytest >= 5.0 # for faulthandler in core pytest-cov >= 2.6.0 -ipython # for the IPython traceback integration tests +# ipython 7.x is the last major version supporting Python 3.7 +ipython ~= 7.31 # for the IPython traceback integration tests pyOpenSSL # for the ssl tests trustme # for the ssl tests pylint # for pylint finding all symbols tests diff --git a/test-requirements.txt b/test-requirements.txt index 2dbe472d96..62a96935e8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -38,9 +38,15 @@ idna==3.3 # via # -r test-requirements.in # trustme +importlib-metadata==4.2.0 + # via + # click + # flake8 + # pluggy + # pytest iniconfig==1.1.1 # via pytest -ipython==7.29.0 +ipython==7.31.1 # via -r test-requirements.in isort==5.9.3 # via pylint @@ -130,15 +136,26 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in +typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" + # via + # -r test-requirements.in + # astroid + # black + # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid # black + # importlib-metadata # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid +zipp==3.7.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From e8da5e6e022e8c10a91da98ddc85a0e468730f01 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:21:58 +0000 Subject: [PATCH 0911/1498] Bump pyparsing from 2.4.7 to 3.0.7 Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 2.4.7 to 3.0.7. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_2.4.7...pyparsing_3.0.7) --- updated-dependencies: - dependency-name: pyparsing dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- test-requirements.txt | 21 ++------------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index afcc93f80d..11da0e9b58 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -38,7 +38,7 @@ immutables==0.16 # via -r docs-requirements.in incremental==21.3.0 # via towncrier -jinja2==3.0.2 +jinja2==3.0.3 # via # sphinx # towncrier @@ -50,7 +50,7 @@ packaging==21.3 # via sphinx pygments==2.10.0 # via sphinx -pyparsing==2.4.7 +pyparsing==3.0.7 # via packaging pytz==2021.3 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index 62a96935e8..bb7dea56a4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -38,19 +38,13 @@ idna==3.3 # via # -r test-requirements.in # trustme -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 # via -r test-requirements.in isort==5.9.3 # via pylint -jedi==0.18.0 +jedi==0.18.1 # via # -r test-requirements.in # ipython @@ -105,7 +99,7 @@ pylint==2.12.2 # via -r test-requirements.in pyopenssl==21.0.0 # via -r test-requirements.in -pyparsing==2.4.7 +pyparsing==3.0.7 # via packaging pytest==6.2.5 # via @@ -136,26 +130,15 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -zipp==3.7.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 441f2b0587970abda4215a415eb5968528935271 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:23:49 +0000 Subject: [PATCH 0912/1498] Bump isort from 5.9.3 to 5.10.1 Bumps [isort](https://github.com/pycqa/isort) from 5.9.3 to 5.10.1. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.9.3...5.10.1) --- updated-dependencies: - dependency-name: isort dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 0bb10ce649..3e2a17ac42 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -38,17 +38,11 @@ idna==3.3 # via # -r test-requirements.in # trustme -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 # via -r test-requirements.in -isort==5.9.3 +isort==5.10.1 # via pylint jedi==0.18.1 # via @@ -136,26 +130,15 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -zipp==3.7.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 6a5a0dce6b22ec0e64ec070924c649bff0beef76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:23:57 +0000 Subject: [PATCH 0913/1498] Bump lazy-object-proxy from 1.6.0 to 1.7.1 Bumps [lazy-object-proxy](https://github.com/ionelmc/python-lazy-object-proxy) from 1.6.0 to 1.7.1. - [Release notes](https://github.com/ionelmc/python-lazy-object-proxy/releases) - [Changelog](https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-lazy-object-proxy/compare/v1.6.0...v1.7.1) --- updated-dependencies: - dependency-name: lazy-object-proxy dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 62a96935e8..f8bf9a09ae 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -38,23 +38,17 @@ idna==3.3 # via # -r test-requirements.in # trustme -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 # via -r test-requirements.in isort==5.9.3 # via pylint -jedi==0.18.0 +jedi==0.18.1 # via # -r test-requirements.in # ipython -lazy-object-proxy==1.6.0 +lazy-object-proxy==1.7.1 # via astroid matplotlib-inline==0.1.3 # via ipython @@ -136,26 +130,15 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -zipp==3.7.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 64d68304fcc7e3ad2eb7b488b682f91a1ed844c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:24:12 +0000 Subject: [PATCH 0914/1498] Bump requests from 2.26.0 to 2.27.1 Bumps [requests](https://github.com/psf/requests) from 2.26.0 to 2.27.1. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.26.0...v2.27.1) --- updated-dependencies: - dependency-name: requests dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 0e5acf365d..26afb66946 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -54,7 +54,7 @@ pyparsing==2.4.7 # via packaging pytz==2021.3 # via babel -requests==2.26.0 +requests==2.27.1 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in From db35bd5b8a0fdd004a8c0a1a24fd52e5f252ad1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:24:26 +0000 Subject: [PATCH 0915/1498] Bump imagesize from 1.2.0 to 1.3.0 Bumps [imagesize](https://github.com/shibukawa/imagesize_py) from 1.2.0 to 1.3.0. - [Release notes](https://github.com/shibukawa/imagesize_py/releases) - [Commits](https://github.com/shibukawa/imagesize_py/compare/1.2.0...1.3.0) --- updated-dependencies: - dependency-name: imagesize dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 0e5acf365d..cc787006c2 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -32,7 +32,7 @@ idna==3.3 # via # -r docs-requirements.in # requests -imagesize==1.2.0 +imagesize==1.3.0 # via sphinx immutables==0.16 # via -r docs-requirements.in From 49a857f02072b6818354a70c5af7d24b34b6a0d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:25:10 +0000 Subject: [PATCH 0916/1498] Bump platformdirs from 2.4.0 to 2.4.1 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.4.0 to 2.4.1. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.4.0...2.4.1) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 62a96935e8..38181d89ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -38,19 +38,13 @@ idna==3.3 # via # -r test-requirements.in # trustme -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 # via -r test-requirements.in isort==5.9.3 # via pylint -jedi==0.18.0 +jedi==0.18.1 # via # -r test-requirements.in # ipython @@ -81,7 +75,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.0 +platformdirs==2.4.1 # via # black # pylint @@ -136,26 +130,15 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -zipp==3.7.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 8e6b360884b4131fb37dc4dcb705e23a08d7c8f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:26:30 +0000 Subject: [PATCH 0917/1498] Bump decorator from 5.1.0 to 5.1.1 Bumps [decorator](https://github.com/micheles/decorator) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/micheles/decorator/releases) - [Changelog](https://github.com/micheles/decorator/blob/master/CHANGES.md) - [Commits](https://github.com/micheles/decorator/commits/5.1.1) --- updated-dependencies: - dependency-name: decorator dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 62a96935e8..c8d0e73bb2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -30,7 +30,7 @@ cryptography==36.0.1 # -r test-requirements.in # pyopenssl # trustme -decorator==5.1.0 +decorator==5.1.1 # via ipython flake8==4.0.1 # via -r test-requirements.in @@ -38,19 +38,13 @@ idna==3.3 # via # -r test-requirements.in # trustme -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 # via -r test-requirements.in isort==5.9.3 # via pylint -jedi==0.18.0 +jedi==0.18.1 # via # -r test-requirements.in # ipython @@ -136,26 +130,15 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -zipp==3.7.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From eff01e31c9f56d2bdb8f59c345f5b479ce68c28b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jan 2022 06:06:52 +0000 Subject: [PATCH 0918/1498] Bump attrs from 21.2.0 to 21.4.0 Bumps [attrs](https://github.com/python-attrs/attrs) from 21.2.0 to 21.4.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/21.2.0...21.4.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 6 +++--- test-requirements.txt | 23 +++-------------------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index efafcd6924..2415d9bafc 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in -attrs==21.2.0 +attrs==21.4.0 # via # -r docs-requirements.in # outcome @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.10.8 # via requests -charset-normalizer==2.0.7 +charset-normalizer==2.0.10 # via requests click==8.0.3 # via @@ -87,5 +87,5 @@ toml==0.10.2 # via towncrier towncrier==21.3.0 # via -r docs-requirements.in -urllib3==1.26.7 +urllib3==1.26.8 # via requests diff --git a/test-requirements.txt b/test-requirements.txt index 0ba5ba35ef..5941cb3148 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,7 @@ astroid==2.9.3 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==21.2.0 +attrs==21.4.0 # via # -r test-requirements.in # outcome @@ -38,12 +38,6 @@ idna==3.3 # via # -r test-requirements.in # trustme -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 @@ -81,7 +75,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.0 +platformdirs==2.4.1 # via # black # pylint @@ -95,7 +89,7 @@ py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 -pycparser==2.20 +pycparser==2.21 # via cffi pyflakes==2.4.0 # via flake8 @@ -136,26 +130,15 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.4.3 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid # black - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -zipp==3.7.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 1d500c2a48241bda3113e518176b132b8acd1676 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jan 2022 06:07:59 +0000 Subject: [PATCH 0919/1498] Bump black from 21.10b0 to 21.12b0 Bumps [black](https://github.com/psf/black) from 21.10b0 to 21.12b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4f017c4e38..f6b1b6c12f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==21.2.0 # pytest backcall==0.2.0 # via ipython -black==21.10b0 ; implementation_name == "cpython" +black==21.12b0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography @@ -107,8 +107,6 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -regex==2022.1.18 - # via black six==1.16.0 # via pyopenssl sniffio==1.2.0 From b5dae091495a705dc03a74b1086abd386299a7af Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 27 Jan 2022 13:49:49 +0400 Subject: [PATCH 0920/1498] Apply black 21.12b0 --- trio/_core/_multierror.py | 1 - trio/_core/tests/tutil.py | 1 - trio/_subprocess_platform/waitid.py | 1 - 3 files changed, 3 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 514f8764c8..6dfdaa7a52 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -288,7 +288,6 @@ def controller(operation): return tputil.make_proxy(controller, type(base_tb), base_tb) - else: # ctypes it is import ctypes diff --git a/trio/_core/tests/tutil.py b/trio/_core/tests/tutil.py index 3e8053f71f..016e0fd3e1 100644 --- a/trio/_core/tests/tutil.py +++ b/trio/_core/tests/tutil.py @@ -108,7 +108,6 @@ def disable_threading_excepthook(): finally: threading.excepthook = prev - else: @contextmanager diff --git a/trio/_subprocess_platform/waitid.py b/trio/_subprocess_platform/waitid.py index 91ba224546..ad69017219 100644 --- a/trio/_subprocess_platform/waitid.py +++ b/trio/_subprocess_platform/waitid.py @@ -13,7 +13,6 @@ def sync_wait_reapable(pid): waitid(os.P_PID, pid, os.WEXITED | os.WNOWAIT) - except ImportError: # pypy doesn't define os.waitid so we need to pull it out ourselves # using cffi: https://bitbucket.org/pypy/pypy/issues/2922/ From 4309dcd4636aa4110294c59f15842e3146feb96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Thu, 27 Jan 2022 22:02:30 +0200 Subject: [PATCH 0921/1498] Update newsfragments/2160.feature.rst --- newsfragments/2160.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst index 9bb4f6ae44..1bca734ed1 100644 --- a/newsfragments/2160.feature.rst +++ b/newsfragments/2160.feature.rst @@ -1,4 +1,4 @@ -Now context variables set with `contextvars` are preserved when runing functions +Now context variables set with `contextvars` are preserved when running functions in a worker thread with `trio.to_thread.run_sync`, or when running functions from the worker thread in the parent Trio thread with `trio.from_thread.run`, and `trio.from_thread.run_sync`. From c965af0c7d205ed5b90e1de000b54d6248f3fd2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jan 2022 10:19:55 +0000 Subject: [PATCH 0922/1498] Bump prompt-toolkit from 3.0.24 to 3.0.26 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.24 to 3.0.26. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.24...3.0.26) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index affebe22df..aa32adab96 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -81,7 +81,7 @@ platformdirs==2.4.1 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.24 +prompt-toolkit==3.0.26 # via ipython ptyprocess==0.7.0 # via pexpect From cd0ec9f43080f1c9f0e5a9df65e8a7a271ff15e6 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 24 Jun 2021 09:16:43 -0700 Subject: [PATCH 0923/1498] First draft --- newsfragments/2010.feature.rst | 4 + trio/_dtls.py | 854 +++++++++++++++++++++++++++++++++ 2 files changed, 858 insertions(+) create mode 100644 newsfragments/2010.feature.rst create mode 100644 trio/_dtls.py diff --git a/newsfragments/2010.feature.rst b/newsfragments/2010.feature.rst new file mode 100644 index 0000000000..f99687652f --- /dev/null +++ b/newsfragments/2010.feature.rst @@ -0,0 +1,4 @@ +Added support for `Datagram TLS +`__, +for secure communication over UDP. Currently requires `PyOpenSSL +`__. diff --git a/trio/_dtls.py b/trio/_dtls.py new file mode 100644 index 0000000000..b825fea5d6 --- /dev/null +++ b/trio/_dtls.py @@ -0,0 +1,854 @@ +# https://datatracker.ietf.org/doc/html/rfc6347 + +# XX: figure out what to do about the pyopenssl dependency +# Maybe the toplevel __init__.py should use __getattr__ trickery to load all +# the DTLS code lazily? + +import struct +import hmac +import os +import io +import enum +from itertools import count +import weakref + +import attr +from OpenSSL import SSL + +import trio +from trio._util import NoPublicConstructor + +MAX_UDP_PACKET_SIZE = 65527 + +# There are a bunch of different RFCs that define these codes, so for a +# comprehensive collection look here: +# https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml +class ContentType(enum.IntEnum): + change_cipher_spec = 20 + alert = 21 + handshake = 22 + application_data = 23 + heartbeat = 24 + + +class HandshakeType(enum.IntEnum): + hello_request = 0 + client_hello = 1 + server_hello = 2 + hello_verify_request = 3 + new_session_ticket = 4 + end_of_early_data = 4 + encrypted_extensions = 8 + certificate = 11 + server_key_exchange = 12 + certificate_request = 13 + server_hello_done = 14 + certificate_verify = 15 + client_key_exchange = 16 + finished = 20 + certificate_url = 21 + certificate_status = 22 + supplemental_data = 23 + key_update = 24 + compressed_certificate = 25 + ekt_key = 26 + message_hash = 254 + + +class ProtocolVersion: + DTLS10 = bytes([254, 255]) + DTLS12 = bytes([254, 253]) + + +EPOCH_MASK = 0xffff << (6 * 8) + + +# Conventions: +# - All functions that handle network data end in _untrusted. +# - All functions end in _untrusted MUST make sure that bad data from the +# network cannot *only* cause BadPacket to be raised. No IndexError or +# struct.error or whatever. +class BadPacket(Exception): + pass + + +# This checks that the DTLS 'epoch' field is 0, which is true iff we're in the +# initial handshake. It doesn't check the ContentType, because not all +# handshake messages have ContentType==handshake -- for example, +# ChangeCipherSpec is used during the handshake but has its own ContentType. +# +# Cannot fail. +def part_of_handshake_untrusted(packet): + # If the packet is too short, then slicing will successfully return a + # short string, which will necessarily fail to match. + return packet[3:5] == b"\x00\x00" + + +# Cannot fail +def is_client_hello_untrusted(packet): + try: + return ( + packet[0] == ContentType.handshake + and packet[13] == HandshakeType.client_hello + ) + except IndexError: + # Invalid DTLS record + return False + + +# DTLS records are: +# - 1 byte content type +# - 2 bytes version +# - 8 bytes epoch+seqno +# Technically this is 2 bytes epoch then 6 bytes seqno, but we treat it as +# a single 8-byte integer, where epoch changes are represented as jumping +# forward by 2**(6*8). +# - 2 bytes payload length (unsigned big-endian) +# - payload +RECORD_HEADER = struct.Struct("!B2sQH") + + +@attr.frozen +class Record: + content_type: int + version: bytes + epoch_seqno: int + payload: bytes + + +def records_untrusted(packet): + i = 0 + while i < len(packet): + try: + ct, version, epoch_seqno, payload_len = RECORD_HEADER.unpack_from(packet, i) + except struct.error as exc: + raise BadPacket("invalid record header") from exc + i += RECORD_HEADER.size + payload = packet[i : i + payload_len] + if len(payload) != payload_len: + raise BadPacket("short record") + i += payload_len + yield Record(ct, version, epoch_seqno, payload) + + +def encode_record(record): + header = RECORD_HEADER.pack( + record.content_type, + record.version, + record.epoch_seqno, + len(record.payload), + ) + return header + record.payload + + +# Handshake messages are: +# - 1 byte message type +# - 3 bytes total message length +# - 2 bytes message sequence number +# - 3 bytes fragment offset +# - 3 bytes fragment length +HANDSHAKE_MESSAGE_HEADER = struct.Struct("!B3sH3s3s") + + +@attr.frozen +class HandshakeFragment: + msg_type: int + msg_len: int, + msg_seq: int + frag_offset: int + frag_len: int + frag: bytes + + +def decode_handshake_fragment_untrusted(payload): + # Raises BadPacket if decoding fails + try: + ( + msg_type, + msg_len_bytes, + msg_seq, + frag_offset_bytes, + frag_len_bytes, + ) = HANDSHAKE_MESSAGE_HEADER.unpack_from(payload) + except struct.error as exc: + raise BadPacket("bad handshake message header") from exc + # 'struct' doesn't have built-in support for 24-bit integers, so we + # have to do it by hand. These can't fail. + msg_len = int.from_bytes(msg_len_bytes, "big") + frag_offset = int.from_bytes(frag_offset_bytes, "big") + frag_len = int.from_bytes(frag_len_bytes, "big") + frag = payload[HANDSHAKE_MESSAGE_HEADER.size:] + if len(frag) != frag_len: + raise BadPacket("short fragment") + return HandshakeFragment( + msg_type, + msg_len, + msg_seq, + frag_offset, + frag_len, + frag, + ) + + +def encode_handshake_fragment(hsf): + hs_header = HANDSHAKE_MESSAGE_HEADER.pack( + hsf.msg_type, + hsf.msg_len.to_bytes(3, "big"), + hsf.msg_seq, + hsf.frag_offset.to_bytes(3, "big"), + hsf.frag_len.to_bytes(3, "big"), + ) + return hs_header + hsf.body + + +def decode_client_hello_untrusted(packet): + # Raises BadPacket if parsing fails + # Returns (record epoch_seqno, cookie from the packet, data that should be + # hashed into cookie) + try: + # ClientHello has to be the first record in the packet + record = next(records_untrusted(packet)) + if record.content_type != ContentType.handshake: + raise BadPacket("not a handshake record") + fragment = decode_handshake_fragment_untrusted(record.payload) + if fragment.msg_type != HandshakeType.client_hello: + raise BadPacket("not a ClientHello") + # ClientHello can't be fragmented, because reassembly requires holding + # per-connection state, and we refuse to allocate per-connection state + # until after we get a valid ClientHello. + if fragment.frag_offset != 0: + raise BadPacket("fragmented ClientHello") + if fragment.frag_len != fragment.msg_len: + raise BadPacket("fragmented ClientHello") + + # As per RFC 6347: + # + # When responding to a HelloVerifyRequest, the client MUST use the + # same parameter values (version, random, session_id, cipher_suites, + # compression_method) as it did in the original ClientHello. The + # server SHOULD use those values to generate its cookie and verify that + # they are correct upon cookie receipt. + # + # However, the record-layer framing can and will change (e.g. the + # second ClientHello will have a new record-layer sequence number). So + # we need to pull out the handshake message alone, discarding the + # record-layer stuff, and then we're going to hash all of it *except* + # the cookie. + + body = fragment.frag + # ClientHello is: + # + # - 2 bytes client_version + # - 32 bytes random + # - 1 byte session_id length + # - session_id + # - 1 byte cookie length + # - cookie + # - everything else + # + # So to find the cookie, so we need to figure out how long the + # session_id is and skip past it. + session_id_len = body[2 + 32] + cookie_len_offset = 2 + 32 + 1 + session_id_len + cookie_len = body[cookie_len_offset] + + cookie_start = cookie_len_offset + 1 + cookie_end = cookie_start + cookie_len + + before_cookie = body[:cookie_start] + cookie = body[cookie_start:cookie_end] + after_cookie = body[cookie_end:] + + if len(cookie) != cookie_len: + raise BadPacket("short cookie") + return (record.epoch_seqno, cookie, before_cookie + after_cookie) + + except (struct.error, IndexError) as exc: + raise BadPacket("bad ClientHello") from exc + + +@attr.frozen +class HandshakeMessage: + record_version: bytes + msg_type: HandshakeType + msg_seq: int + body: bytearray + + +# ChangeCipherSpec is part of the handshake, but it's not a "handshake +# message" and can't be fragmented the same way. Sigh. +@attr.frozen +class PseudoHandshakeMessage: + record_version: bytes + content_type: int + payload: bytes + + +# This takes a raw outgoing handshake volley that openssl generated, and +# reconstructs the handshake messages inside it, so that we can repack them +# into records while retransmitting. So the data ought to be well-behaved -- +# it's not coming from the network. +def decode_volley_trusted(volley): + messages = [] + messages_by_seq = {} + for record in records_untrusted(volley): + if record.content_type == ContentType.change_cipher_spec: + messages.append(PseudoHandshakeMessage(record.version, + record.content_type, record.payload)) + else: + assert record.content_type == ContentType.handshake + fragment = decode_handshake_fragment_untrusted(record.payload) + msg_type = HandshakeType(fragment.msg_type) + if fragment.msg_seq not in messages_by_seq: + msg = HandshakeMessage( + record.version, msg_type, fragment.msg_seq, bytearray(fragment.msg_len) + ) + messages.append(msg) + messages_by_seq[fragment.msg_seq] = msg + else: + msg = messages_by_seq[msg_seq] + assert msg.msg_type == fragment.msg_type + assert msg.msg_seq == fragment.msg_seq + assert len(msg.body) == fragment.msg_len + + msg.body[fragment.frag_offset : fragment.frag_offset + fragment.frag_len] = fragment.frag + + return messages + + +class RecordEncoder: + def __init__(self, first_record_seq): + self._record_seq = count(first_record_seq) + + def skip_first_record_number(self): + assert next(self._record_seq) == 0 + + def encode_volley(self, messages, mtu): + packets = [] + packet = bytearray() + for message in messages: + if isinstance(message, PseudoHandshakeMessage): + space = mtu - len(packet) - RECORD_HEADER.size - len(message.payload) + if space <= 0: + packets.append(packet) + packet = bytearray() + packet += RECORD_HEADER.pack( + message.content_type, + message.record_version, + next(self._record_seq), + len(message.payload), + ) + packet += payload + assert len(packet) <= mtu + else: + msg_len_bytes = message.msg_len.to_bytes(3, "big") + frag_offset = 0 + while frag_offset < len(message.body): + space = mtu - len(packet) - RECORD_HEADER.size - HANDSHAKE_MESSAGE_HEADER.size + if space <= 0: + packets.append(packet) + packet = bytearray() + continue + frag = message.body[frag_offset:frag_offset + space] + frag_offset_bytes = frag_offset.to_bytes(3, "big") + frag_len_bytes = len(frag).to_bytes(3, "big") + + packet += RECORD_HEADER.pack( + ContentType.handshake, + message.record_version, + next(self._record_seq), + HANDSHAKE_MESSAGE_HEADER.size + len(frag), + ) + + packet += HANDSHAKE_MESSAGE_HEADER.pack( + message.msg_type, + msg_len_bytes, + message.msg_seq, + frag_offset_bytes, + frag_len_bytes, + ) + + packet += frag + + assert len(packet) <= mtu + + if packet: + packets.append(packet) + + return packets + + +# This bit requires implementing a bona fide cryptographic protocol, so even though it's +# a simple one let's take a moment to discuss the design. +# +# Our goal is to force new incoming handshakes that claim to be coming from a +# given ip:port to prove that they can also receive packets sent to that +# ip:port. (There's nothing in UDP to stop someone from forging the return +# address, and it's often used for stuff like DoS reflection attacks, where +# an attacker tries to trick us into sending data at some innocent victim.) +# For more details, see: +# +# https://datatracker.ietf.org/doc/html/rfc6347#section-4.2.1 +# +# To do this, when we receive an initial ClientHello, we calculate a magic +# cookie, and send it back as a HelloVerifyRequest. Then the client sends us a +# second ClientHello, this time with the magic cookie included, and after we +# check that this cookie is valid we go ahead and start the handshake proper. +# +# So the magic cookie needs the following properties: +# - No-one can forge it without knowing our secret key +# - It ensures that the ip, port, and ClientHello contents from the response +# match those in the challenge +# - It expires after a short-ish period (so that if an attacker manages to steal one, it +# won't be useful for long) +# - It doesn't require storing any peer-specific state on our side +# +# To do that, we take the ip/port/ClientHello data and compute an HMAC of them, using a +# secret key we generate on startup. We also include: +# +# - The current time (using Trio's clock), rounded to the nearest 30 seconds +# - A random salt +# +# Then the cookie the salt + the HMAC digest. +# +# When verifying a cookie, we use the salt + new ip/port/ClientHello data to recompute +# the HMAC digest, for both the current time and the current time minus 30 seconds, and +# if either of them match, we consider the cookie good. +# +# Including the rounded-off time like this means that each cookie is good for at least +# 30 seconds, and possibly as much as 60 seconds. +# +# The salt is probably not necessary -- I'm pretty sure that all it does is make it hard +# for an attacker to figure out when our clock ticks over a 30 second boundary. Which is +# probably pretty harmless? But it's easier to add the salt than to convince myself that +# it's *completely* harmless, so, salt it is. + +COOKIE_REFRESH_INTERVAL = 30 # seconds +KEY = None +KEY_BYTES = 8 +COOKIE_HASH = "sha256" +SALT_BYTES = 8 + + +def _current_cookie_tick(): + return int(trio.current_time() / COOKIE_REFRESH_INTERVAL) + + +# Simple deterministic, bijective serializer -- i.e., a useful tool for hashing +# structured data. +def _signable(*fields): + out = [] + for field in fields: + out.append(struct.encode("!Q", len(field))) + out.append(field) + return b"".join(out) + + +def _make_cookie(salt, tick, address, client_hello_bits): + assert len(salt) == SALT_BYTES + + global KEY + if KEY is None: + KEY = os.urandom(KEY_BYTES) + + signable_data = _signable( + salt, + struct.encode("!Q", tick), + # address is a mix of strings and ints, and variable length, so pack + # it into a single nested field + _signable(*(str(part).encode() for part in address)), + client_hello_bits, + ) + + return salt + hmac.digest(KEY, signable_data, COOKIE_HASH) + + +def valid_cookie(cookie, address, client_hello_bits): + if len(cookie) > SALT_BYTES: + salt = cookie[:SALT_BYTES] + + cur_cookie = _make_cookie(salt, tick, address, client_hello_bits) + old_cookie = _make_cookie(salt, tick - 1, address, client_hello_bits) + + return ( + hmac.compare_digest(cookie, cur_cookie) + | hmac.compare_digest(cookie, old_cookie) + ) + else: + return False + + +def challenge_for(address, epoch_seqno, client_hello_bits): + salt = os.urandom(SALT_BYTES) + tick = _current_cookie_tick() + cookie = _make_cookie(salt, tick, address, client_hello_bits) + + # HelloVerifyRequest body is: + # - 2 bytes version + # - length-prefixed cookie + # + # The DTLS 1.2 spec says that for this message specifically we should use + # the DTLS 1.0 version. + # + # (It also says the opposite of that, but that part is a mistake: + # https://www.rfc-editor.org/errata/eid4103 + # ). + # + # And I guess we use this for both the message-level and record-level + # ProtocolVersions, since we haven't negotiated anything else yet? + body = ProtocolVersion.DTLS10 + bytes([len(cookie)]) + cookie + + # RFC says have to copy the client's record number + # Errata says it should be handshake message number + # Openssl copies back record sequence number, and always sets message seq + # number 0. So I guess we'll follow openssl. + hs = HandshakeFragment( + msg_type=HandshakeType.hello_verify_request, + msg_len=len(body), + msg_seq=0, + frag_offset=0, + frag_len=len(body), + frag_bytes=body, + ) + payload = encode_handshake_fragment(hs) + + packet = encode_record(Record(ContentType.handshake, + ProtocolVersion.DTLS10, epoch_seqno, payload)) + return packet + + +# when listening, definitely needs background tasks to handle reading+handshaking +# connect() (add_peer()?) should probably handle handshake directly +# in *theory* if only had client-mode, then could have multiple tasks calling +# add_peer() simultaneously + +# +# alternatively: add_peer/listen as sync functions to express intentions, plus +# user is expected to regularly be pumping the receive loop. Handshakes don't +# progress unless you're doing this. Each packet receive call runs through the +# whole DTLS state machine. +# +# Problem: in this model, handshake timeouts are a pain in the butt. (Need to +# track them in some kind of manual timer wheel etc.). We don't +# want to write an explicit state machine; we want to take advantage of trio's +# tools. +# +# Therefore, handshakes should have a host task. +# +# And if handshakes have a host task, then what? +# +# main thing with handshakes is that might want to read packets to handle them +# even if user isn't otherwise reading packets. Our two options are to either +# have handshakes running continuously and potentially drop data packets +# internally, or else to only process handshakes while the user is asking for +# data. +# +# I guess dropping is better? We can have an internal queue size + stats on +# what's dropped? +# +# and if we're going to do our own dropping, then it's ok to have a constant +# reader task running... + +# One option would be to have a fully connection-based API. Make a DTLS socket +# endpoint, and then call `await connect(...)` to get a DTLS socket +# connection, or `serve(task_fn)` so `task_fn` runs on each incoming +# handshake. Lifetimes are then clear. (GC shutdown still a bit of a pain, but +# whatever, can handle it.) +# +# Alternatively, lean into the single socket API. Handshakes happen in +# background, peers are identified by magic tokens returned from 'connect' or +# 'receive'. Need some synthetic event for "new client connected" to avoid the +# possibility of clients filling up connection table with immortal nonsense. +# Need way to forget specific peers. +# +# In both cases: what to do when in server mode, and a new connection +# *replaces* an existing connection? (or in mixed client/server mode +# similarly, I guess?) I guess for single-socket API you need a way to get +# these notifications anyway... for server mode maybe the old one gets marked +# closed and a new one is created? so need some way to signal EOF, which isn't +# a thing otherwise. (BrokenResourceError?) +# +# Also for future-DTLS (and QUIC etc.) there's connection migration, where the +# peer address changes. I guess the user doesn't necessarily care about +# getting notifications of this, as long as their connection remains bound to +# the right peer? It does rule out the use of connected UDP sockets though. + + +class _Queue: + def __init__(self, incoming_packets_buffer): + self._s, self._r = trio.open_memory_channel(incoming_packets_buffer) + + async def put(self, obj): + await self._s.send(obj) + + async def get(self): + return self._r.receive() + + +def _read_loop(read_fn): + chunks = [] + while True: + try: + chunk = read_fn(2 ** 14) # max TLS record size + except SSL.WantReadError: + break + if not chunk: + break + chunks.append(chunk) + return b"".join(chunks) + + +async def handle_client_hello_untrusted(dtls, address, packet): + if dtls._listening_context is None: + return + + try: + epoch_seqno, cookie, bits = decode_client_hello_untrusted(address, packet) + except BadPacket: + return + + if not valid_cookie(cookie, address, bits): + challenge_packet = challenge_for(address, epoch_seqno, bits) + try: + await dtls.sock.sendto(address, challenge_packet) + except OSError: + pass + else: + stream = DTLSStream(dtls, address, dtls._listening_context) + stream._inject_client_hello(packet) + old_stream = dlts._streams.get(address) + if old_stream is not None: + old_stream._break(RuntimeError("peer started a new DTLS connection")) + dtls._streams[address] = stream + dtls._incoming_connections_q.put_nowait(stream) + + +async def dtls_receive_loop(dtls): + sock = dtls.socket + dtls_ref = weakref.weakref(dtls) + del dtls + while True: + try: + address, packet = await sock.recvfrom() + except ClosedResourceError: + return + except OSError as exc: + dtls = dtls_ref() + if dtls is None: + return + dtls._break(exc) + return + # All of the following is sync, so we can be confident that our + # reference to dtls remains valid. + dtls = dtls_ref() + try: + if dtls is None: + return + if is_client_hello_untrusted(packet): + await handle_client_hello_untrusted(dtls, address, packet) + elif address in dtls._streams: + stream = dtls._streams[address] + if stream._did_handshake and part_of_handshake_untrusted(packet): + # The peer just sent us more handshake messages, that aren't a + # ClientHello, and we thought the handshake was done. Some of the + # packets that we sent to finish the handshake must have gotten + # lost. So re-send them. We do this directly here instead of just + # putting it into the queue, because there's no guarantee that + # anyone is reading from the queue, because we think the handshake + # is done! + await stream._resend_final_volley() + else: + try: + stream._q.put_nowait(packet) + except trio.WouldBlock: + stream.packets_dropped_in_trio += 1 + else: + # Drop packet + pass + finally: + del dtls + + +class DTLSStream(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): + def __init__(self, dtls, peer_address, ctx): + self.dtls = dtls + self.peer_address = peer_address + self.packets_dropped_in_trio = 0 + self._mtu = 1472 # XX + self._did_handshake = False + self._ssl = SSL.Connection(ctx) + self._broken = False + self._closed = False + self._q = Queue(dtls.incoming_packets_buffer) + self._handshake_lock = trio.Lock() + self._record_encoder = RecordEncoder() + + def _break(self, reason: BaseException): + self._broken = True + self._broken_reason = reason + # XX wake things up + + def close(self): + if self._closed: + return + self._closed = True + if self.dtls._streams.get(self.peer_address) is self: + del self.dtls._streams[self.peer_address] + # Will wake any tasks waiting on self._q.get with a + # ClosedResourceError + self._q._r.close() + + async def aclose(self): + self.close() + await trio.lowlevel.checkpoint() + + def _inject_client_hello(self, packet): + stream._ssl.bio_write(packet) + # If we're on the server side, then we already sent record 0 as our cookie + # challenge. So we want to start the handshake proper with record 1. + self._record_encoder.skip_first_record_number() + + async def _send_volley(self, volley_messages): + packets = self._record_encoder(volley_messages, self._mtu) + for packet in packets: + await self.dtls.socket.sendto(self.peer_address, packet) + + async def _resend_final_volley(self): + await self._send_volley(self._final_volley) + + async def do_handshake(self): + async with self._handshake_lock: + if self._did_handshake: + return + # If we're a client, we send the initial volley. If we're a server, then + # the initial ClientHello has already been inserted into self._ssl's + # read BIO. So either way, we start by generating a new volley. + try: + self._ssl.do_handshake() + except SSL.WantReadError: + pass + + volley_messages = [] + def read_volley(): + volley_bytes = read_loop(self._ssl.bio_read) + new_volley_messages = decode_volley_trusted(volley_bytes) + if ( + new_volley_messages and volley_messages and + new_volley_messages[0].msg_seq == volley_messages[0].msg_seq + ): + # openssl decided to retransmit; discard because we'll handle this + # ourselves + return [] + else: + return new_volley_messages + + volley_messages = read_volley() + # If we don't have messages to send in our initial volley, then something + # has gone very wrong. (I'm not sure this can actually happen without an + # error from OpenSSL, but let's cover our bases.) + if not volley_messages: + self._break(SSL.Error("something wrong with peer's ClientHello")) + # XX raise + return + + while True: + assert volley_messages + await self._send_volley(volley_messages) + with trio.move_on_after(1) as cscope: + async for packet in self._q._r: + self._ssl.bio_write(packet) + try: + self._ssl.do_handshake() + except SSL.WantReadError: + pass + else: + # No exception -> the handshake is done, and we can + # switch into data transfer mode. + self._did_handshake = True + # Might be empty, but that's ok + self._final_volley = read_volley() + await self._send_volley(self._final_volley) + return + maybe_volley = read_volley() + if maybe_volley: + # We managed to get all of the peer's volley and generate a + # new one ourselves! break out of the 'for' loop and restart + # the timer. + volley_messages = new_volley + break + if cscope.cancelled_caught: + # timeout expired, adjust timeout/mtu + # Good guidance here: https://tlswg.org/dtls13-spec/draft-ietf-tls-dtls13.html#name-timer-values + XX + + async def send(self, data): + if not self._did_handshake: + await self.do_handshake() + self._ssl.write(data) + await self.dtls.socket.sendto(self.peer_address, read_loop(self._ssl.bio_read)) + + async def receive(self): + if not self._did_handshake: + await self.do_handshake() + packet = await self._q.get() + self._ssl.bio_write(packet) + return read_loop(self._ssl.read) + + +class DTLS: + def __init__(self, socket, *, incoming_packets_buffer=10): + if socket.type != trio.socket.SOCK_DGRAM: + raise BadPacket("DTLS requires a SOCK_DGRAM socket") + self.socket = socket + self.incoming_packets_buffer = incoming_packets_buffer + self._token = trio.lowlevel.current_trio_token() + # We don't need to track handshaking vs non-handshake connections + # separately. We only keep one connection per remote address; as soon + # as a peer provides a valid cookie, we can immediately tear down the + # old connection. + # {remote address: DTLSStream} + self._streams = weakref.WeakValueDictionary() + self._listening_context = None + self._incoming_connections_q = Queue(float("inf")) + + trio.lowlevel.spawn_system_task(dtls_receive_loop, self) + + def __del__(self): + # Close the socket in Trio context (if our Trio context still exists), so that + # the background task gets notified about the closure and can exit. + try: + self._token.run_sync_soon(self.socket.close) + except RuntimeError: + pass + + def close(self): + self.socket.close() + for stream in self._streams.values(): + stream.close() + self._incoming_connections_q._s.close() + + async def aclose(self): + self.close() + await trio.lowlevel.checkpoint() + + async def serve(self, ssl_context, async_fn, *args): + if self._listening_context is not None: + raise trio.BusyResourceError("another task is already listening") + try: + self._listening_context = ssl_context + async with trio.open_nursery() as nursery: + async for stream in self._incoming_connections_q._r: + nursery.start_soon(async_fn, stream, *args) + finally: + self._listening_context = None + + def _set_stream_for(self, address, stream): + old_stream = self._streams.get(address) + if old_stream is not None: + old_stream._break(RuntimeError("replaced by a new DTLS association")) + self._streams[address] = stream + + async def connect(self, address, ssl_context): + stream = DTLSStream(self, address, ssl_context) + self._set_stream_for(address, stream) + await stream.do_handshake() + return stream From 8c2fafa98100ad2a5ae038eb8f70092816a0474e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 25 Jun 2021 02:55:56 -0700 Subject: [PATCH 0924/1498] smoke test passing! --- trio/_dtls.py | 291 ++++++++++++++++++++++------------------ trio/tests/test_dtls.py | 40 ++++++ 2 files changed, 203 insertions(+), 128 deletions(-) create mode 100644 trio/tests/test_dtls.py diff --git a/trio/_dtls.py b/trio/_dtls.py index b825fea5d6..4b738581e5 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -153,7 +153,7 @@ def encode_record(record): @attr.frozen class HandshakeFragment: msg_type: int - msg_len: int, + msg_len: int msg_seq: int frag_offset: int frag_len: int @@ -179,7 +179,7 @@ def decode_handshake_fragment_untrusted(payload): frag_len = int.from_bytes(frag_len_bytes, "big") frag = payload[HANDSHAKE_MESSAGE_HEADER.size:] if len(frag) != frag_len: - raise BadPacket("short fragment") + raise BadPacket("handshake fragment length doesn't match record length") return HandshakeFragment( msg_type, msg_len, @@ -198,7 +198,7 @@ def encode_handshake_fragment(hsf): hsf.frag_offset.to_bytes(3, "big"), hsf.frag_len.to_bytes(3, "big"), ) - return hs_header + hsf.body + return hs_header + hsf.frag def decode_client_hello_untrusted(packet): @@ -255,7 +255,7 @@ def decode_client_hello_untrusted(packet): cookie_start = cookie_len_offset + 1 cookie_end = cookie_start + cookie_len - before_cookie = body[:cookie_start] + before_cookie = body[:cookie_len_offset] cookie = body[cookie_start:cookie_end] after_cookie = body[cookie_end:] @@ -284,6 +284,16 @@ class PseudoHandshakeMessage: payload: bytes +# The final record in a handshake is Finished, which is encrypted, can't be fragmented +# (at least by us), and keeps its record number (because it's in a new epoch). So we +# just pass it through unchanged. (Fortunately, the payload is only a single hash value, +# so the largest it will ever be is 64 bytes for a 512-bit hash. Which is small enough +# that it never requires fragmenting to fit into a UDP packet. +@attr.frozen +class OpaqueHandshakeMessage: + record: Record + + # This takes a raw outgoing handshake volley that openssl generated, and # reconstructs the handshake messages inside it, so that we can repack them # into records while retransmitting. So the data ought to be well-behaved -- @@ -292,9 +302,18 @@ def decode_volley_trusted(volley): messages = [] messages_by_seq = {} for record in records_untrusted(volley): - if record.content_type == ContentType.change_cipher_spec: - messages.append(PseudoHandshakeMessage(record.version, - record.content_type, record.payload)) + # ChangeCipherSpec isn't a handshake message, so it can't be fragmented. + # Handshake messages with epoch > 0 are encrypted, so we can't fragment them + # either. Fortunately, ChangeCipherSpec has a 1 byte payload, and the only + # encrypted handshake message is Finished, whose payload is a single hash value + # -- so 32 bytes for SHA-256, 64 for SHA-512, etc. Neither is going to be so + # large that it has to be fragmented to fit into a single packet. + if record.epoch_seqno & EPOCH_MASK: + messages.append(OpaqueHandshakeMessage(record)) + elif record.content_type == ContentType.change_cipher_spec: + messages.append( + PseudoHandshakeMessage(record.version, record.content_type, record.payload) + ) else: assert record.content_type == ContentType.handshake fragment = decode_handshake_fragment_untrusted(record.payload) @@ -306,7 +325,7 @@ def decode_volley_trusted(volley): messages.append(msg) messages_by_seq[fragment.msg_seq] = msg else: - msg = messages_by_seq[msg_seq] + msg = messages_by_seq[fragment.msg_seq] assert msg.msg_type == fragment.msg_type assert msg.msg_seq == fragment.msg_seq assert len(msg.body) == fragment.msg_len @@ -317,8 +336,8 @@ def decode_volley_trusted(volley): class RecordEncoder: - def __init__(self, first_record_seq): - self._record_seq = count(first_record_seq) + def __init__(self): + self._record_seq = count() def skip_first_record_number(self): assert next(self._record_seq) == 0 @@ -327,7 +346,14 @@ def encode_volley(self, messages, mtu): packets = [] packet = bytearray() for message in messages: - if isinstance(message, PseudoHandshakeMessage): + if isinstance(message, OpaqueHandshakeMessage): + encoded = encode_record(message.record) + if mtu - len(packet) - len(encoded) <= 0: + packets.append(packet) + packet = bytearray() + packet += encoded + assert len(packet) <= mtu + elif isinstance(message, PseudoHandshakeMessage): space = mtu - len(packet) - RECORD_HEADER.size - len(message.payload) if space <= 0: packets.append(packet) @@ -338,12 +364,15 @@ def encode_volley(self, messages, mtu): next(self._record_seq), len(message.payload), ) - packet += payload + packet += message.payload assert len(packet) <= mtu else: - msg_len_bytes = message.msg_len.to_bytes(3, "big") + msg_len_bytes = len(message.body).to_bytes(3, "big") frag_offset = 0 - while frag_offset < len(message.body): + frags_encoded = 0 + # If message.body is empty, then we still want to encode it in one + # fragment, not zero. + while frag_offset < len(message.body) or not frags_encoded: space = mtu - len(packet) - RECORD_HEADER.size - HANDSHAKE_MESSAGE_HEADER.size if space <= 0: packets.append(packet) @@ -352,6 +381,7 @@ def encode_volley(self, messages, mtu): frag = message.body[frag_offset:frag_offset + space] frag_offset_bytes = frag_offset.to_bytes(3, "big") frag_len_bytes = len(frag).to_bytes(3, "big") + frag_offset += len(frag) packet += RECORD_HEADER.pack( ContentType.handshake, @@ -370,6 +400,7 @@ def encode_volley(self, messages, mtu): packet += frag + frags_encoded += 1 assert len(packet) <= mtu if packet: @@ -423,6 +454,10 @@ def encode_volley(self, messages, mtu): # probably pretty harmless? But it's easier to add the salt than to convince myself that # it's *completely* harmless, so, salt it is. +# XX maybe the cookie should also sign the *local* address, so you can't take a cookie +# from one socket and use it on another socket on the same trio? or just generate the +# key in each call to 'serve'. + COOKIE_REFRESH_INTERVAL = 30 # seconds KEY = None KEY_BYTES = 8 @@ -434,12 +469,12 @@ def _current_cookie_tick(): return int(trio.current_time() / COOKIE_REFRESH_INTERVAL) -# Simple deterministic, bijective serializer -- i.e., a useful tool for hashing -# structured data. +# Simple deterministic and invertible serializer -- i.e., a useful tool for converting +# structured data into something we can cryptographically sign. def _signable(*fields): out = [] for field in fields: - out.append(struct.encode("!Q", len(field))) + out.append(struct.pack("!Q", len(field))) out.append(field) return b"".join(out) @@ -453,7 +488,7 @@ def _make_cookie(salt, tick, address, client_hello_bits): signable_data = _signable( salt, - struct.encode("!Q", tick), + struct.pack("!Q", tick), # address is a mix of strings and ints, and variable length, so pack # it into a single nested field _signable(*(str(part).encode() for part in address)), @@ -467,9 +502,13 @@ def valid_cookie(cookie, address, client_hello_bits): if len(cookie) > SALT_BYTES: salt = cookie[:SALT_BYTES] + tick = _current_cookie_tick() + cur_cookie = _make_cookie(salt, tick, address, client_hello_bits) old_cookie = _make_cookie(salt, tick - 1, address, client_hello_bits) + # I doubt using a short-circuiting 'or' here would leak any meaningful + # information, but why risk it when '|' is just as easy. return ( hmac.compare_digest(cookie, cur_cookie) | hmac.compare_digest(cookie, old_cookie) @@ -508,7 +547,7 @@ def challenge_for(address, epoch_seqno, client_hello_bits): msg_seq=0, frag_offset=0, frag_len=len(body), - frag_bytes=body, + frag=body, ) payload = encode_handshake_fragment(hs) @@ -517,71 +556,9 @@ def challenge_for(address, epoch_seqno, client_hello_bits): return packet -# when listening, definitely needs background tasks to handle reading+handshaking -# connect() (add_peer()?) should probably handle handshake directly -# in *theory* if only had client-mode, then could have multiple tasks calling -# add_peer() simultaneously + -# -# alternatively: add_peer/listen as sync functions to express intentions, plus -# user is expected to regularly be pumping the receive loop. Handshakes don't -# progress unless you're doing this. Each packet receive call runs through the -# whole DTLS state machine. -# -# Problem: in this model, handshake timeouts are a pain in the butt. (Need to -# track them in some kind of manual timer wheel etc.). We don't -# want to write an explicit state machine; we want to take advantage of trio's -# tools. -# -# Therefore, handshakes should have a host task. -# -# And if handshakes have a host task, then what? -# -# main thing with handshakes is that might want to read packets to handle them -# even if user isn't otherwise reading packets. Our two options are to either -# have handshakes running continuously and potentially drop data packets -# internally, or else to only process handshakes while the user is asking for -# data. -# -# I guess dropping is better? We can have an internal queue size + stats on -# what's dropped? -# -# and if we're going to do our own dropping, then it's ok to have a constant -# reader task running... - -# One option would be to have a fully connection-based API. Make a DTLS socket -# endpoint, and then call `await connect(...)` to get a DTLS socket -# connection, or `serve(task_fn)` so `task_fn` runs on each incoming -# handshake. Lifetimes are then clear. (GC shutdown still a bit of a pain, but -# whatever, can handle it.) -# -# Alternatively, lean into the single socket API. Handshakes happen in -# background, peers are identified by magic tokens returned from 'connect' or -# 'receive'. Need some synthetic event for "new client connected" to avoid the -# possibility of clients filling up connection table with immortal nonsense. -# Need way to forget specific peers. -# -# In both cases: what to do when in server mode, and a new connection -# *replaces* an existing connection? (or in mixed client/server mode -# similarly, I guess?) I guess for single-socket API you need a way to get -# these notifications anyway... for server mode maybe the old one gets marked -# closed and a new one is created? so need some way to signal EOF, which isn't -# a thing otherwise. (BrokenResourceError?) -# -# Also for future-DTLS (and QUIC etc.) there's connection migration, where the -# peer address changes. I guess the user doesn't necessarily care about -# getting notifications of this, as long as their connection remains bound to -# the right peer? It does rule out the use of connected UDP sockets though. - - class _Queue: def __init__(self, incoming_packets_buffer): - self._s, self._r = trio.open_memory_channel(incoming_packets_buffer) - - async def put(self, obj): - await self._s.send(obj) - - async def get(self): - return self._r.receive() + self.s, self.r = trio.open_memory_channel(incoming_packets_buffer) def _read_loop(read_fn): @@ -602,43 +579,54 @@ async def handle_client_hello_untrusted(dtls, address, packet): return try: - epoch_seqno, cookie, bits = decode_client_hello_untrusted(address, packet) + epoch_seqno, cookie, bits = decode_client_hello_untrusted(packet) except BadPacket: return if not valid_cookie(cookie, address, bits): challenge_packet = challenge_for(address, epoch_seqno, bits) try: - await dtls.sock.sendto(address, challenge_packet) - except OSError: + async with dtls._send_lock: + await dtls.socket.sendto(challenge_packet, address) + except (OSError, trio.ClosedResourceError): pass else: - stream = DTLSStream(dtls, address, dtls._listening_context) - stream._inject_client_hello(packet) - old_stream = dlts._streams.get(address) + # We got a real, valid ClientHello! + stream = DTLSStream._create(dtls, address, dtls._listening_context) + try: + stream._inject_client_hello_untrusted(packet) + except BadPacket: + # ...or, well, OpenSSL didn't like it, so I guess we didn't. + return + old_stream = dtls._streams.get(address) if old_stream is not None: - old_stream._break(RuntimeError("peer started a new DTLS connection")) + if old_stream._client_hello == packet: + # ...but it's just a duplicate of a packet we got before, so never mind. + return + else: + # Ok, this *really is* a new handshake; the old stream should go away. + old_stream._replaced() dtls._streams[address] = stream - dtls._incoming_connections_q.put_nowait(stream) + dtls._incoming_connections_q.s.send_nowait(stream) async def dtls_receive_loop(dtls): sock = dtls.socket - dtls_ref = weakref.weakref(dtls) + dtls_ref = weakref.ref(dtls) del dtls while True: try: - address, packet = await sock.recvfrom() - except ClosedResourceError: + packet, address = await sock.recvfrom(MAX_UDP_PACKET_SIZE) + except trio.ClosedResourceError: return except OSError as exc: + # XX need to handle this better + # https://bobobobo.wordpress.com/2009/05/17/udp-an-existing-connection-was-forcibly-closed-by-the-remote-host/ dtls = dtls_ref() if dtls is None: return dtls._break(exc) return - # All of the following is sync, so we can be confident that our - # reference to dtls remains valid. dtls = dtls_ref() try: if dtls is None: @@ -652,13 +640,16 @@ async def dtls_receive_loop(dtls): # ClientHello, and we thought the handshake was done. Some of the # packets that we sent to finish the handshake must have gotten # lost. So re-send them. We do this directly here instead of just - # putting it into the queue, because there's no guarantee that - # anyone is reading from the queue, because we think the handshake - # is done! - await stream._resend_final_volley() + # putting it into the queue and letting the receiver do it, because + # there's no guarantee that anyone is reading from the queue, + # because we think the handshake is done! + try: + await stream._resend_final_volley() + except trio.ClosedResourceError: + return else: try: - stream._q.put_nowait(packet) + stream._q.s.send_nowait(packet) except trio.WouldBlock: stream.packets_dropped_in_trio += 1 else: @@ -674,18 +665,30 @@ def __init__(self, dtls, peer_address, ctx): self.peer_address = peer_address self.packets_dropped_in_trio = 0 self._mtu = 1472 # XX + self._client_hello = None self._did_handshake = False self._ssl = SSL.Connection(ctx) - self._broken = False + # Arbitrary b/c we repack messages anyway, but has to be set + self._ssl.set_ciphertext_mtu(1500) + self._replaced = False self._closed = False - self._q = Queue(dtls.incoming_packets_buffer) + self._q = _Queue(dtls.incoming_packets_buffer) self._handshake_lock = trio.Lock() self._record_encoder = RecordEncoder() - def _break(self, reason: BaseException): - self._broken = True - self._broken_reason = reason - # XX wake things up + def _replaced(self): + self._replaced = True + # Any packets we already received could maybe possibly still be processed, but + # there are no more coming. So we close this on the sender side. + self._q.s.close() + + def _check_replaced(self): + if self._replaced: + raise BrokenResourceError("peer tore down this connection to start a new one") + + # XX expose knobs and queries for MTU information + # XX on systems where we can (maybe just Linux?) take advantage of the kernel's PMTU + # estimate def close(self): if self._closed: @@ -695,22 +698,37 @@ def close(self): del self.dtls._streams[self.peer_address] # Will wake any tasks waiting on self._q.get with a # ClosedResourceError - self._q._r.close() + self._q.r.close() async def aclose(self): self.close() await trio.lowlevel.checkpoint() - def _inject_client_hello(self, packet): - stream._ssl.bio_write(packet) + def _inject_client_hello_untrusted(self, packet): + self._client_hello = packet + self._ssl.bio_write(packet) # If we're on the server side, then we already sent record 0 as our cookie # challenge. So we want to start the handshake proper with record 1. self._record_encoder.skip_first_record_number() + # We've already validated this cookie. But, we still have to call DTLSv1_listen + # so OpenSSL thinks that it's verified the cookie. The problem is that + # if you're doing cookie challenges, then the actual ClientHello has msg_seq=1 + # instead of msg_seq=0, and OpenSSL will refuse to process a ClientHello with + # msg_seq=1 unless you've called DTLSv1_listen. It also gets OpenSSL to bump the + # outgoing ServerHello's msg_seq to 1. + try: + self._ssl.DTLSv1_listen() + except SSL.Error: + raise BadPacket async def _send_volley(self, volley_messages): - packets = self._record_encoder(volley_messages, self._mtu) + packets = self._record_encoder.encode_volley(volley_messages, self._mtu) + # XX debug + # decoded = decode_volley_trusted(b"".join(packets)) + # assert decoded == volley_messages for packet in packets: - await self.dtls.socket.sendto(self.peer_address, packet) + async with self.dtls._send_lock: + await self.dtls.socket.sendto(packet, self.peer_address) async def _resend_final_volley(self): await self._send_volley(self._final_volley) @@ -729,7 +747,7 @@ async def do_handshake(self): volley_messages = [] def read_volley(): - volley_bytes = read_loop(self._ssl.bio_read) + volley_bytes = _read_loop(self._ssl.bio_read) new_volley_messages = decode_volley_trusted(volley_bytes) if ( new_volley_messages and volley_messages and @@ -746,15 +764,13 @@ def read_volley(): # has gone very wrong. (I'm not sure this can actually happen without an # error from OpenSSL, but let's cover our bases.) if not volley_messages: - self._break(SSL.Error("something wrong with peer's ClientHello")) - # XX raise - return + raise SSL.Error("something wrong with peer's ClientHello") while True: assert volley_messages await self._send_volley(volley_messages) - with trio.move_on_after(1) as cscope: - async for packet in self._q._r: + with trio.move_on_after(10) as cscope: + async for packet in self._q.r: self._ssl.bio_write(packet) try: self._ssl.do_handshake() @@ -773,25 +789,36 @@ def read_volley(): # We managed to get all of the peer's volley and generate a # new one ourselves! break out of the 'for' loop and restart # the timer. - volley_messages = new_volley + volley_messages = maybe_volley break + else: + assert self._replaced + self._check_replaced() if cscope.cancelled_caught: # timeout expired, adjust timeout/mtu # Good guidance here: https://tlswg.org/dtls13-spec/draft-ietf-tls-dtls13.html#name-timer-values XX async def send(self, data): + if self._closed: + raise trio.ClosedResourceError if not self._did_handshake: await self.do_handshake() + self._check_replaced() self._ssl.write(data) - await self.dtls.socket.sendto(self.peer_address, read_loop(self._ssl.bio_read)) + async with self.dtls._send_lock: + await self.dtls.socket.sendto(_read_loop(self._ssl.bio_read), self.peer_address) async def receive(self): if not self._did_handshake: await self.do_handshake() - packet = await self._q.get() + try: + packet = await self._q.r.receive() + except trio.EndOfChannel: + assert self._replaced + self._check_replaced() self._ssl.bio_write(packet) - return read_loop(self._ssl.read) + return _read_loop(self._ssl.read) class DTLS: @@ -808,7 +835,8 @@ def __init__(self, socket, *, incoming_packets_buffer=10): # {remote address: DTLSStream} self._streams = weakref.WeakValueDictionary() self._listening_context = None - self._incoming_connections_q = Queue(float("inf")) + self._incoming_connections_q = _Queue(float("inf")) + self._send_lock = trio.Lock() trio.lowlevel.spawn_system_task(dtls_receive_loop, self) @@ -824,19 +852,25 @@ def close(self): self.socket.close() for stream in self._streams.values(): stream.close() - self._incoming_connections_q._s.close() + self._incoming_connections_q.s.close() - async def aclose(self): + def __enter__(self): + return self + + def __exit__(self, *args): self.close() - await trio.lowlevel.checkpoint() - async def serve(self, ssl_context, async_fn, *args): + async def serve(self, ssl_context, async_fn, *args, task_status=trio.TASK_STATUS_IGNORED): if self._listening_context is not None: raise trio.BusyResourceError("another task is already listening") + # We do cookie verification ourselves, so tell OpenSSL not to worry about it. + # (See also _inject_client_hello_untrusted.) + ssl_context.set_cookie_verify_callback(lambda *_: True) try: self._listening_context = ssl_context + task_status.started() async with trio.open_nursery() as nursery: - async for stream in self._incoming_connections_q._r: + async for stream in self._incoming_connections_q.r: nursery.start_soon(async_fn, stream, *args) finally: self._listening_context = None @@ -848,7 +882,8 @@ def _set_stream_for(self, address, stream): self._streams[address] = stream async def connect(self, address, ssl_context): - stream = DTLSStream(self, address, ssl_context) + stream = DTLSStream._create(self, address, ssl_context) + stream._ssl.set_connect_state() self._set_stream_for(address, stream) await stream.do_handshake() return stream diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py new file mode 100644 index 0000000000..5e7f70ad55 --- /dev/null +++ b/trio/tests/test_dtls.py @@ -0,0 +1,40 @@ +import trio +from trio._dtls import DTLS + +import trustme +from OpenSSL import SSL + +ca = trustme.CA() +server_cert = ca.issue_cert("example.com") + +server_ctx = SSL.Context(SSL.DTLS_METHOD) +server_cert.configure_cert(server_ctx) + +client_ctx = SSL.Context(SSL.DTLS_METHOD) +ca.configure_trust(client_ctx) + +# XX this should be handled in the real code +server_ctx.set_options(SSL.OP_NO_QUERY_MTU | SSL.OP_NO_RENEGOTIATION) +client_ctx.set_options(SSL.OP_NO_QUERY_MTU | SSL.OP_NO_RENEGOTIATION) + +async def test_smoke(): + server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) + with server_sock: + await server_sock.bind(("127.0.0.1", 54321)) + server_dtls = DTLS(server_sock) + + async with trio.open_nursery() as nursery: + + async def handle_client(dtls_stream): + await dtls_stream.do_handshake() + assert await dtls_stream.receive() == b"hello" + await dtls_stream.send(b"goodbye") + + await nursery.start(server_dtls.serve, server_ctx, handle_client) + + client_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) + client_dtls = DTLS(client_sock) + client = await client_dtls.connect(server_sock.getsockname(), client_ctx) + await client.send(b"hello") + assert await client.receive() == b"goodbye" + nursery.cancel_scope.cancel() From 66595f56393c0d575600b87ac361e43649436576 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 25 Jun 2021 02:58:27 -0700 Subject: [PATCH 0925/1498] Move required SSL OP_* settings into the proper place --- trio/_dtls.py | 6 ++++++ trio/tests/test_dtls.py | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 4b738581e5..c37c9ba20e 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -667,6 +667,12 @@ def __init__(self, dtls, peer_address, ctx): self._mtu = 1472 # XX self._client_hello = None self._did_handshake = False + # These are mandatory for all DTLS connections. OP_NO_QUERY_MTU is required to + # stop openssl from trying to query the memory BIO's MTU and then breaking, and + # OP_NO_RENEGOTIATION disables renegotiation, which is too complex for us to + # support and isn't useful anyway -- especially for DTLS where it's equivalent + # to just performing a new handshake. + ctx.set_options(SSL.OP_NO_QUERY_MTU | SSL.OP_NO_RENEGOTIATION) self._ssl = SSL.Connection(ctx) # Arbitrary b/c we repack messages anyway, but has to be set self._ssl.set_ciphertext_mtu(1500) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 5e7f70ad55..5149e8d56f 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -13,10 +13,6 @@ client_ctx = SSL.Context(SSL.DTLS_METHOD) ca.configure_trust(client_ctx) -# XX this should be handled in the real code -server_ctx.set_options(SSL.OP_NO_QUERY_MTU | SSL.OP_NO_RENEGOTIATION) -client_ctx.set_options(SSL.OP_NO_QUERY_MTU | SSL.OP_NO_RENEGOTIATION) - async def test_smoke(): server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) with server_sock: From 641494a0e6c8308c015d57dae4aea88b8509f97d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 25 Jun 2021 05:32:59 -0700 Subject: [PATCH 0926/1498] All logic implemented, I think (probably not all correct though) --- trio/_dtls.py | 104 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 28 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index c37c9ba20e..f9e9363863 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -20,6 +20,24 @@ MAX_UDP_PACKET_SIZE = 65527 +def packet_header_overhead(sock): + if sock.family == trio.socket.AF_INET: + return 28 + else: + return 48 + + +def worst_case_mtu(sock): + if sock.family == trio.socket.AF_INET: + return 576 - packet_header_overhead(sock) + else: + return 1280 - packet_header_overhead(sock) + + +def best_guess_mtu(sock): + return 1500 - packet_header_overhead(sock) + + # There are a bunch of different RFCs that define these codes, so for a # comprehensive collection look here: # https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml @@ -620,13 +638,16 @@ async def dtls_receive_loop(dtls): except trio.ClosedResourceError: return except OSError as exc: - # XX need to handle this better - # https://bobobobo.wordpress.com/2009/05/17/udp-an-existing-connection-was-forcibly-closed-by-the-remote-host/ - dtls = dtls_ref() - if dtls is None: + if exc.errno in (errno.EBADF, errno.ENOTSOCK): + # Socket was closed return - dtls._break(exc) - return + else: + # Some weird error, e.g. apparently some versions of Windows can do + # ECONNRESET here to report that some previous UDP packet got an ICMP + # Port Unreachable: + # https://bobobobo.wordpress.com/2009/05/17/udp-an-existing-connection-was-forcibly-closed-by-the-remote-host/ + # We'll assume that whatever it is, it's a transient problem. + continue dtls = dtls_ref() try: if dtls is None: @@ -664,7 +685,6 @@ def __init__(self, dtls, peer_address, ctx): self.dtls = dtls self.peer_address = peer_address self.packets_dropped_in_trio = 0 - self._mtu = 1472 # XX self._client_hello = None self._did_handshake = False # These are mandatory for all DTLS connections. OP_NO_QUERY_MTU is required to @@ -674,8 +694,10 @@ def __init__(self, dtls, peer_address, ctx): # to just performing a new handshake. ctx.set_options(SSL.OP_NO_QUERY_MTU | SSL.OP_NO_RENEGOTIATION) self._ssl = SSL.Connection(ctx) - # Arbitrary b/c we repack messages anyway, but has to be set - self._ssl.set_ciphertext_mtu(1500) + self._mtu = None + # This calls self._ssl.set_ciphertext_mtu, which is important, because if you + # don't call it then openssl doesn't work. + self.set_ciphertext_mtu(best_guess_mtu(self.dtls.socket)) self._replaced = False self._closed = False self._q = _Queue(dtls.incoming_packets_buffer) @@ -692,10 +714,20 @@ def _check_replaced(self): if self._replaced: raise BrokenResourceError("peer tore down this connection to start a new one") - # XX expose knobs and queries for MTU information + def set_ciphertext_mtu(self, new_mtu): + self._mtu = new_mtu + self._ssl.set_ciphertext_mtu(new_mtu) + + def get_cleartext_mtu(self): + return self._ssl.get_cleartext_mtu() + # XX on systems where we can (maybe just Linux?) take advantage of the kernel's PMTU # estimate + # XX should we send close-notify when closing? It seems particularly pointless for + # DTLS where packets are all independent and can be lost anyway. We do at least need + # to handle receiving it properly though, which might be easier if we send it... + def close(self): if self._closed: return @@ -729,9 +761,6 @@ def _inject_client_hello_untrusted(self, packet): async def _send_volley(self, volley_messages): packets = self._record_encoder.encode_volley(volley_messages, self._mtu) - # XX debug - # decoded = decode_volley_trusted(b"".join(packets)) - # assert decoded == volley_messages for packet in packets: async with self.dtls._send_lock: await self.dtls.socket.sendto(packet, self.peer_address) @@ -739,19 +768,15 @@ async def _send_volley(self, volley_messages): async def _resend_final_volley(self): await self._send_volley(self._final_volley) - async def do_handshake(self): + async def do_handshake(self, *, initial_retransmit_timeout=1.0): async with self._handshake_lock: if self._did_handshake: return - # If we're a client, we send the initial volley. If we're a server, then - # the initial ClientHello has already been inserted into self._ssl's - # read BIO. So either way, we start by generating a new volley. - try: - self._ssl.do_handshake() - except SSL.WantReadError: - pass + timeout = initial_retransmit_timeout volley_messages = [] + volley_failed_sends = 0 + def read_volley(): volley_bytes = _read_loop(self._ssl.bio_read) new_volley_messages = decode_volley_trusted(volley_bytes) @@ -759,22 +784,31 @@ def read_volley(): new_volley_messages and volley_messages and new_volley_messages[0].msg_seq == volley_messages[0].msg_seq ): - # openssl decided to retransmit; discard because we'll handle this - # ourselves + # openssl decided to retransmit; discard because we handle + # retransmits ourselves return [] else: return new_volley_messages + # If we're a client, we send the initial volley. If we're a server, then + # the initial ClientHello has already been inserted into self._ssl's + # read BIO. So either way, we start by generating a new volley. + try: + self._ssl.do_handshake() + except SSL.WantReadError: + pass volley_messages = read_volley() # If we don't have messages to send in our initial volley, then something # has gone very wrong. (I'm not sure this can actually happen without an - # error from OpenSSL, but let's cover our bases.) + # error from OpenSSL, but we check just in case.) if not volley_messages: raise SSL.Error("something wrong with peer's ClientHello") while True: + # -- at this point, we need to either send or re-send a volley -- assert volley_messages await self._send_volley(volley_messages) + # -- then this is where we wait for a reply -- with trio.move_on_after(10) as cscope: async for packet in self._q.r: self._ssl.bio_write(packet) @@ -786,7 +820,8 @@ def read_volley(): # No exception -> the handshake is done, and we can # switch into data transfer mode. self._did_handshake = True - # Might be empty, but that's ok + # Might be empty, but that's ok -- we'll just send no + # packets. self._final_volley = read_volley() await self._send_volley(self._final_volley) return @@ -796,14 +831,27 @@ def read_volley(): # new one ourselves! break out of the 'for' loop and restart # the timer. volley_messages = maybe_volley + # "Implementations SHOULD retain the current timer value + # until a transmission without loss occurs, at which time + # the value may be reset to the initial value." + if volley_failed_sends == 0: + timeout = initial_retransmit_timeout + volley_failed_sends = 0 break else: assert self._replaced self._check_replaced() if cscope.cancelled_caught: - # timeout expired, adjust timeout/mtu - # Good guidance here: https://tlswg.org/dtls13-spec/draft-ietf-tls-dtls13.html#name-timer-values - XX + # Timeout expired. Double timeout for backoff, with a limit of 60 + # seconds (this matches what openssl does, and also the + # recommendation in draft-ietf-tls-dtls13). + timeout = min(2 * timeout, 60.0) + volley_failed_sends += 1 + if volley_failed_sends == 2: + # We tried sending this twice and they both failed. Maybe our + # PMTU estimate is wrong? Let's try dropping it to the minimum + # and hope that helps. + self.set_ciphertext_mtu(min(self._mtu, worst_case_mtu(self.dtls.socket))) async def send(self, data): if self._closed: From d581e35988e498004218e9e4f3981fb4cf2bfe10 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 26 Jun 2021 02:11:09 -0700 Subject: [PATCH 0927/1498] Run black --- trio/_dtls.py | 72 ++++++++++++++++++++++++----------------- trio/tests/test_dtls.py | 1 + 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index f9e9363863..4a841a48d4 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -20,6 +20,7 @@ MAX_UDP_PACKET_SIZE = 65527 + def packet_header_overhead(sock): if sock.family == trio.socket.AF_INET: return 28 @@ -78,7 +79,7 @@ class ProtocolVersion: DTLS12 = bytes([254, 253]) -EPOCH_MASK = 0xffff << (6 * 8) +EPOCH_MASK = 0xFFFF << (6 * 8) # Conventions: @@ -151,10 +152,7 @@ def records_untrusted(packet): def encode_record(record): header = RECORD_HEADER.pack( - record.content_type, - record.version, - record.epoch_seqno, - len(record.payload), + record.content_type, record.version, record.epoch_seqno, len(record.payload), ) return header + record.payload @@ -195,17 +193,10 @@ def decode_handshake_fragment_untrusted(payload): msg_len = int.from_bytes(msg_len_bytes, "big") frag_offset = int.from_bytes(frag_offset_bytes, "big") frag_len = int.from_bytes(frag_len_bytes, "big") - frag = payload[HANDSHAKE_MESSAGE_HEADER.size:] + frag = payload[HANDSHAKE_MESSAGE_HEADER.size :] if len(frag) != frag_len: raise BadPacket("handshake fragment length doesn't match record length") - return HandshakeFragment( - msg_type, - msg_len, - msg_seq, - frag_offset, - frag_len, - frag, - ) + return HandshakeFragment(msg_type, msg_len, msg_seq, frag_offset, frag_len, frag,) def encode_handshake_fragment(hsf): @@ -330,7 +321,9 @@ def decode_volley_trusted(volley): messages.append(OpaqueHandshakeMessage(record)) elif record.content_type == ContentType.change_cipher_spec: messages.append( - PseudoHandshakeMessage(record.version, record.content_type, record.payload) + PseudoHandshakeMessage( + record.version, record.content_type, record.payload + ) ) else: assert record.content_type == ContentType.handshake @@ -338,7 +331,10 @@ def decode_volley_trusted(volley): msg_type = HandshakeType(fragment.msg_type) if fragment.msg_seq not in messages_by_seq: msg = HandshakeMessage( - record.version, msg_type, fragment.msg_seq, bytearray(fragment.msg_len) + record.version, + msg_type, + fragment.msg_seq, + bytearray(fragment.msg_len), ) messages.append(msg) messages_by_seq[fragment.msg_seq] = msg @@ -348,7 +344,9 @@ def decode_volley_trusted(volley): assert msg.msg_seq == fragment.msg_seq assert len(msg.body) == fragment.msg_len - msg.body[fragment.frag_offset : fragment.frag_offset + fragment.frag_len] = fragment.frag + msg.body[ + fragment.frag_offset : fragment.frag_offset + fragment.frag_len + ] = fragment.frag return messages @@ -391,12 +389,17 @@ def encode_volley(self, messages, mtu): # If message.body is empty, then we still want to encode it in one # fragment, not zero. while frag_offset < len(message.body) or not frags_encoded: - space = mtu - len(packet) - RECORD_HEADER.size - HANDSHAKE_MESSAGE_HEADER.size + space = ( + mtu + - len(packet) + - RECORD_HEADER.size + - HANDSHAKE_MESSAGE_HEADER.size + ) if space <= 0: packets.append(packet) packet = bytearray() continue - frag = message.body[frag_offset:frag_offset + space] + frag = message.body[frag_offset : frag_offset + space] frag_offset_bytes = frag_offset.to_bytes(3, "big") frag_len_bytes = len(frag).to_bytes(3, "big") frag_offset += len(frag) @@ -527,9 +530,8 @@ def valid_cookie(cookie, address, client_hello_bits): # I doubt using a short-circuiting 'or' here would leak any meaningful # information, but why risk it when '|' is just as easy. - return ( - hmac.compare_digest(cookie, cur_cookie) - | hmac.compare_digest(cookie, old_cookie) + return hmac.compare_digest(cookie, cur_cookie) | hmac.compare_digest( + cookie, old_cookie ) else: return False @@ -569,8 +571,9 @@ def challenge_for(address, epoch_seqno, client_hello_bits): ) payload = encode_handshake_fragment(hs) - packet = encode_record(Record(ContentType.handshake, - ProtocolVersion.DTLS10, epoch_seqno, payload)) + packet = encode_record( + Record(ContentType.handshake, ProtocolVersion.DTLS10, epoch_seqno, payload) + ) return packet @@ -712,7 +715,9 @@ def _replaced(self): def _check_replaced(self): if self._replaced: - raise BrokenResourceError("peer tore down this connection to start a new one") + raise BrokenResourceError( + "peer tore down this connection to start a new one" + ) def set_ciphertext_mtu(self, new_mtu): self._mtu = new_mtu @@ -781,8 +786,9 @@ def read_volley(): volley_bytes = _read_loop(self._ssl.bio_read) new_volley_messages = decode_volley_trusted(volley_bytes) if ( - new_volley_messages and volley_messages and - new_volley_messages[0].msg_seq == volley_messages[0].msg_seq + new_volley_messages + and volley_messages + and new_volley_messages[0].msg_seq == volley_messages[0].msg_seq ): # openssl decided to retransmit; discard because we handle # retransmits ourselves @@ -851,7 +857,9 @@ def read_volley(): # We tried sending this twice and they both failed. Maybe our # PMTU estimate is wrong? Let's try dropping it to the minimum # and hope that helps. - self.set_ciphertext_mtu(min(self._mtu, worst_case_mtu(self.dtls.socket))) + self.set_ciphertext_mtu( + min(self._mtu, worst_case_mtu(self.dtls.socket)) + ) async def send(self, data): if self._closed: @@ -861,7 +869,9 @@ async def send(self, data): self._check_replaced() self._ssl.write(data) async with self.dtls._send_lock: - await self.dtls.socket.sendto(_read_loop(self._ssl.bio_read), self.peer_address) + await self.dtls.socket.sendto( + _read_loop(self._ssl.bio_read), self.peer_address + ) async def receive(self): if not self._did_handshake: @@ -914,7 +924,9 @@ def __enter__(self): def __exit__(self, *args): self.close() - async def serve(self, ssl_context, async_fn, *args, task_status=trio.TASK_STATUS_IGNORED): + async def serve( + self, ssl_context, async_fn, *args, task_status=trio.TASK_STATUS_IGNORED + ): if self._listening_context is not None: raise trio.BusyResourceError("another task is already listening") # We do cookie verification ourselves, so tell OpenSSL not to worry about it. diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 5149e8d56f..783a9e5e8f 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -13,6 +13,7 @@ client_ctx = SSL.Context(SSL.DTLS_METHOD) ca.configure_trust(client_ctx) + async def test_smoke(): server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) with server_sock: From 4903a6133e446c2b7723ccdb3ae576ed3ee36eb0 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 30 Jun 2021 23:45:02 -0700 Subject: [PATCH 0928/1498] Delay importing OpenSSL.SSL until a DTLS object is constructed --- trio/__init__.py | 2 ++ trio/_dtls.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/trio/__init__.py b/trio/__init__.py index a50ec33310..b4b77f7fa9 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -74,6 +74,8 @@ from ._ssl import SSLStream, SSLListener, NeedHandshakeError +from ._dtls import DTLS, DTLSStream + from ._highlevel_serve_listeners import serve_listeners from ._highlevel_open_tcp_stream import open_tcp_stream diff --git a/trio/_dtls.py b/trio/_dtls.py index 4a841a48d4..a3650c3598 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -13,7 +13,6 @@ import weakref import attr -from OpenSSL import SSL import trio from trio._util import NoPublicConstructor @@ -887,6 +886,11 @@ async def receive(self): class DTLS: def __init__(self, socket, *, incoming_packets_buffer=10): + # We do this lazily on first construction, so only people who actually use DTLS + # have to install PyOpenSSL. + global SSL + from OpenSSL import SSL + if socket.type != trio.socket.SOCK_DGRAM: raise BadPacket("DTLS requires a SOCK_DGRAM socket") self.socket = socket From 9304e3860a6df146f5346a228b7a1495f2d59ae8 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 3 Sep 2021 09:33:42 -0700 Subject: [PATCH 0929/1498] Close test socket properly --- trio/tests/test_dtls.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 783a9e5e8f..8fcf24a112 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -29,9 +29,9 @@ async def handle_client(dtls_stream): await nursery.start(server_dtls.serve, server_ctx, handle_client) - client_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) - client_dtls = DTLS(client_sock) - client = await client_dtls.connect(server_sock.getsockname(), client_ctx) - await client.send(b"hello") - assert await client.receive() == b"goodbye" - nursery.cancel_scope.cancel() + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: + client_dtls = DTLS(client_sock) + client = await client_dtls.connect(server_sock.getsockname(), client_ctx) + await client.send(b"hello") + assert await client.receive() == b"goodbye" + nursery.cancel_scope.cancel() From cd7bf54ec035057e95ae08cece784abe3a6d79e5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 3 Sep 2021 09:33:52 -0700 Subject: [PATCH 0930/1498] mark DTLS class as final --- trio/_dtls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index a3650c3598..6a0ca3edb2 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -15,7 +15,7 @@ import attr import trio -from trio._util import NoPublicConstructor +from trio._util import NoPublicConstructor, Final MAX_UDP_PACKET_SIZE = 65527 @@ -884,7 +884,7 @@ async def receive(self): return _read_loop(self._ssl.read) -class DTLS: +class DTLS(metaclass=Final): def __init__(self, socket, *, incoming_packets_buffer=10): # We do this lazily on first construction, so only people who actually use DTLS # have to install PyOpenSSL. From c44f29fccd59acfdb4ce887f397dcf3638e33ce9 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 3 Sep 2021 09:34:21 -0700 Subject: [PATCH 0931/1498] Refactor socket address resolution Split out the address resolution code so that fake net can reuse it more easily --- trio/_socket.py | 174 +++++++++++++++++++------------------- trio/tests/test_socket.py | 21 +++-- 2 files changed, 98 insertions(+), 97 deletions(-) diff --git a/trio/_socket.py b/trio/_socket.py index 886f5614f6..3747bfe4d5 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -349,6 +349,82 @@ async def wrapper(self, *args, **kwargs): return wrapper +# Helpers to work with the (hostname, port) language that Python uses for socket +# addresses everywhere. Split out into a standalone function so it can be reused by +# FakeNet. + +# Take an address in Python's representation, and returns a new address in +# the same representation, but with names resolved to numbers, +# etc. +# +# local=True means that the address is being used with bind() or similar +# local=False means that the address is being used with connect() or sendto() or +# similar. +# +# NOTE: this function does not always checkpoint +async def _resolve_address_nocp(type, family, proto, *, ipv6_v6only, address, local): + # Do some pre-checking (or exit early for non-IP sockets) + if family == _stdlib_socket.AF_INET: + if not isinstance(address, tuple) or not len(address) == 2: + raise ValueError("address should be a (host, port) tuple") + elif family == _stdlib_socket.AF_INET6: + if not isinstance(address, tuple) or not 2 <= len(address) <= 4: + raise ValueError( + "address should be a (host, port, [flowinfo, [scopeid]]) tuple" + ) + elif family == _stdlib_socket.AF_UNIX: + # unwrap path-likes + return os.fspath(address) + else: + return address + + # -- From here on we know we have IPv4 or IPV6 -- + host, port, *_ = address + # Fast path for the simple case: already-resolved IP address, + # already-resolved port. This is particularly important for UDP, since + # every sendto call goes through here. + if isinstance(port, int): + try: + _stdlib_socket.inet_pton(family, address[0]) + except (OSError, TypeError): + pass + else: + return address + # Special cases to match the stdlib, see gh-277 + if host == "": + host = None + if host == "": + host = "255.255.255.255" + flags = 0 + if local: + flags |= _stdlib_socket.AI_PASSIVE + # Since we always pass in an explicit family here, AI_ADDRCONFIG + # doesn't add any value -- if we have no ipv6 connectivity and are + # working with an ipv6 socket, then things will break soon enough! And + # if we do enable it, then it makes it impossible to even run tests + # for ipv6 address resolution on travis-ci, which as of 2017-03-07 has + # no ipv6. + # flags |= AI_ADDRCONFIG + if family == _stdlib_socket.AF_INET6 and not ipv6_v6only: + flags |= _stdlib_socket.AI_V4MAPPED + gai_res = await getaddrinfo(host, port, family, type, proto, flags) + # AFAICT from the spec it's not possible for getaddrinfo to return an + # empty list. + assert len(gai_res) >= 1 + # Address is the last item in the first entry + (*_, normed), *_ = gai_res + # The above ignored any flowid and scopeid in the passed-in address, + # so restore them if present: + if family == _stdlib_socket.AF_INET6: + normed = list(normed) + assert len(normed) == 4 + if len(address) >= 3: + normed[2] = address[2] + if len(address) >= 4: + normed[3] = address[3] + normed = tuple(normed) + return normed + class SocketType: def __init__(self): raise TypeError( @@ -444,7 +520,7 @@ def close(self): self._sock.close() async def bind(self, address): - address = await self._resolve_local_address_nocp(address) + address = await self._resolve_address_nocp(address, local=True) if ( hasattr(_stdlib_socket, "AF_UNIX") and self.family == _stdlib_socket.AF_UNIX @@ -480,89 +556,15 @@ def is_readable(self): async def wait_writable(self): await _core.wait_writable(self._sock) - ################################################################ - # Address handling - ################################################################ - - # Take an address in Python's representation, and returns a new address in - # the same representation, but with names resolved to numbers, - # etc. - # - # NOTE: this function does not always checkpoint - async def _resolve_address_nocp(self, address, flags): - # Do some pre-checking (or exit early for non-IP sockets) - if self._sock.family == _stdlib_socket.AF_INET: - if not isinstance(address, tuple) or not len(address) == 2: - raise ValueError("address should be a (host, port) tuple") - elif self._sock.family == _stdlib_socket.AF_INET6: - if not isinstance(address, tuple) or not 2 <= len(address) <= 4: - raise ValueError( - "address should be a (host, port, [flowinfo, [scopeid]]) tuple" - ) - elif self._sock.family == _stdlib_socket.AF_UNIX: - # unwrap path-likes - return os.fspath(address) + async def _resolve_address_nocp(self, address, *, local): + if self.family == _stdlib_socket.AF_INET6: + ipv6_v6only = self._sock.getsockopt(IPPROTO_IPV6, _stdlib_socket.IPV6_V6ONLY) else: - return address - - # -- From here on we know we have IPv4 or IPV6 -- - host, port, *_ = address - # Fast path for the simple case: already-resolved IP address, - # already-resolved port. This is particularly important for UDP, since - # every sendto call goes through here. - if isinstance(port, int): - try: - _stdlib_socket.inet_pton(self._sock.family, address[0]) - except (OSError, TypeError): - pass - else: - return address - # Special cases to match the stdlib, see gh-277 - if host == "": - host = None - if host == "": - host = "255.255.255.255" - # Since we always pass in an explicit family here, AI_ADDRCONFIG - # doesn't add any value -- if we have no ipv6 connectivity and are - # working with an ipv6 socket, then things will break soon enough! And - # if we do enable it, then it makes it impossible to even run tests - # for ipv6 address resolution on travis-ci, which as of 2017-03-07 has - # no ipv6. - # flags |= AI_ADDRCONFIG - if self._sock.family == _stdlib_socket.AF_INET6: - if not self._sock.getsockopt(IPPROTO_IPV6, _stdlib_socket.IPV6_V6ONLY): - flags |= _stdlib_socket.AI_V4MAPPED - gai_res = await getaddrinfo( - host, port, self._sock.family, self.type, self._sock.proto, flags - ) - # AFAICT from the spec it's not possible for getaddrinfo to return an - # empty list. - assert len(gai_res) >= 1 - # Address is the last item in the first entry - (*_, normed), *_ = gai_res - # The above ignored any flowid and scopeid in the passed-in address, - # so restore them if present: - if self._sock.family == _stdlib_socket.AF_INET6: - normed = list(normed) - assert len(normed) == 4 - if len(address) >= 3: - normed[2] = address[2] - if len(address) >= 4: - normed[3] = address[3] - normed = tuple(normed) - return normed - - # Returns something appropriate to pass to bind() - # - # NOTE: this function does not always checkpoint - async def _resolve_local_address_nocp(self, address): - return await self._resolve_address_nocp(address, _stdlib_socket.AI_PASSIVE) - - # Returns something appropriate to pass to connect()/sendto()/sendmsg() - # - # NOTE: this function does not always checkpoint - async def _resolve_remote_address_nocp(self, address): - return await self._resolve_address_nocp(address, 0) + ipv6_v6only = False + return await _resolve_address_nocp( + self.type, self.family, self.proto, + ipv6_v6only=ipv6_v6only, + address=address, local=local) async def _nonblocking_helper(self, fn, args, kwargs, wait_fn): # We have to reconcile two conflicting goals: @@ -617,7 +619,7 @@ async def connect(self, address): # notification. This means it isn't really cancellable... we close the # socket if cancelled, to avoid confusion. try: - address = await self._resolve_remote_address_nocp(address) + address = await self._resolve_address_nocp(address, local=False) async with _try_sync(): # An interesting puzzle: can a non-blocking connect() return EINTR # (= raise InterruptedError)? PEP 475 specifically left this as @@ -741,7 +743,7 @@ async def sendto(self, *args): # args is: data[, flags], address) # and kwargs are not accepted args = list(args) - args[-1] = await self._resolve_remote_address_nocp(args[-1]) + args[-1] = await self._resolve_address_nocp(args[-1], local=False) return await self._nonblocking_helper( _stdlib_socket.socket.sendto, args, {}, _core.wait_writable ) @@ -766,7 +768,7 @@ async def sendmsg(self, *args): # and kwargs are not accepted if len(args) == 4 and args[-1] is not None: args = list(args) - args[-1] = await self._resolve_remote_address_nocp(args[-1]) + args[-1] = await self._resolve_address_nocp(args[-1], local=False) return await self._nonblocking_helper( _stdlib_socket.socket.sendmsg, args, {}, _core.wait_writable ) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index d891041ab2..b7c4839981 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -495,21 +495,20 @@ def assert_eq(actual, expected): with tsocket.socket(family=socket_type) as sock: # For some reason the stdlib special-cases "" to pass NULL to - # getaddrinfo They also error out on None, but whatever, None is much + # getaddrinfo. They also error out on None, but whatever, None is much # more consistent, so we accept it too. for null in [None, ""]: - got = await sock._resolve_local_address_nocp((null, 80)) + got = await sock._resolve_address_nocp((null, 80), local=True) assert_eq(got, (addrs.bind_all, 80)) - got = await sock._resolve_remote_address_nocp((null, 80)) + got = await sock._resolve_address_nocp((null, 80), local=False) assert_eq(got, (addrs.localhost, 80)) # AI_PASSIVE only affects the wildcard address, so for everything else - # _resolve_local_address_nocp and _resolve_remote_address_nocp should - # work the same: - for resolver in ["_resolve_local_address_nocp", "_resolve_remote_address_nocp"]: + # local=True/local=False should work the same: + for local in [False, True]: async def res(*args): - return await getattr(sock, resolver)(*args) + return await sock._resolve_address_nocp(*args, local=local) assert_eq(await res((addrs.arbitrary, "http")), (addrs.arbitrary, 80)) if v6: @@ -560,7 +559,7 @@ async def res(*args): except (AttributeError, OSError): pass else: - assert await getattr(netlink_sock, resolver)("asdf") == "asdf" + assert await netlink_sock._resolve_address_nocp("asdf", local=local) == "asdf" netlink_sock.close() with pytest.raises(ValueError): @@ -721,16 +720,16 @@ def connect(self, *args, **kwargs): await sock.connect(("127.0.0.1", 2)) -async def test_resolve_remote_address_exception_closes_socket(): +async def test_resolve_address_exception_in_connect_closes_socket(): # Here we are testing issue 247, any cancellation will leave the socket closed with _core.CancelScope() as cancel_scope: with tsocket.socket() as sock: - async def _resolve_remote_address_nocp(self, *args, **kwargs): + async def _resolve_address_nocp(self, *args, **kwargs): cancel_scope.cancel() await _core.checkpoint() - sock._resolve_remote_address_nocp = _resolve_remote_address_nocp + sock._resolve_address_nocp = _resolve_address_nocp with assert_checkpoints(): with pytest.raises(_core.Cancelled): await sock.connect("") From d1866f618117ebb38ac5996a5fc296636fcaed9d Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 18:27:22 -0700 Subject: [PATCH 0932/1498] Add randomized dtls handshake robustness test --- trio/_dtls.py | 71 ++++--- trio/testing/__init__.py | 2 + trio/testing/_fake_net.py | 388 +++++++++++++++++++++++++++++++++++++ trio/tests/test_dtls.py | 91 ++++++++- trio/tests/test_fakenet.py | 42 ++++ 5 files changed, 566 insertions(+), 28 deletions(-) create mode 100644 trio/testing/_fake_net.py create mode 100644 trio/tests/test_fakenet.py diff --git a/trio/_dtls.py b/trio/_dtls.py index 6a0ca3edb2..07aaec64f5 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -11,6 +11,7 @@ import enum from itertools import count import weakref +import errno import attr @@ -318,7 +319,7 @@ def decode_volley_trusted(volley): # large that it has to be fragmented to fit into a single packet. if record.epoch_seqno & EPOCH_MASK: messages.append(OpaqueHandshakeMessage(record)) - elif record.content_type == ContentType.change_cipher_spec: + elif record.content_type in (ContentType.change_cipher_spec, ContentType.alert): messages.append( PseudoHandshakeMessage( record.version, record.content_type, record.payload @@ -460,7 +461,7 @@ def encode_volley(self, messages, mtu): # - The current time (using Trio's clock), rounded to the nearest 30 seconds # - A random salt # -# Then the cookie the salt + the HMAC digest. +# Then the cookie is the salt and the HMAC digest concatenated together. # # When verifying a cookie, we use the salt + new ip/port/ClientHello data to recompute # the HMAC digest, for both the current time and the current time minus 30 seconds, and @@ -525,7 +526,7 @@ def valid_cookie(cookie, address, client_hello_bits): tick = _current_cookie_tick() cur_cookie = _make_cookie(salt, tick, address, client_hello_bits) - old_cookie = _make_cookie(salt, tick - 1, address, client_hello_bits) + old_cookie = _make_cookie(salt, max(tick - 1, 0), address, client_hello_bits) # I doubt using a short-circuiting 'or' here would leak any meaningful # information, but why risk it when '|' is just as easy. @@ -620,12 +621,13 @@ async def handle_client_hello_untrusted(dtls, address, packet): return old_stream = dtls._streams.get(address) if old_stream is not None: - if old_stream._client_hello == packet: + if old_stream._client_hello == (cookie, bits): # ...but it's just a duplicate of a packet we got before, so never mind. return else: # Ok, this *really is* a new handshake; the old stream should go away. - old_stream._replaced() + old_stream._set_replaced() + stream._client_hello = (cookie, bits) dtls._streams[address] = stream dtls._incoming_connections_q.s.send_nowait(stream) @@ -706,7 +708,7 @@ def __init__(self, dtls, peer_address, ctx): self._handshake_lock = trio.Lock() self._record_encoder = RecordEncoder() - def _replaced(self): + def _set_replaced(self): self._replaced = True # Any packets we already received could maybe possibly still be processed, but # there are no more coming. So we close this on the sender side. @@ -714,7 +716,7 @@ def _replaced(self): def _check_replaced(self): if self._replaced: - raise BrokenResourceError( + raise trio.BrokenResourceError( "peer tore down this connection to start a new one" ) @@ -747,7 +749,6 @@ async def aclose(self): await trio.lowlevel.checkpoint() def _inject_client_hello_untrusted(self, packet): - self._client_hello = packet self._ssl.bio_write(packet) # If we're on the server side, then we already sent record 0 as our cookie # challenge. So we want to start the handshake proper with record 1. @@ -787,6 +788,7 @@ def read_volley(): if ( new_volley_messages and volley_messages + and isinstance(new_volley_messages[0], HandshakeMessage) and new_volley_messages[0].msg_seq == volley_messages[0].msg_seq ): # openssl decided to retransmit; discard because we handle @@ -819,7 +821,9 @@ def read_volley(): self._ssl.bio_write(packet) try: self._ssl.do_handshake() - except SSL.WantReadError: + # We ignore generic SSL.Error here, because you can get those + # from random invalid packets + except (SSL.WantReadError, SSL.Error): pass else: # No exception -> the handshake is done, and we can @@ -832,17 +836,25 @@ def read_volley(): return maybe_volley = read_volley() if maybe_volley: - # We managed to get all of the peer's volley and generate a - # new one ourselves! break out of the 'for' loop and restart - # the timer. - volley_messages = maybe_volley - # "Implementations SHOULD retain the current timer value - # until a transmission without loss occurs, at which time - # the value may be reset to the initial value." - if volley_failed_sends == 0: - timeout = initial_retransmit_timeout - volley_failed_sends = 0 - break + if (isinstance(maybe_volley[0], PseudoHandshakeMessage) + and maybe_volley[0].content_type == ContentType.alert): + # we're sending an alert (e.g. due to a corrupted + # packet). We want to send it once, but don't save it to + # retransmit -- keep the last volley as the current + # volley. + await self._send_volley(maybe_volley) + else: + # We managed to get all of the peer's volley and + # generate a new one ourselves! break out of the 'for' + # loop and restart the timer. + volley_messages = maybe_volley + # "Implementations SHOULD retain the current timer value + # until a transmission without loss occurs, at which + # time the value may be reset to the initial value." + if volley_failed_sends == 0: + timeout = initial_retransmit_timeout + volley_failed_sends = 0 + break else: assert self._replaced self._check_replaced() @@ -875,13 +887,18 @@ async def send(self, data): async def receive(self): if not self._did_handshake: await self.do_handshake() - try: - packet = await self._q.r.receive() - except trio.EndOfChannel: - assert self._replaced - self._check_replaced() - self._ssl.bio_write(packet) - return _read_loop(self._ssl.read) + while True: + try: + packet = await self._q.r.receive() + except trio.EndOfChannel: + assert self._replaced + self._check_replaced() + # Don't return spurious empty packets because of stray handshake packets + # coming in late + if part_of_handshake_untrusted(packet): + continue + self._ssl.bio_write(packet) + return _read_loop(self._ssl.read) class DTLS(metaclass=Final): diff --git a/trio/testing/__init__.py b/trio/testing/__init__.py index aa15c4743e..cf631f5716 100644 --- a/trio/testing/__init__.py +++ b/trio/testing/__init__.py @@ -24,6 +24,8 @@ from ._network import open_stream_to_socket_listener +from ._fake_net import FakeNet + ################################################################ from .._util import fixup_module_metadata diff --git a/trio/testing/_fake_net.py b/trio/testing/_fake_net.py new file mode 100644 index 0000000000..50a233ec71 --- /dev/null +++ b/trio/testing/_fake_net.py @@ -0,0 +1,388 @@ +# This should eventually be cleaned up and become public, but for right now I'm just +# implementing enough to test DTLS. + +# TODO: +# - user-defined routers +# - TCP +# - UDP broadcast + +import trio +import attr +import ipaddress +from collections import deque +import errno +import os +from typing import Union, List, Optional +import enum +from contextlib import contextmanager + +from trio._util import Final, NoPublicConstructor + +IPAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] + +def _family_for(ip: IPAddress) -> int: + if isinstance(ip, ipaddress.IPv4Address): + return trio.socket.AF_INET + elif isinstance(ip, ipaddress.IPv6Address): + return trio.socket.AF_INET6 + assert False # pragma: no cover + + +def _wildcard_ip_for(family: int) -> IPAddress: + if family == trio.socket.AF_INET: + return ipaddress.ip_address("0.0.0.0") + elif family == trio.socket.AF_INET6: + return ipaddress.ip_address("::") + else: + assert False + + +def _localhost_ip_for(family: int) -> IPAddress: + if family == trio.socket.AF_INET: + return ipaddress.ip_address("127.0.0.1") + elif family == trio.socket.AF_INET6: + return ipaddress.ip_address("::1") + else: + assert False + + +def _fake_err(code): + raise OSError(code, os.strerror(code)) + + +def _scatter(data, buffers): + written = 0 + for buf in buffers: + next_piece = data[written:written + len(buf)] + with memoryview(buf) as mbuf: + mbuf[:len(next_piece)] = next_piece + written += len(next_piece) + if written == len(data): + break + return written + + +@attr.frozen +class UDPEndpoint: + ip: IPAddress + port: int + + def as_python_sockaddr(self): + sockaddr = (self.ip.compressed, self.port) + if isinstance(self.ip, ipaddress.IPv6Address): + sockaddr += (0, 0) + return sockaddr + + @classmethod + def from_python_sockaddr(cls, sockaddr): + ip, port = sockaddr[:2] + return cls(ip=ipaddress.ip_address(ip), port=port) + + +@attr.frozen +class UDPBinding: + local: UDPEndpoint + + +@attr.frozen +class UDPPacket: + source: UDPEndpoint + destination: UDPEndpoint + payload: bytes + + def reply(self, payload): + return UDPPacket( + source=self.destination, destination=self.source, payload=payload + ) + + +@attr.frozen +class FakeSocketFactory(trio.abc.SocketFactory): + fake_net: "FakeNet" + + def socket(self, family: int, type: int, proto: int) -> "FakeSocket": + return FakeSocket._create(self.fake_net, family, type, proto) + + +@attr.frozen +class FakeHostnameResolver(trio.abc.HostnameResolver): + fake_net: "FakeNet" + + async def getaddrinfo( + self, host: str, port: Union[int, str], family=0, type=0, proto=0, flags=0 + ): + raise NotImplementedError("FakeNet doesn't do fake DNS yet") + + async def getnameinfo(self, sockaddr, flags: int): + raise NotImplementedError("FakeNet doesn't do fake DNS yet") + + +class FakeNet(metaclass=Final): + def __init__(self): + # When we need to pick an arbitrary unique ip address/port, use these: + self._auto_ipv4_iter = ipaddress.IPv4Network("1.0.0.0/8").hosts() + self._auto_ipv4_iter = ipaddress.IPv6Network("1::/16").hosts() + self._auto_port_iter = iter(range(50000, 65535)) + + self._bound: Dict[UDPBinding, FakeSocket] = {} + + self.route_packet = None + + def _bind(self, binding: UDPBinding, socket: "FakeSocket") -> None: + if binding in self._bound: + _fake_err(errno.EADDRINUSE) + self._bound[binding] = socket + + def enable(self) -> None: + trio.socket.set_custom_socket_factory(FakeSocketFactory(self)) + trio.socket.set_custom_hostname_resolver(FakeHostnameResolver(self)) + + def send_packet(self, packet) -> None: + if self.route_packet is None: + self.deliver_packet(packet) + else: + self.route_packet(packet) + + def deliver_packet(self, packet) -> None: + binding = UDPBinding(local=packet.destination) + if binding in self._bound: + self._bound[binding]._deliver_packet(packet) + else: + # No valid destination, so drop it + pass + + +class FakeSocket(trio.socket.SocketType, metaclass=NoPublicConstructor): + def __init__(self, fake_net: FakeNet, family: int, type: int, proto: int): + self._fake_net = fake_net + + if not family: + family = trio.socket.AF_INET + if not type: + type = trio.socket.SOCK_STREAM + + if family not in (trio.socket.AF_INET, trio.socket.AF_INET6): + raise NotImplementedError(f"FakeNet doesn't (yet) support family={family}") + if type != trio.socket.SOCK_DGRAM: + raise NotImplementedError(f"FakeNet doesn't (yet) support type={type}") + + self.family = family + self.type = type + self.proto = proto + + self._closed = False + + self._packet_sender, self._packet_receiver = trio.open_memory_channel(float("inf")) + + # This is the source-of-truth for what port etc. this socket is bound to + self._binding: Optional[UDPBinding] = None + + def _check_closed(self): + if self._closed: + _fake_err(errno.EBADF) + + def close(self): + #breakpoint() + if self._closed: + return + self._closed = True + if self._binding is not None: + del self._fake_net._bound[self._binding] + self._packet_receiver.close() + + async def _resolve_address_nocp(self, address, *, local): + return await trio._socket._resolve_address_nocp(self.type, self.family, + self.proto, address=address, + ipv6_v6only=False, local=local) + + def _deliver_packet(self, packet: UDPPacket): + try: + self._packet_sender.send_nowait(packet) + except trio.BrokenResourceError: + # sending to a closed socket -- UDP packets get dropped + pass + + ################################################################ + # Actual IO operation implementations + ################################################################ + + async def bind(self, addr): + self._check_closed() + if self._binding is not None: + _fake_error(errno.EINVAL) + await trio.lowlevel.checkpoint() + ip_str, port = await self._resolve_address_nocp(addr, local=True) + ip = ipaddress.ip_address(ip_str) + assert _family_for(ip) == self.family + # We convert binds to INET_ANY into binds to localhost + if ip == ipaddress.ip_address("0.0.0.0"): + ip = ipaddress.ip_address("127.0.0.1") + elif ip == ipaddress.ip_address("::"): + ip = ipaddress.ip_address("::1") + if port == 0: + port = next(self._fake_net._auto_port_iter) + binding = UDPBinding(local=UDPEndpoint(ip, port)) + self._fake_net._bind(binding, self) + self._binding = binding + + async def connect(self, peer): + raise NotImplementedError("FakeNet does not (yet) support connected sockets") + + async def sendmsg(self, *args): + self._check_closed() + ancdata = [] + flags = 0 + address = None + if len(args) == 1: + (buffers,) = args + elif len(args) == 2: + buffers, address = args + elif len(args) == 3: + buffers, flags, address = args + elif len(args) == 4: + buffers, ancdata, flags, address = args + else: + raise TypeError("wrong number of arguments") + + await trio.lowlevel.checkpoint() + + if address is not None: + address = await self._resolve_address_nocp(address, local=False) + if ancdata: + raise NotImplementedError("FakeNet doesn't support ancillary data") + if flags: + raise NotImplementedError(f"FakeNet send flags must be 0, not {flags}") + + if address is None: + _fake_err(errno.ENOTCONN) + + destination = UDPEndpoint.from_python_sockaddr(address) + + if self._binding is None: + await self.bind((_wildcard_ip_for(self.family).compressed, 0)) + + payload = b"".join(buffers) + + packet = UDPPacket( + source=self._binding.local, + destination=destination, + payload=payload, + ) + + self._fake_net.send_packet(packet) + + return len(payload) + + async def recvmsg_into(self, buffers, ancbufsize=0, flags=0): + if ancbufsize != 0: + raise NotImplementedError("FakeNet doesn't support ancillary data") + if flags != 0: + raise NotImplementedError("FakeNet doesn't support any recv flags") + + self._check_closed() + + ancdata = [] + msg_flags = 0 + + packet = await self._packet_receiver.receive() + address = packet.source.as_python_sockaddr() + written = _scatter(packet.payload, buffers) + if written < len(packet.payload): + msg_flags |= trio.socket.MSG_TRUNC + return written, ancdata, msg_flags, address + + ################################################################ + # Simple state query stuff + ################################################################ + + def getsockname(self): + self._check_closed() + if self._binding is not None: + return self._binding.local.as_python_sockaddr() + elif self.family == trio.socket.AF_INET: + return ("0.0.0.0", 0) + else: + assert self.family == trio.socket.AF_INET6 + return ("::", 0) + + def getpeername(self): + self._check_closed() + if self._binding is not None: + if self._binding.remote is not None: + return self._binding.remote.as_python_sockaddr() + _fake_err(errno.ENOTCONN) + + def getsockopt(self, level, item): + self._check_closed() + raise OSError(f"FakeNet doesn't implement getsockopt({level}, {item})") + + def setsockopt(self, level, item, value): + self._check_closed() + + if (level, item) == (trio.socket.IPPROTO_IPV6, trio.socket.IPV6_V6ONLY): + if not value: + raise NotImplementedError("FakeNet always has IPV6_V6ONLY=True") + + raise OSError(f"FakeNet doesn't implement setsockopt({level}, {item}, ...)") + + ################################################################ + # Various boilerplate and trivial stubs + ################################################################ + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self.close() + + async def send(self, data, flags=0): + return await self.sendto(data, flags, None) + + async def sendto(self, *args): + if len(args) == 2: + data, address = args + flags = 0 + elif len(args) == 3: + data, flags, address = args + else: + raise TypeError("wrong number of arguments") + return await self.sendmsg([data], [], flags, address) + + async def recv(self, bufsize, flags=0): + data, address = await self.recvfrom(bufsize, flags) + return data + + async def recv_into(self, buf, nbytes=0, flags=0): + got_bytes, address = await self.recvfrom_into(buf, nbytes, flags) + return got_bytes + + async def recvfrom(self, bufsize, flags=0): + data, ancdata, msg_flags, address = await self.recvmsg(bufsize, flags) + return data, address + + async def recvfrom_into(self, buf, nbytes=0, flags=0): + if nbytes != 0 and nbytes != len(buf): + raise NotImplementedError("partial recvfrom_into") + got_nbytes, ancdata, msg_flags, address = await self.recvmsg_into([buf], 0, flags) + return got_nbytes, address + + async def recvmsg(self, bufsize, ancbufsize=0, flags=0): + buf = bytearray(bufsize) + got_nbytes, ancdata, msg_flags, address = await self.recvmsg_into([buf], ancbufsize, flags) + return (bytes(buf[:got_nbytes]), ancdata, msg_flags, address) + + def fileno(self): + raise NotImplementedError("can't get fileno() for FakeNet sockets") + + def detach(self): + raise NotImplementedError("can't detach() a FakeNet socket") + + def get_inheritable(self): + return False + + def set_inheritable(self, inheritable): + if inheritable: + raise NotImplementedError("FakeNet can't make inheritable sockets") + + def share(self, process_id): + raise NotImplementedError("FakeNet can't share sockets") diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 8fcf24a112..5527c63f1e 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -1,5 +1,7 @@ import trio from trio._dtls import DTLS +import random +import attr import trustme from OpenSSL import SSL @@ -17,7 +19,7 @@ async def test_smoke(): server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) with server_sock: - await server_sock.bind(("127.0.0.1", 54321)) + await server_sock.bind(("127.0.0.1", 0)) server_dtls = DTLS(server_sock) async with trio.open_nursery() as nursery: @@ -35,3 +37,90 @@ async def handle_client(dtls_stream): await client.send(b"hello") assert await client.receive() == b"goodbye" nursery.cancel_scope.cancel() + + +async def test_handshake_over_terrible_network(autojump_clock): + HANDSHAKES = 1000 + r = random.Random(0) + fn = trio.testing.FakeNet() + fn.enable() + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as server_sock: + async with trio.open_nursery() as nursery: + async def route_packet(packet): + while True: + op = r.choices(["deliver", "drop", "dupe", "delay"], + weights=[0.7, 0.1, 0.1, 0.1])[0] + print(f"{packet.source} -> {packet.destination}: {op}") + if op == "drop": + return + elif op == "dupe": + fn.send_packet(packet) + elif op == "delay": + await trio.sleep(r.random() * 3) + else: + assert op == "deliver" + print(f"{packet.source} -> {packet.destination}: delivered {packet.payload.hex()}") + fn.deliver_packet(packet) + break + + def route_packet_wrapper(packet): + try: + nursery.start_soon(route_packet, packet) + except RuntimeError: + # We're exiting the nursery, so any remaining packets can just get + # dropped + pass + + fn.route_packet = route_packet_wrapper + + await server_sock.bind(("1.1.1.1", 54321)) + server_dtls = DTLS(server_sock) + + next_client_idx = 0 + next_client_msg_recvd = trio.Event() + + async def handle_client(dtls_stream): + print("handling new client") + try: + await dtls_stream.do_handshake() + while True: + data = await dtls_stream.receive() + print(f"server received plaintext: {data}") + if not data: + continue + assert int(data.decode()) == next_client_idx + next_client_msg_recvd.set() + break + except trio.BrokenResourceError: + # client might have timed out on handshake and started a new one + # so we'll let this task die and let the new task do the check + print("new handshake restarting") + pass + except: + print("server handler saw") + import traceback + traceback.print_exc() + raise + + await nursery.start(server_dtls.serve, server_ctx, handle_client) + + for _ in range(HANDSHAKES): + print("#" * 80) + print("#" * 80) + print("#" * 80) + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: + client_dtls = DTLS(client_sock) + client = await client_dtls.connect(server_sock.getsockname(), client_ctx) + while True: + data = str(next_client_idx).encode() + print(f"client sending plaintext: {data}") + await client.send(data) + with trio.move_on_after(10) as cscope: + await next_client_msg_recvd.wait() + if not cscope.cancelled_caught: + break + + next_client_idx += 1 + next_client_msg_recvd = trio.Event() + + nursery.cancel_scope.cancel() diff --git a/trio/tests/test_fakenet.py b/trio/tests/test_fakenet.py new file mode 100644 index 0000000000..623976f2be --- /dev/null +++ b/trio/tests/test_fakenet.py @@ -0,0 +1,42 @@ +import pytest + +import trio +from trio.testing import FakeNet + +def fn(): + fn = FakeNet() + fn.enable() + return fn + +async def test_basic_udp(): + fn() + s1 = trio.socket.socket(type=trio.socket.SOCK_DGRAM) + s2 = trio.socket.socket(type=trio.socket.SOCK_DGRAM) + + await s1.bind(("127.0.0.1", 0)) + ip, port = s1.getsockname() + assert ip == "127.0.0.1" + assert port != 0 + await s2.sendto(b"xyz", s1.getsockname()) + data, addr = await s1.recvfrom(10) + assert data == b"xyz" + assert addr == s2.getsockname() + await s1.sendto(b"abc", s2.getsockname()) + data, addr = await s2.recvfrom(10) + assert data == b"abc" + assert addr == s1.getsockname() + + +async def test_msg_trunc(): + fn() + s1 = trio.socket.socket(type=trio.socket.SOCK_DGRAM) + s2 = trio.socket.socket(type=trio.socket.SOCK_DGRAM) + await s1.bind(("127.0.0.1", 0)) + await s2.sendto(b"xyz", s1.getsockname()) + data, addr = await s1.recvfrom(10) + + +async def test_basic_tcp(): + fn() + with pytest.raises(NotImplementedError): + trio.socket.socket() From 7378ce7f6fd6ca90a9045c4454afcc7132aeb278 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 18:27:59 -0700 Subject: [PATCH 0933/1498] run black --- trio/_dtls.py | 20 ++++++++++++++++---- trio/_socket.py | 13 ++++++++++--- trio/testing/_fake_net.py | 32 ++++++++++++++++++++++---------- trio/tests/test_dtls.py | 20 +++++++++++++++----- trio/tests/test_fakenet.py | 2 ++ trio/tests/test_socket.py | 5 ++++- 6 files changed, 69 insertions(+), 23 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 07aaec64f5..c0ecb771dd 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -152,7 +152,10 @@ def records_untrusted(packet): def encode_record(record): header = RECORD_HEADER.pack( - record.content_type, record.version, record.epoch_seqno, len(record.payload), + record.content_type, + record.version, + record.epoch_seqno, + len(record.payload), ) return header + record.payload @@ -196,7 +199,14 @@ def decode_handshake_fragment_untrusted(payload): frag = payload[HANDSHAKE_MESSAGE_HEADER.size :] if len(frag) != frag_len: raise BadPacket("handshake fragment length doesn't match record length") - return HandshakeFragment(msg_type, msg_len, msg_seq, frag_offset, frag_len, frag,) + return HandshakeFragment( + msg_type, + msg_len, + msg_seq, + frag_offset, + frag_len, + frag, + ) def encode_handshake_fragment(hsf): @@ -836,8 +846,10 @@ def read_volley(): return maybe_volley = read_volley() if maybe_volley: - if (isinstance(maybe_volley[0], PseudoHandshakeMessage) - and maybe_volley[0].content_type == ContentType.alert): + if ( + isinstance(maybe_volley[0], PseudoHandshakeMessage) + and maybe_volley[0].content_type == ContentType.alert + ): # we're sending an alert (e.g. due to a corrupted # packet). We want to send it once, but don't save it to # retransmit -- keep the last volley as the current diff --git a/trio/_socket.py b/trio/_socket.py index 3747bfe4d5..bcff1ee9e7 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -425,6 +425,7 @@ async def _resolve_address_nocp(type, family, proto, *, ipv6_v6only, address, lo normed = tuple(normed) return normed + class SocketType: def __init__(self): raise TypeError( @@ -558,13 +559,19 @@ async def wait_writable(self): async def _resolve_address_nocp(self, address, *, local): if self.family == _stdlib_socket.AF_INET6: - ipv6_v6only = self._sock.getsockopt(IPPROTO_IPV6, _stdlib_socket.IPV6_V6ONLY) + ipv6_v6only = self._sock.getsockopt( + IPPROTO_IPV6, _stdlib_socket.IPV6_V6ONLY + ) else: ipv6_v6only = False return await _resolve_address_nocp( - self.type, self.family, self.proto, + self.type, + self.family, + self.proto, ipv6_v6only=ipv6_v6only, - address=address, local=local) + address=address, + local=local, + ) async def _nonblocking_helper(self, fn, args, kwargs, wait_fn): # We have to reconcile two conflicting goals: diff --git a/trio/testing/_fake_net.py b/trio/testing/_fake_net.py index 50a233ec71..df853cb87b 100644 --- a/trio/testing/_fake_net.py +++ b/trio/testing/_fake_net.py @@ -20,6 +20,7 @@ IPAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] + def _family_for(ip: IPAddress) -> int: if isinstance(ip, ipaddress.IPv4Address): return trio.socket.AF_INET @@ -53,9 +54,9 @@ def _fake_err(code): def _scatter(data, buffers): written = 0 for buf in buffers: - next_piece = data[written:written + len(buf)] + next_piece = data[written : written + len(buf)] with memoryview(buf) as mbuf: - mbuf[:len(next_piece)] = next_piece + mbuf[: len(next_piece)] = next_piece written += len(next_piece) if written == len(data): break @@ -109,7 +110,7 @@ class FakeHostnameResolver(trio.abc.HostnameResolver): fake_net: "FakeNet" async def getaddrinfo( - self, host: str, port: Union[int, str], family=0, type=0, proto=0, flags=0 + self, host: str, port: Union[int, str], family=0, type=0, proto=0, flags=0 ): raise NotImplementedError("FakeNet doesn't do fake DNS yet") @@ -172,7 +173,9 @@ def __init__(self, fake_net: FakeNet, family: int, type: int, proto: int): self._closed = False - self._packet_sender, self._packet_receiver = trio.open_memory_channel(float("inf")) + self._packet_sender, self._packet_receiver = trio.open_memory_channel( + float("inf") + ) # This is the source-of-truth for what port etc. this socket is bound to self._binding: Optional[UDPBinding] = None @@ -182,7 +185,7 @@ def _check_closed(self): _fake_err(errno.EBADF) def close(self): - #breakpoint() + # breakpoint() if self._closed: return self._closed = True @@ -191,9 +194,14 @@ def close(self): self._packet_receiver.close() async def _resolve_address_nocp(self, address, *, local): - return await trio._socket._resolve_address_nocp(self.type, self.family, - self.proto, address=address, - ipv6_v6only=False, local=local) + return await trio._socket._resolve_address_nocp( + self.type, + self.family, + self.proto, + address=address, + ipv6_v6only=False, + local=local, + ) def _deliver_packet(self, packet: UDPPacket): try: @@ -363,12 +371,16 @@ async def recvfrom(self, bufsize, flags=0): async def recvfrom_into(self, buf, nbytes=0, flags=0): if nbytes != 0 and nbytes != len(buf): raise NotImplementedError("partial recvfrom_into") - got_nbytes, ancdata, msg_flags, address = await self.recvmsg_into([buf], 0, flags) + got_nbytes, ancdata, msg_flags, address = await self.recvmsg_into( + [buf], 0, flags + ) return got_nbytes, address async def recvmsg(self, bufsize, ancbufsize=0, flags=0): buf = bytearray(bufsize) - got_nbytes, ancdata, msg_flags, address = await self.recvmsg_into([buf], ancbufsize, flags) + got_nbytes, ancdata, msg_flags, address = await self.recvmsg_into( + [buf], ancbufsize, flags + ) return (bytes(buf[:got_nbytes]), ancdata, msg_flags, address) def fileno(self): diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 5527c63f1e..61e6fc2f97 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -33,7 +33,9 @@ async def handle_client(dtls_stream): with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: client_dtls = DTLS(client_sock) - client = await client_dtls.connect(server_sock.getsockname(), client_ctx) + client = await client_dtls.connect( + server_sock.getsockname(), client_ctx + ) await client.send(b"hello") assert await client.receive() == b"goodbye" nursery.cancel_scope.cancel() @@ -46,10 +48,13 @@ async def test_handshake_over_terrible_network(autojump_clock): fn.enable() with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as server_sock: async with trio.open_nursery() as nursery: + async def route_packet(packet): while True: - op = r.choices(["deliver", "drop", "dupe", "delay"], - weights=[0.7, 0.1, 0.1, 0.1])[0] + op = r.choices( + ["deliver", "drop", "dupe", "delay"], + weights=[0.7, 0.1, 0.1, 0.1], + )[0] print(f"{packet.source} -> {packet.destination}: {op}") if op == "drop": return @@ -59,7 +64,9 @@ async def route_packet(packet): await trio.sleep(r.random() * 3) else: assert op == "deliver" - print(f"{packet.source} -> {packet.destination}: delivered {packet.payload.hex()}") + print( + f"{packet.source} -> {packet.destination}: delivered {packet.payload.hex()}" + ) fn.deliver_packet(packet) break @@ -99,6 +106,7 @@ async def handle_client(dtls_stream): except: print("server handler saw") import traceback + traceback.print_exc() raise @@ -110,7 +118,9 @@ async def handle_client(dtls_stream): print("#" * 80) with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: client_dtls = DTLS(client_sock) - client = await client_dtls.connect(server_sock.getsockname(), client_ctx) + client = await client_dtls.connect( + server_sock.getsockname(), client_ctx + ) while True: data = str(next_client_idx).encode() print(f"client sending plaintext: {data}") diff --git a/trio/tests/test_fakenet.py b/trio/tests/test_fakenet.py index 623976f2be..4e1c45b45f 100644 --- a/trio/tests/test_fakenet.py +++ b/trio/tests/test_fakenet.py @@ -3,11 +3,13 @@ import trio from trio.testing import FakeNet + def fn(): fn = FakeNet() fn.enable() return fn + async def test_basic_udp(): fn() s1 = trio.socket.socket(type=trio.socket.SOCK_DGRAM) diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index b7c4839981..1fa3721f91 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -559,7 +559,10 @@ async def res(*args): except (AttributeError, OSError): pass else: - assert await netlink_sock._resolve_address_nocp("asdf", local=local) == "asdf" + assert ( + await netlink_sock._resolve_address_nocp("asdf", local=local) + == "asdf" + ) netlink_sock.close() with pytest.raises(ValueError): From dcc5eae5de83a66f8b8d6c826e676dc73390416a Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 19:22:34 -0700 Subject: [PATCH 0934/1498] Mark randomized test as slow --- trio/tests/test_dtls.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 61e6fc2f97..24693ef1ed 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -1,3 +1,4 @@ +import pytest import trio from trio._dtls import DTLS import random @@ -6,6 +7,8 @@ import trustme from OpenSSL import SSL +from .._core.tests.tutil import slow + ca = trustme.CA() server_cert = ca.issue_cert("example.com") @@ -41,6 +44,7 @@ async def handle_client(dtls_stream): nursery.cancel_scope.cancel() +@slow async def test_handshake_over_terrible_network(autojump_clock): HANDSHAKES = 1000 r = random.Random(0) From 4a0bf1d029a92ab8658dc641be2628cf9af50e4f Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 19:25:46 -0700 Subject: [PATCH 0935/1498] Make FakeNet private for now --- trio/testing/__init__.py | 2 -- trio/tests/test_dtls.py | 3 ++- trio/tests/test_fakenet.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/trio/testing/__init__.py b/trio/testing/__init__.py index cf631f5716..aa15c4743e 100644 --- a/trio/testing/__init__.py +++ b/trio/testing/__init__.py @@ -24,8 +24,6 @@ from ._network import open_stream_to_socket_listener -from ._fake_net import FakeNet - ################################################################ from .._util import fixup_module_metadata diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 24693ef1ed..b2ba10442c 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -7,6 +7,7 @@ import trustme from OpenSSL import SSL +from trio.testing._fake_net import FakeNet from .._core.tests.tutil import slow ca = trustme.CA() @@ -48,7 +49,7 @@ async def handle_client(dtls_stream): async def test_handshake_over_terrible_network(autojump_clock): HANDSHAKES = 1000 r = random.Random(0) - fn = trio.testing.FakeNet() + fn = FakeNet() fn.enable() with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as server_sock: async with trio.open_nursery() as nursery: diff --git a/trio/tests/test_fakenet.py b/trio/tests/test_fakenet.py index 4e1c45b45f..bc691c9db5 100644 --- a/trio/tests/test_fakenet.py +++ b/trio/tests/test_fakenet.py @@ -1,7 +1,7 @@ import pytest import trio -from trio.testing import FakeNet +from trio.testing._fake_net import FakeNet def fn(): From c2cfb1c1729c4fcb77ddeafa823f7be9c75eb23e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 20:37:15 -0700 Subject: [PATCH 0936/1498] make list of tests to write --- trio/tests/test_dtls.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index b2ba10442c..1bdf4ffe8d 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -139,3 +139,24 @@ async def handle_client(dtls_stream): next_client_msg_recvd = trio.Event() nursery.cancel_scope.cancel() + + +# send all kinds of garbage at a server socket +# send hello at a client-only socket +# socket closed at terrible times +# cancelling and restarting a client handshake +# garbage collecting DTLS object without closing it +# incoming packets buffer overflow +# set/get mtu +# closing a DTLSStream +# two simultaneous calls to .do_handshake() +# openssl retransmit +# receive a piece of garbage from the correct source during a handshake (corrupted +# packet, someone being a jerk) -- though can't necessarily tolerate someone sending a +# fake HelloRetryRequest +# implicit handshake on send/receive +# send/receive after closing +# DTLS close +# DTLS on SOCK_STREAM socket +# calling serve twice +# connect() that replaces an existing association (currently totally broken!) From 5a224c1b5d2f08ad118abc31ed669b3667ef1ef3 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 20:42:34 -0700 Subject: [PATCH 0937/1498] Rename DTLSStream -> DTLSChannel --- trio/__init__.py | 2 +- trio/_dtls.py | 8 ++++---- trio/tests/test_dtls.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index b4b77f7fa9..782220231e 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -74,7 +74,7 @@ from ._ssl import SSLStream, SSLListener, NeedHandshakeError -from ._dtls import DTLS, DTLSStream +from ._dtls import DTLS, DTLSChannel from ._highlevel_serve_listeners import serve_listeners diff --git a/trio/_dtls.py b/trio/_dtls.py index c0ecb771dd..585b565351 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -623,7 +623,7 @@ async def handle_client_hello_untrusted(dtls, address, packet): pass else: # We got a real, valid ClientHello! - stream = DTLSStream._create(dtls, address, dtls._listening_context) + stream = DTLSChannel._create(dtls, address, dtls._listening_context) try: stream._inject_client_hello_untrusted(packet) except BadPacket: @@ -694,7 +694,7 @@ async def dtls_receive_loop(dtls): del dtls -class DTLSStream(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): +class DTLSChannel(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): def __init__(self, dtls, peer_address, ctx): self.dtls = dtls self.peer_address = peer_address @@ -929,7 +929,7 @@ def __init__(self, socket, *, incoming_packets_buffer=10): # separately. We only keep one connection per remote address; as soon # as a peer provides a valid cookie, we can immediately tear down the # old connection. - # {remote address: DTLSStream} + # {remote address: DTLSChannel} self._streams = weakref.WeakValueDictionary() self._listening_context = None self._incoming_connections_q = _Queue(float("inf")) @@ -981,7 +981,7 @@ def _set_stream_for(self, address, stream): self._streams[address] = stream async def connect(self, address, ssl_context): - stream = DTLSStream._create(self, address, ssl_context) + stream = DTLSChannel._create(self, address, ssl_context) stream._ssl.set_connect_state() self._set_stream_for(address, stream) await stream.do_handshake() diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 1bdf4ffe8d..2fc124a7b3 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -148,7 +148,7 @@ async def handle_client(dtls_stream): # garbage collecting DTLS object without closing it # incoming packets buffer overflow # set/get mtu -# closing a DTLSStream +# closing a DTLSChannel # two simultaneous calls to .do_handshake() # openssl retransmit # receive a piece of garbage from the correct source during a handshake (corrupted From d94713a3081d5ebda56409dc424c342b6952b5ea Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 21:06:26 -0700 Subject: [PATCH 0938/1498] more tests --- trio/tests/test_dtls.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 2fc124a7b3..f50986daf1 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -8,7 +8,7 @@ from OpenSSL import SSL from trio.testing._fake_net import FakeNet -from .._core.tests.tutil import slow +from .._core.tests.tutil import slow, can_bind_ipv6 ca = trustme.CA() server_cert = ca.issue_cert("example.com") @@ -20,10 +20,19 @@ ca.configure_trust(client_ctx) -async def test_smoke(): - server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) +families = [trio.socket.AF_INET] +if can_bind_ipv6: + families.append(trio.socket.AF_INET6) + +@pytest.mark.parametrize("family", families) +async def test_smoke(family): + if family == trio.socket.AF_INET: + localhost = "127.0.0.1" + else: + localhost = "::1" + server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) with server_sock: - await server_sock.bind(("127.0.0.1", 0)) + await server_sock.bind((localhost, 0)) server_dtls = DTLS(server_sock) async with trio.open_nursery() as nursery: @@ -35,13 +44,21 @@ async def handle_client(dtls_stream): await nursery.start(server_dtls.serve, server_ctx, handle_client) - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: + with trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) as client_sock: client_dtls = DTLS(client_sock) client = await client_dtls.connect( server_sock.getsockname(), client_ctx ) await client.send(b"hello") assert await client.receive() == b"goodbye" + + client.set_ciphertext_mtu(1234) + cleartext_mtu_1234 = client.get_cleartext_mtu() + client.set_ciphertext_mtu(4321) + assert client.get_cleartext_mtu() > cleartext_mtu_1234 + client.set_ciphertext_mtu(1234) + assert client.get_cleartext_mtu() == cleartext_mtu_1234 + nursery.cancel_scope.cancel() @@ -141,22 +158,22 @@ async def handle_client(dtls_stream): nursery.cancel_scope.cancel() +# implicit handshake on send/receive +# send/receive after closing +# DTLS close +# DTLS on SOCK_STREAM socket +# incoming packets buffer overflow + # send all kinds of garbage at a server socket # send hello at a client-only socket # socket closed at terrible times -# cancelling and restarting a client handshake +# cancelling a client handshake and then starting a new one # garbage collecting DTLS object without closing it -# incoming packets buffer overflow -# set/get mtu # closing a DTLSChannel # two simultaneous calls to .do_handshake() # openssl retransmit # receive a piece of garbage from the correct source during a handshake (corrupted # packet, someone being a jerk) -- though can't necessarily tolerate someone sending a # fake HelloRetryRequest -# implicit handshake on send/receive -# send/receive after closing -# DTLS close -# DTLS on SOCK_STREAM socket # calling serve twice # connect() that replaces an existing association (currently totally broken!) From def73ede4e82aa0e1fa3d1f4080876e55f8d8caa Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 21:48:29 -0700 Subject: [PATCH 0939/1498] more tests --- trio/_dtls.py | 16 +++++- trio/tests/test_dtls.py | 108 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 111 insertions(+), 13 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 585b565351..a72a12cd8e 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -920,8 +920,10 @@ def __init__(self, socket, *, incoming_packets_buffer=10): global SSL from OpenSSL import SSL + self.socket = None # for __del__ if socket.type != trio.socket.SOCK_DGRAM: - raise BadPacket("DTLS requires a SOCK_DGRAM socket") + raise ValueError("DTLS requires a SOCK_DGRAM socket") + self.socket = socket self.incoming_packets_buffer = incoming_packets_buffer self._token = trio.lowlevel.current_trio_token() @@ -934,20 +936,24 @@ def __init__(self, socket, *, incoming_packets_buffer=10): self._listening_context = None self._incoming_connections_q = _Queue(float("inf")) self._send_lock = trio.Lock() + self._closed = False trio.lowlevel.spawn_system_task(dtls_receive_loop, self) def __del__(self): # Close the socket in Trio context (if our Trio context still exists), so that # the background task gets notified about the closure and can exit. + if self.socket is None: + return try: self._token.run_sync_soon(self.socket.close) except RuntimeError: pass def close(self): + self._closed = True self.socket.close() - for stream in self._streams.values(): + for stream in list(self._streams.values()): stream.close() self._incoming_connections_q.s.close() @@ -957,9 +963,14 @@ def __enter__(self): def __exit__(self, *args): self.close() + def _check_closed(self): + if self._closed: + raise trio.ClosedResourceError + async def serve( self, ssl_context, async_fn, *args, task_status=trio.TASK_STATUS_IGNORED ): + self._check_closed() if self._listening_context is not None: raise trio.BusyResourceError("another task is already listening") # We do cookie verification ourselves, so tell OpenSSL not to worry about it. @@ -981,6 +992,7 @@ def _set_stream_for(self, address, stream): self._streams[address] = stream async def connect(self, address, ssl_context): + self._check_closed() stream = DTLSChannel._create(self, address, ssl_context) stream._ssl.set_connect_state() self._set_stream_for(address, stream) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index f50986daf1..d350a539b7 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -3,6 +3,7 @@ from trio._dtls import DTLS import random import attr +from contextlib import asynccontextmanager import trustme from OpenSSL import SSL @@ -37,10 +38,10 @@ async def test_smoke(family): async with trio.open_nursery() as nursery: - async def handle_client(dtls_stream): - await dtls_stream.do_handshake() - assert await dtls_stream.receive() == b"hello" - await dtls_stream.send(b"goodbye") + async def handle_client(dtls_channel): + await dtls_channel.do_handshake() + assert await dtls_channel.receive() == b"hello" + await dtls_channel.send(b"goodbye") await nursery.start(server_dtls.serve, server_ctx, handle_client) @@ -108,12 +109,12 @@ def route_packet_wrapper(packet): next_client_idx = 0 next_client_msg_recvd = trio.Event() - async def handle_client(dtls_stream): + async def handle_client(dtls_channel): print("handling new client") try: - await dtls_stream.do_handshake() + await dtls_channel.do_handshake() while True: - data = await dtls_stream.receive() + data = await dtls_channel.receive() print(f"server received plaintext: {data}") if not data: continue @@ -158,10 +159,95 @@ async def handle_client(dtls_stream): nursery.cancel_scope.cancel() -# implicit handshake on send/receive -# send/receive after closing -# DTLS close -# DTLS on SOCK_STREAM socket +async def test_implicit_handshake(): + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as server_sock: + await server_sock.bind(("127.0.0.1", 0)) + server_dtls = DTLS(server_sock) + + +def dtls(): + sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) + return DTLS(sock) + + +@asynccontextmanager +async def dtls_echo_server(*, autocancel=True): + with dtls() as server: + await server.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as nursery: + async def echo_handler(dtls_channel): + async for packet in dtls_channel: + await dtls_channel.send(packet) + + await nursery.start(server.serve, server_ctx, echo_handler) + + yield server, server.socket.getsockname() + + if autocancel: + nursery.cancel_scope.cancel() + + +async def test_implicit_handshake(): + async with dtls_echo_server() as (_, address): + with dtls() as client_endpoint: + client = await client_endpoint.connect(address, client_ctx) + + # Implicit handshake + await client.send(b"xyz") + assert await client.receive() == b"xyz" + + +async def test_channel_closing(): + async with dtls_echo_server() as (_, address): + with dtls() as client_endpoint: + client = await client_endpoint.connect(address, client_ctx) + client.close() + + with pytest.raises(trio.ClosedResourceError): + await client.send(b"abc") + with pytest.raises(trio.ClosedResourceError): + await client.receive() + + +async def test_serve_exits_cleanly_on_close(): + async with dtls_echo_server(autocancel=False) as (server_endpoint, address): + server_endpoint.close() + # Testing that the nursery exits even without being cancelled + + +async def test_client_multiplex(): + async with dtls_echo_server() as (_, address1), dtls_echo_server() as (_, address2): + with dtls() as client_endpoint: + client1 = await client_endpoint.connect(address1, client_ctx) + client2 = await client_endpoint.connect(address2, client_ctx) + + await client1.send(b"abc") + await client2.send(b"xyz") + assert await client2.receive() == b"xyz" + assert await client1.receive() == b"abc" + + client_endpoint.close() + + with pytest.raises(trio.ClosedResourceError): + await client1.send("xxx") + with pytest.raises(trio.ClosedResourceError): + await client2.receive() + with pytest.raises(trio.ClosedResourceError): + await client_endpoint.connect(address1, client_ctx) + + async with trio.open_nursery() as nursery: + with pytest.raises(trio.ClosedResourceError): + async def null_handler(_): # pragma: no cover + pass + await nursery.start(client_endpoint.serve, server_ctx, null_handler) + + +async def test_dtls_over_dgram_only(): + with trio.socket.socket() as s: + with pytest.raises(ValueError): + DTLS(s) + + # incoming packets buffer overflow # send all kinds of garbage at a server socket From f2600525a103dc0ce8fce81fa0377cc59f53bdad Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 4 Sep 2021 21:51:54 -0700 Subject: [PATCH 0940/1498] rename DTLS -> DTLSEndpoint --- trio/__init__.py | 2 +- trio/_dtls.py | 2 +- trio/tests/test_dtls.py | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index 782220231e..b35fa076b3 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -74,7 +74,7 @@ from ._ssl import SSLStream, SSLListener, NeedHandshakeError -from ._dtls import DTLS, DTLSChannel +from ._dtls import DTLSEndpoint, DTLSChannel from ._highlevel_serve_listeners import serve_listeners diff --git a/trio/_dtls.py b/trio/_dtls.py index a72a12cd8e..7d5f598ec0 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -913,7 +913,7 @@ async def receive(self): return _read_loop(self._ssl.read) -class DTLS(metaclass=Final): +class DTLSEndpoint(metaclass=Final): def __init__(self, socket, *, incoming_packets_buffer=10): # We do this lazily on first construction, so only people who actually use DTLS # have to install PyOpenSSL. diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index d350a539b7..763affa00e 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -1,6 +1,6 @@ import pytest import trio -from trio._dtls import DTLS +from trio import DTLSEndpoint import random import attr from contextlib import asynccontextmanager @@ -34,7 +34,7 @@ async def test_smoke(family): server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) with server_sock: await server_sock.bind((localhost, 0)) - server_dtls = DTLS(server_sock) + server_dtls = DTLSEndpoint(server_sock) async with trio.open_nursery() as nursery: @@ -46,7 +46,7 @@ async def handle_client(dtls_channel): await nursery.start(server_dtls.serve, server_ctx, handle_client) with trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) as client_sock: - client_dtls = DTLS(client_sock) + client_dtls = DTLSEndpoint(client_sock) client = await client_dtls.connect( server_sock.getsockname(), client_ctx ) @@ -104,7 +104,7 @@ def route_packet_wrapper(packet): fn.route_packet = route_packet_wrapper await server_sock.bind(("1.1.1.1", 54321)) - server_dtls = DTLS(server_sock) + server_dtls = DTLSEndpoint(server_sock) next_client_idx = 0 next_client_msg_recvd = trio.Event() @@ -140,7 +140,7 @@ async def handle_client(dtls_channel): print("#" * 80) print("#" * 80) with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: - client_dtls = DTLS(client_sock) + client_dtls = DTLSEndpoint(client_sock) client = await client_dtls.connect( server_sock.getsockname(), client_ctx ) @@ -162,12 +162,12 @@ async def handle_client(dtls_channel): async def test_implicit_handshake(): with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as server_sock: await server_sock.bind(("127.0.0.1", 0)) - server_dtls = DTLS(server_sock) + server_dtls = DTLSEndpoint(server_sock) def dtls(): sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) - return DTLS(sock) + return DTLSEndpoint(sock) @asynccontextmanager @@ -245,7 +245,7 @@ async def null_handler(_): # pragma: no cover async def test_dtls_over_dgram_only(): with trio.socket.socket() as s: with pytest.raises(ValueError): - DTLS(s) + DTLSEndpoint(s) # incoming packets buffer overflow From e694e8ee142d78280e690eabedff9dcbc0016813 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 5 Sep 2021 10:53:22 -0700 Subject: [PATCH 0941/1498] full duplex test (+ racing do_handshake) --- trio/tests/test_dtls.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 763affa00e..000d46cdf1 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -197,6 +197,24 @@ async def test_implicit_handshake(): assert await client.receive() == b"xyz" +async def test_full_duplex(): + with dtls() as server_endpoint, dtls() as client_endpoint: + await server_endpoint.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as server_nursery: + async def handler(channel): + async with trio.open_nursery() as nursery: + nursery.start_soon(channel.send, b"from server") + nursery.start_soon(channel.receive) + + await server_nursery.start(server_endpoint.serve, server_ctx, handler) + + client = await client_endpoint.connect(server_endpoint.socket.getsockname(), client_ctx) + async with trio.open_nursery() as nursery: + nursery.start_soon(client.send, b"from client") + nursery.start_soon(client.receive) + + server_nursery.cancel_scope.cancel() + async def test_channel_closing(): async with dtls_echo_server() as (_, address): with dtls() as client_endpoint: @@ -255,8 +273,6 @@ async def test_dtls_over_dgram_only(): # socket closed at terrible times # cancelling a client handshake and then starting a new one # garbage collecting DTLS object without closing it -# closing a DTLSChannel -# two simultaneous calls to .do_handshake() # openssl retransmit # receive a piece of garbage from the correct source during a handshake (corrupted # packet, someone being a jerk) -- though can't necessarily tolerate someone sending a From 17b5f902b6b1f1a14364500023d70c7a7d1a58bf Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 5 Sep 2021 11:06:54 -0700 Subject: [PATCH 0942/1498] testing testing --- trio/_dtls.py | 25 ++++++++++++++----------- trio/tests/test_dtls.py | 21 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 7d5f598ec0..6544cb0d1f 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -695,8 +695,8 @@ async def dtls_receive_loop(dtls): class DTLSChannel(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): - def __init__(self, dtls, peer_address, ctx): - self.dtls = dtls + def __init__(self, endpoint, peer_address, ctx): + self.endpoint = endpoint self.peer_address = peer_address self.packets_dropped_in_trio = 0 self._client_hello = None @@ -711,10 +711,10 @@ def __init__(self, dtls, peer_address, ctx): self._mtu = None # This calls self._ssl.set_ciphertext_mtu, which is important, because if you # don't call it then openssl doesn't work. - self.set_ciphertext_mtu(best_guess_mtu(self.dtls.socket)) + self.set_ciphertext_mtu(best_guess_mtu(self.endpoint.socket)) self._replaced = False self._closed = False - self._q = _Queue(dtls.incoming_packets_buffer) + self._q = _Queue(endpoint.incoming_packets_buffer) self._handshake_lock = trio.Lock() self._record_encoder = RecordEncoder() @@ -748,8 +748,8 @@ def close(self): if self._closed: return self._closed = True - if self.dtls._streams.get(self.peer_address) is self: - del self.dtls._streams[self.peer_address] + if self.endpoint._streams.get(self.peer_address) is self: + del self.endpoint._streams[self.peer_address] # Will wake any tasks waiting on self._q.get with a # ClosedResourceError self._q.r.close() @@ -777,8 +777,8 @@ def _inject_client_hello_untrusted(self, packet): async def _send_volley(self, volley_messages): packets = self._record_encoder.encode_volley(volley_messages, self._mtu) for packet in packets: - async with self.dtls._send_lock: - await self.dtls.socket.sendto(packet, self.peer_address) + async with self.endpoint._send_lock: + await self.endpoint.socket.sendto(packet, self.peer_address) async def _resend_final_volley(self): await self._send_volley(self._final_volley) @@ -881,7 +881,7 @@ def read_volley(): # PMTU estimate is wrong? Let's try dropping it to the minimum # and hope that helps. self.set_ciphertext_mtu( - min(self._mtu, worst_case_mtu(self.dtls.socket)) + min(self._mtu, worst_case_mtu(self.endpoint.socket)) ) async def send(self, data): @@ -891,8 +891,8 @@ async def send(self, data): await self.do_handshake() self._check_replaced() self._ssl.write(data) - async with self.dtls._send_lock: - await self.dtls.socket.sendto( + async with self.endpoint._send_lock: + await self.endpoint.socket.sendto( _read_loop(self._ssl.bio_read), self.peer_address ) @@ -992,6 +992,9 @@ def _set_stream_for(self, address, stream): self._streams[address] = stream async def connect(self, address, ssl_context): + # it would be nice if we could detect when 'address' is our own endpoint (a + # loopback connection), because that can't work + # but I don't see how to do it reliably self._check_closed() stream = DTLSChannel._create(self, address, ssl_context) stream._ssl.set_connect_state() diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 000d46cdf1..c8ac750a99 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -176,6 +176,9 @@ async def dtls_echo_server(*, autocancel=True): await server.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as nursery: async def echo_handler(dtls_channel): + print(f"echo handler started: " + f"server {dtls_channel.endpoint.socket.getsockname()} " + f"client {dtls_channel.peer_address}") async for packet in dtls_channel: await dtls_channel.send(packet) @@ -266,6 +269,23 @@ async def test_dtls_over_dgram_only(): DTLSEndpoint(s) +async def test_double_serve(): + async def null_handler(_): # pragma: no cover + pass + + with dtls() as endpoint: + async with trio.open_nursery() as nursery: + await nursery.start(endpoint.serve, server_ctx, null_handler) + with pytest.raises(trio.BusyResourceError): + await nursery.start(endpoint.serve, server_ctx, null_handler) + + nursery.cancel_scope.cancel() + + async with trio.open_nursery() as nursery: + await nursery.start(endpoint.serve, server_ctx, null_handler) + nursery.cancel_scope.cancel() + + # incoming packets buffer overflow # send all kinds of garbage at a server socket @@ -277,5 +297,4 @@ async def test_dtls_over_dgram_only(): # receive a piece of garbage from the correct source during a handshake (corrupted # packet, someone being a jerk) -- though can't necessarily tolerate someone sending a # fake HelloRetryRequest -# calling serve twice # connect() that replaces an existing association (currently totally broken!) From 8a1e2cbb36ce1214311ae1659f88c195d29acbeb Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 5 Sep 2021 12:59:34 -0700 Subject: [PATCH 0943/1498] Switch DTLSChannel to follow '.statistics()' convention --- trio/_dtls.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 6544cb0d1f..00e8c05ace 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -686,7 +686,7 @@ async def dtls_receive_loop(dtls): try: stream._q.s.send_nowait(packet) except trio.WouldBlock: - stream.packets_dropped_in_trio += 1 + stream._packets_dropped_in_trio += 1 else: # Drop packet pass @@ -694,11 +694,16 @@ async def dtls_receive_loop(dtls): del dtls +@attr.frozen +class DTLSChannelStatistics: + incoming_packets_dropped_in_trio: int + + class DTLSChannel(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): def __init__(self, endpoint, peer_address, ctx): self.endpoint = endpoint self.peer_address = peer_address - self.packets_dropped_in_trio = 0 + self._packets_dropped_in_trio = 0 self._client_hello = None self._did_handshake = False # These are mandatory for all DTLS connections. OP_NO_QUERY_MTU is required to @@ -718,6 +723,9 @@ def __init__(self, endpoint, peer_address, ctx): self._handshake_lock = trio.Lock() self._record_encoder = RecordEncoder() + def statistics(self) -> DTLSChannelStatistics: + return DTLSChannelStatistics(self._packets_dropped_in_trio) + def _set_replaced(self): self._replaced = True # Any packets we already received could maybe possibly still be processed, but From db4b54947f3a606a709a822114cd2041b1e43b1b Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 5 Sep 2021 12:59:49 -0700 Subject: [PATCH 0944/1498] moar tests --- trio/tests/test_dtls.py | 163 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 5 deletions(-) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index c8ac750a99..f6ac32822d 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -4,6 +4,7 @@ import random import attr from contextlib import asynccontextmanager +from itertools import count import trustme from OpenSSL import SSL @@ -165,9 +166,9 @@ async def test_implicit_handshake(): server_dtls = DTLSEndpoint(server_sock) -def dtls(): +def dtls(**kwargs): sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) - return DTLSEndpoint(sock) + return DTLSEndpoint(sock, **kwargs) @asynccontextmanager @@ -180,6 +181,7 @@ async def echo_handler(dtls_channel): f"server {dtls_channel.endpoint.socket.getsockname()} " f"client {dtls_channel.peer_address}") async for packet in dtls_channel: + print(f"echoing {packet} -> {dtls_channel.peer_address}") await dtls_channel.send(packet) await nursery.start(server.serve, server_ctx, echo_handler) @@ -286,13 +288,164 @@ async def null_handler(_): # pragma: no cover nursery.cancel_scope.cancel() -# incoming packets buffer overflow +async def test_connect_to_non_server(autojump_clock): + fn = FakeNet() + fn.enable() + with dtls() as client1, dtls() as client2: + await client1.socket.bind(("127.0.0.1", 0)) + # This should just time out + with trio.move_on_after(100) as cscope: + await client2.connect(client1.socket.getsockname(), client_ctx) + assert cscope.cancelled_caught + + +async def test_incoming_buffer_overflow(autojump_clock): + fn = FakeNet() + fn.enable() + for buffer_size in [10, 20]: + async with dtls_echo_server() as (_, address): + with dtls(incoming_packets_buffer=buffer_size) as client_endpoint: + assert client_endpoint.incoming_packets_buffer == buffer_size + client = await client_endpoint.connect(address, client_ctx) + for i in range(buffer_size + 15): + await client.send(str(i).encode()) + await trio.sleep(1) + stats = client.statistics() + assert stats.incoming_packets_dropped_in_trio == 15 + for i in range(buffer_size): + assert await client.receive() == str(i).encode() + await client.send(b"buffer clear now") + assert await client.receive() == b"buffer clear now" + + +async def test_server_socket_doesnt_crash_on_garbage(autojump_clock): + fn = FakeNet() + fn.enable() + + from trio._dtls import ( + Record, encode_record, HandshakeFragment, encode_handshake_fragment, + ContentType, HandshakeType, ProtocolVersion, + ) + + client_hello = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=10, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ) + ), + ) + ) + + client_hello_extended = client_hello + b"\x00" + client_hello_short = client_hello[:-1] + # cuts off in middle of handshake message header + client_hello_really_short = client_hello[:14] + client_hello_corrupt_record_len = bytearray(client_hello) + client_hello_corrupt_record_len[11] = 0xff + + client_hello_fragmented = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=20, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ) + ), + ) + ) + + client_hello_trailing_data_in_record = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=encode_handshake_fragment( + HandshakeFragment( + msg_type=HandshakeType.client_hello, + msg_len=20, + msg_seq=0, + frag_offset=0, + frag_len=10, + frag=bytes(10), + ) + ) + b"\x00", + ) + ) + async with dtls_echo_server() as (_, address): + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as sock: + for bad_packet in [ + b"", + b"xyz", + client_hello_extended, + client_hello_short, + client_hello_really_short, + client_hello_corrupt_record_len, + client_hello_fragmented, + client_hello_trailing_data_in_record, + ]: + await sock.sendto(bad_packet, address) + await trio.sleep(1) + + +async def test_invalid_cookie_rejected(autojump_clock): + fn = FakeNet() + fn.enable() + + from trio._dtls import decode_client_hello_untrusted, BadPacket + + offset_to_corrupt = count() + def route_packet(packet): + try: + _, cookie, _ = decode_client_hello_untrusted(packet.payload) + except BadPacket: + pass + else: + if len(cookie) != 0: + # this is a challenge response packet + # let's corrupt the next offset so the handshake should fail + payload = bytearray(packet.payload) + offset = next(offset_to_corrupt) + if offset >= len(payload): + # We've tried all offsets + # clamp offset to the end of the payload, and tell the client to stop + # trying to connect + offset = len(payload) - 1 + cscope.cancel() + payload[offset] ^= 0x01 + packet = attr.evolve(packet, payload=payload) + + fn.deliver_packet(packet) + + fn.route_packet = route_packet + + async with dtls_echo_server() as (_, address): + with trio.CancelScope() as cscope: + while True: + with dtls() as client: + await client.connect(address, client_ctx) + assert cscope.cancelled_caught -# send all kinds of garbage at a server socket -# send hello at a client-only socket # socket closed at terrible times # cancelling a client handshake and then starting a new one # garbage collecting DTLS object without closing it + # use fakenet, send a packet to the server, then immediately drop the dtls object and + # run gc before `sock.recvfrom()` can return # openssl retransmit # receive a piece of garbage from the correct source during a handshake (corrupted # packet, someone being a jerk) -- though can't necessarily tolerate someone sending a From 57d75addbe713e4cad9124cc61365d789919e377 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sun, 5 Sep 2021 23:41:22 -0700 Subject: [PATCH 0945/1498] Fixes + tests - use correct initial seqno on server in case of packet loss - calculate timeouts correctly - moar tests --- trio/_dtls.py | 106 ++++++++------- trio/testing/_fake_net.py | 2 +- trio/tests/test_dtls.py | 277 +++++++++++++++++++++++++++++++------- 3 files changed, 288 insertions(+), 97 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 00e8c05ace..b9eb913482 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -127,12 +127,15 @@ def is_client_hello_untrusted(packet): RECORD_HEADER = struct.Struct("!B2sQH") +hex_repr = attr.ib(repr=lambda data: data.hex()) + + @attr.frozen class Record: content_type: int - version: bytes + version: bytes = hex_repr epoch_seqno: int - payload: bytes + payload: bytes = hex_repr def records_untrusted(packet): @@ -176,7 +179,7 @@ class HandshakeFragment: msg_seq: int frag_offset: int frag_len: int - frag: bytes + frag: bytes = attr.ib(repr=lambda f: f.hex()) def decode_handshake_fragment_untrusted(payload): @@ -288,19 +291,19 @@ def decode_client_hello_untrusted(packet): @attr.frozen class HandshakeMessage: - record_version: bytes + record_version: bytes = hex_repr msg_type: HandshakeType msg_seq: int - body: bytearray + body: bytearray = hex_repr # ChangeCipherSpec is part of the handshake, but it's not a "handshake # message" and can't be fragmented the same way. Sigh. @attr.frozen class PseudoHandshakeMessage: - record_version: bytes + record_version: bytes = hex_repr content_type: int - payload: bytes + payload: bytes = hex_repr # The final record in a handshake is Finished, which is encrypted, can't be fragmented @@ -365,8 +368,8 @@ class RecordEncoder: def __init__(self): self._record_seq = count() - def skip_first_record_number(self): - assert next(self._record_seq) == 0 + def set_first_record_number(self, n): + self._record_seq = count(n) def encode_volley(self, messages, mtu): packets = [] @@ -624,15 +627,28 @@ async def handle_client_hello_untrusted(dtls, address, packet): else: # We got a real, valid ClientHello! stream = DTLSChannel._create(dtls, address, dtls._listening_context) + # Our HelloRetryRequest had some sequence number. We need our future sequence + # numbers to be larger than it, so our peer knows that our future records aren't + # stale/duplicates. But, we don't know what this sequence number was. What we do + # know is: + # - the HelloRetryRequest seqno was copied it from the initial ClientHello + # - the new ClientHello has a higher seqno than the initial ClientHello + # So, if we copy the new ClientHello's seqno into our first real handshake + # record and increment from there, that should work. + stream._record_encoder.set_first_record_number(epoch_seqno) + # Process the ClientHello try: - stream._inject_client_hello_untrusted(packet) - except BadPacket: - # ...or, well, OpenSSL didn't like it, so I guess we didn't. + stream._ssl.bio_write(packet) + stream._ssl.DTLSv1_listen() + except SSL.Error: + # ...OpenSSL didn't like it, so I guess we didn't have a valid ClientHello + # after all. return + # Check if we have an existing association old_stream = dtls._streams.get(address) if old_stream is not None: if old_stream._client_hello == (cookie, bits): - # ...but it's just a duplicate of a packet we got before, so never mind. + # ...This was just a duplicate of the last ClientHello, so never mind. return else: # Ok, this *really is* a new handshake; the old stream should go away. @@ -762,26 +778,16 @@ def close(self): # ClosedResourceError self._q.r.close() + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + async def aclose(self): self.close() await trio.lowlevel.checkpoint() - def _inject_client_hello_untrusted(self, packet): - self._ssl.bio_write(packet) - # If we're on the server side, then we already sent record 0 as our cookie - # challenge. So we want to start the handshake proper with record 1. - self._record_encoder.skip_first_record_number() - # We've already validated this cookie. But, we still have to call DTLSv1_listen - # so OpenSSL thinks that it's verified the cookie. The problem is that - # if you're doing cookie challenges, then the actual ClientHello has msg_seq=1 - # instead of msg_seq=0, and OpenSSL will refuse to process a ClientHello with - # msg_seq=1 unless you've called DTLSv1_listen. It also gets OpenSSL to bump the - # outgoing ServerHello's msg_seq to 1. - try: - self._ssl.DTLSv1_listen() - except SSL.Error: - raise BadPacket - async def _send_volley(self, volley_messages): packets = self._record_encoder.encode_volley(volley_messages, self._mtu) for packet in packets: @@ -826,7 +832,7 @@ def read_volley(): # If we don't have messages to send in our initial volley, then something # has gone very wrong. (I'm not sure this can actually happen without an # error from OpenSSL, but we check just in case.) - if not volley_messages: + if not volley_messages: # pragma: no cover raise SSL.Error("something wrong with peer's ClientHello") while True: @@ -834,14 +840,14 @@ def read_volley(): assert volley_messages await self._send_volley(volley_messages) # -- then this is where we wait for a reply -- - with trio.move_on_after(10) as cscope: + with trio.move_on_after(timeout) as cscope: async for packet in self._q.r: self._ssl.bio_write(packet) try: self._ssl.do_handshake() # We ignore generic SSL.Error here, because you can get those # from random invalid packets - except (SSL.WantReadError, SSL.Error): + except (SSL.WantReadError, SSL.Error) as exc: pass else: # No exception -> the handshake is done, and we can @@ -987,25 +993,33 @@ async def serve( try: self._listening_context = ssl_context task_status.started() + + async def handler_wrapper(stream): + with stream: + await async_fn(stream, *args) + async with trio.open_nursery() as nursery: - async for stream in self._incoming_connections_q.r: - nursery.start_soon(async_fn, stream, *args) + async for stream in self._incoming_connections_q.r: # pragma: no branch + nursery.start_soon(handler_wrapper, stream) finally: self._listening_context = None - def _set_stream_for(self, address, stream): - old_stream = self._streams.get(address) - if old_stream is not None: - old_stream._break(RuntimeError("replaced by a new DTLS association")) - self._streams[address] = stream - - async def connect(self, address, ssl_context): + async def connect(self, address, ssl_context, *, initial_retransmit_timeout=1.0): # it would be nice if we could detect when 'address' is our own endpoint (a # loopback connection), because that can't work # but I don't see how to do it reliably self._check_closed() - stream = DTLSChannel._create(self, address, ssl_context) - stream._ssl.set_connect_state() - self._set_stream_for(address, stream) - await stream.do_handshake() - return stream + channel = DTLSChannel._create(self, address, ssl_context) + channel._ssl.set_connect_state() + old_channel = self._streams.get(address) + if old_channel is not None: + old_channel._set_replaced() + self._streams[address] = channel + try: + await channel.do_handshake( + initial_retransmit_timeout=initial_retransmit_timeout + ) + except: + channel.close() + raise + return channel diff --git a/trio/testing/_fake_net.py b/trio/testing/_fake_net.py index df853cb87b..f0ea927734 100644 --- a/trio/testing/_fake_net.py +++ b/trio/testing/_fake_net.py @@ -89,7 +89,7 @@ class UDPBinding: class UDPPacket: source: UDPEndpoint destination: UDPEndpoint - payload: bytes + payload: bytes = attr.ib(repr=lambda p: p.hex()) def reply(self, payload): return UDPPacket( diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index f6ac32822d..bd1842fb25 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -26,6 +26,7 @@ if can_bind_ipv6: families.append(trio.socket.AF_INET6) + @pytest.mark.parametrize("family", families) async def test_smoke(family): if family == trio.socket.AF_INET: @@ -46,7 +47,9 @@ async def handle_client(dtls_channel): await nursery.start(server_dtls.serve, server_ctx, handle_client) - with trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) as client_sock: + with trio.socket.socket( + type=trio.socket.SOCK_DGRAM, family=family + ) as client_sock: client_dtls = DTLSEndpoint(client_sock) client = await client_dtls.connect( server_sock.getsockname(), client_ctx @@ -176,10 +179,13 @@ async def dtls_echo_server(*, autocancel=True): with dtls() as server: await server.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as nursery: + async def echo_handler(dtls_channel): - print(f"echo handler started: " - f"server {dtls_channel.endpoint.socket.getsockname()} " - f"client {dtls_channel.peer_address}") + print( + f"echo handler started: " + f"server {dtls_channel.endpoint.socket.getsockname()} " + f"client {dtls_channel.peer_address}" + ) async for packet in dtls_channel: print(f"echoing {packet} -> {dtls_channel.peer_address}") await dtls_channel.send(packet) @@ -206,6 +212,7 @@ async def test_full_duplex(): with dtls() as server_endpoint, dtls() as client_endpoint: await server_endpoint.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as server_nursery: + async def handler(channel): async with trio.open_nursery() as nursery: nursery.start_soon(channel.send, b"from server") @@ -213,13 +220,16 @@ async def handler(channel): await server_nursery.start(server_endpoint.serve, server_ctx, handler) - client = await client_endpoint.connect(server_endpoint.socket.getsockname(), client_ctx) + client = await client_endpoint.connect( + server_endpoint.socket.getsockname(), client_ctx + ) async with trio.open_nursery() as nursery: nursery.start_soon(client.send, b"from client") nursery.start_soon(client.receive) server_nursery.cancel_scope.cancel() + async def test_channel_closing(): async with dtls_echo_server() as (_, address): with dtls() as client_endpoint: @@ -231,11 +241,18 @@ async def test_channel_closing(): with pytest.raises(trio.ClosedResourceError): await client.receive() + # close is idempotent + client.close() + # can also aclose + await client.aclose() + async def test_serve_exits_cleanly_on_close(): async with dtls_echo_server(autocancel=False) as (server_endpoint, address): server_endpoint.close() # Testing that the nursery exits even without being cancelled + # close is idempotent + server_endpoint.close() async def test_client_multiplex(): @@ -260,8 +277,10 @@ async def test_client_multiplex(): async with trio.open_nursery() as nursery: with pytest.raises(trio.ClosedResourceError): + async def null_handler(_): # pragma: no cover pass + await nursery.start(client_endpoint.serve, server_ctx, null_handler) @@ -323,8 +342,13 @@ async def test_server_socket_doesnt_crash_on_garbage(autojump_clock): fn.enable() from trio._dtls import ( - Record, encode_record, HandshakeFragment, encode_handshake_fragment, - ContentType, HandshakeType, ProtocolVersion, + Record, + encode_record, + HandshakeFragment, + encode_handshake_fragment, + ContentType, + HandshakeType, + ProtocolVersion, ) client_hello = encode_record( @@ -350,7 +374,7 @@ async def test_server_socket_doesnt_crash_on_garbage(autojump_clock): # cuts off in middle of handshake message header client_hello_really_short = client_hello[:14] client_hello_corrupt_record_len = bytearray(client_hello) - client_hello_corrupt_record_len[11] = 0xff + client_hello_corrupt_record_len[11] = 0xFF client_hello_fragmented = encode_record( Record( @@ -384,20 +408,21 @@ async def test_server_socket_doesnt_crash_on_garbage(autojump_clock): frag_len=10, frag=bytes(10), ) - ) + b"\x00", + ) + + b"\x00", ) ) async with dtls_echo_server() as (_, address): with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as sock: for bad_packet in [ - b"", - b"xyz", - client_hello_extended, - client_hello_short, - client_hello_really_short, - client_hello_corrupt_record_len, - client_hello_fragmented, - client_hello_trailing_data_in_record, + b"", + b"xyz", + client_hello_extended, + client_hello_short, + client_hello_really_short, + client_hello_corrupt_record_len, + client_hello_fragmented, + client_hello_trailing_data_in_record, ]: await sock.sendto(bad_packet, address) await trio.sleep(1) @@ -409,45 +434,197 @@ async def test_invalid_cookie_rejected(autojump_clock): from trio._dtls import decode_client_hello_untrusted, BadPacket - offset_to_corrupt = count() - def route_packet(packet): - try: - _, cookie, _ = decode_client_hello_untrusted(packet.payload) - except BadPacket: - pass - else: - if len(cookie) != 0: - # this is a challenge response packet - # let's corrupt the next offset so the handshake should fail - payload = bytearray(packet.payload) - offset = next(offset_to_corrupt) - if offset >= len(payload): - # We've tried all offsets - # clamp offset to the end of the payload, and tell the client to stop - # trying to connect - offset = len(payload) - 1 - cscope.cancel() - payload[offset] ^= 0x01 - packet = attr.evolve(packet, payload=payload) + with trio.CancelScope() as cscope: + + offset_to_corrupt = count() + + def route_packet(packet): + try: + _, cookie, _ = decode_client_hello_untrusted(packet.payload) + except BadPacket: + pass + else: + if len(cookie) != 0: + # this is a challenge response packet + # let's corrupt the next offset so the handshake should fail + payload = bytearray(packet.payload) + offset = next(offset_to_corrupt) + if offset >= len(payload): + # We've tried all offsets. Clamp offset to the end of the + # payload, and terminate the test. + offset = len(payload) - 1 + cscope.cancel() + payload[offset] ^= 0x01 + packet = attr.evolve(packet, payload=payload) + + fn.deliver_packet(packet) + + fn.route_packet = route_packet + + async with dtls_echo_server() as (_, address): + while True: + with dtls() as client: + await client.connect(address, client_ctx) + assert cscope.cancelled_caught + + +async def test_client_cancels_handshake_and_starts_new_one(autojump_clock): + # if a client disappears during the handshake, and then starts a new handshake from + # scratch, then the first handler's channel should fail, and a new handler get + # started + fn = FakeNet() + fn.enable() + + with dtls() as server, dtls() as client: + await server.socket.bind(("127.0.0.1", 0)) + async with trio.open_nursery() as nursery: + first_time = True + + async def handler(channel): + nonlocal first_time + if first_time: + first_time = False + print("handler: first time, cancelling connect") + connect_cscope.cancel() + await trio.sleep(0.5) + print("handler: handshake should fail now") + with pytest.raises(trio.BrokenResourceError): + await channel.do_handshake() + else: + print("handler: not first time, sending hello") + await channel.send(b"hello") + + await nursery.start(server.serve, server_ctx, handler) + + print("client: starting first connect") + with trio.CancelScope() as connect_cscope: + await client.connect(server.socket.getsockname(), client_ctx) + assert connect_cscope.cancelled_caught + + print("client: starting second connect") + channel = await client.connect(server.socket.getsockname(), client_ctx) + assert await channel.receive() == b"hello" + + nursery.cancel_scope.cancel() + + +async def test_swap_client_server(): + with dtls() as a, dtls() as b: + await a.socket.bind(("127.0.0.1", 0)) + await b.socket.bind(("127.0.0.1", 0)) + + async def echo_handler(channel): + async for packet in channel: + await channel.send(packet) + + async def crashing_echo_handler(channel): + with pytest.raises(trio.BrokenResourceError): + await echo_handler(channel) + + async with trio.open_nursery() as nursery: + await nursery.start(a.serve, server_ctx, crashing_echo_handler) + await nursery.start(b.serve, server_ctx, echo_handler) + + b_to_a = await b.connect(a.socket.getsockname(), client_ctx) + await b_to_a.send(b"b as client") + assert await b_to_a.receive() == b"b as client" + a_to_b = await a.connect(b.socket.getsockname(), client_ctx) + with pytest.raises(trio.BrokenResourceError): + await b_to_a.send(b"association broken") + await a_to_b.send(b"a as client") + assert await a_to_b.receive() == b"a as client" + + nursery.cancel_scope.cancel() + + +@slow +async def test_openssl_retransmit_doesnt_break_stuff(): + # can't use autojump_clock here, because the point of the test is to wait for + # openssl's built-in retransmit timer to expire, which is hard-coded to use + # wall-clock time. + fn = FakeNet() + fn.enable() + + blackholed = True + + def route_packet(packet): + if blackholed: + print("dropped packet", packet) + return + print("delivered packet", packet) + # packets.append( + # scapy.all.IP( + # src=packet.source.ip.compressed, dst=packet.destination.ip.compressed + # ) + # / scapy.all.UDP(sport=packet.source.port, dport=packet.destination.port) + # / packet.payload + # ) fn.deliver_packet(packet) fn.route_packet = route_packet + async with dtls_echo_server() as (server_endpoint, address): + with dtls() as client_endpoint: + async with trio.open_nursery() as nursery: + + async def connecter(): + client = await client_endpoint.connect( + address, client_ctx, initial_retransmit_timeout=1.5 + ) + await client.send(b"hi") + assert await client.receive() == b"hi" + + nursery.start_soon(connecter) + + # openssl's default timeout is 1 second, so this ensures that it thinks + # the timeout has expired + await trio.sleep(1.1) + # disable blackholing and send a garbage packet to wake up openssl so it + # notices the timeout has expired + blackholed = False + await server_endpoint.socket.sendto( + b"xxx", client_endpoint.socket.getsockname() + ) + # now the client task should finish connecting and exit cleanly + + # scapy.all.wrpcap("/tmp/trace.pcap", packets) + + +async def test_initial_retransmit_timeout(autojump_clock): + fn = FakeNet() + fn.enable() + + blackholed = True + + def route_packet(packet): + nonlocal blackholed + if blackholed: + blackholed = False + else: + fn.deliver_packet(packet) + + fn.route_packet = route_packet + async with dtls_echo_server() as (_, address): - with trio.CancelScope() as cscope: - while True: - with dtls() as client: - await client.connect(address, client_ctx) - assert cscope.cancelled_caught + for t in [1, 2, 4]: + with dtls() as client: + before = trio.current_time() + blackholed = True + await client.connect(address, client_ctx, initial_retransmit_timeout=t) + after = trio.current_time() + assert after - before == t + + +async def test_tiny_mtu(): + async with dtls_echo_server() as (server, address): + with dtls() as client: + pass + # socket closed at terrible times -# cancelling a client handshake and then starting a new one # garbage collecting DTLS object without closing it - # use fakenet, send a packet to the server, then immediately drop the dtls object and - # run gc before `sock.recvfrom()` can return -# openssl retransmit -# receive a piece of garbage from the correct source during a handshake (corrupted -# packet, someone being a jerk) -- though can't necessarily tolerate someone sending a -# fake HelloRetryRequest -# connect() that replaces an existing association (currently totally broken!) +# use fakenet, send a packet to the server, then immediately drop the dtls object and +# run gc before `sock.recvfrom()` can return + +# ...connect() probably shouldn't do the handshake. makes it impossible to set MTU! From f7250c336fcb2bef9147827630e48bab15327822 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 6 Sep 2021 14:28:15 -0700 Subject: [PATCH 0946/1498] Cleanup pass on names and cookie crypto --- trio/_dtls.py | 86 ++++++++++++++++++++++------------------- trio/tests/test_dtls.py | 47 ++++++++++++---------- 2 files changed, 73 insertions(+), 60 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index b9eb913482..c1b084519f 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -488,15 +488,19 @@ def encode_volley(self, messages, mtu): # probably pretty harmless? But it's easier to add the salt than to convince myself that # it's *completely* harmless, so, salt it is. -# XX maybe the cookie should also sign the *local* address, so you can't take a cookie -# from one socket and use it on another socket on the same trio? or just generate the -# key in each call to 'serve'. - COOKIE_REFRESH_INTERVAL = 30 # seconds -KEY = None -KEY_BYTES = 8 +KEY_BYTES = 32 COOKIE_HASH = "sha256" SALT_BYTES = 8 +# 32 bytes was the maximum cookie length in DTLS 1.0. DTLS 1.2 raised it to 255. I doubt +# there are any DTLS 1.0 implementations still in the wild, but really 32 bytes is +# plenty, and it also gets rid of a confusing warning in Wireshark output. +# +# We truncate the cookie to 32 bytes, of which 8 bytes is salt, so that leaves 24 bytes +# of truncated HMAC = 192 bit security, which is still massive overkill. (TCP uses 32 +# *bits* for this.) HMAC truncation is explicitly noted as safe in RFC 2104: +# https://datatracker.ietf.org/doc/html/rfc2104#section-5 +COOKIE_LENGTH = 32 def _current_cookie_tick(): @@ -513,12 +517,9 @@ def _signable(*fields): return b"".join(out) -def _make_cookie(salt, tick, address, client_hello_bits): +def _make_cookie(key, salt, tick, address, client_hello_bits): assert len(salt) == SALT_BYTES - - global KEY - if KEY is None: - KEY = os.urandom(KEY_BYTES) + assert len(key) == KEY_BYTES signable_data = _signable( salt, @@ -529,17 +530,17 @@ def _make_cookie(salt, tick, address, client_hello_bits): client_hello_bits, ) - return salt + hmac.digest(KEY, signable_data, COOKIE_HASH) + return (salt + hmac.digest(key, signable_data, COOKIE_HASH))[:COOKIE_LENGTH] -def valid_cookie(cookie, address, client_hello_bits): +def valid_cookie(key, cookie, address, client_hello_bits): if len(cookie) > SALT_BYTES: salt = cookie[:SALT_BYTES] tick = _current_cookie_tick() - cur_cookie = _make_cookie(salt, tick, address, client_hello_bits) - old_cookie = _make_cookie(salt, max(tick - 1, 0), address, client_hello_bits) + cur_cookie = _make_cookie(key, salt, tick, address, client_hello_bits) + old_cookie = _make_cookie(key, salt, max(tick - 1, 0), address, client_hello_bits) # I doubt using a short-circuiting 'or' here would leak any meaningful # information, but why risk it when '|' is just as easy. @@ -550,10 +551,10 @@ def valid_cookie(cookie, address, client_hello_bits): return False -def challenge_for(address, epoch_seqno, client_hello_bits): +def challenge_for(key, address, epoch_seqno, client_hello_bits): salt = os.urandom(SALT_BYTES) tick = _current_cookie_tick() - cookie = _make_cookie(salt, tick, address, client_hello_bits) + cookie = _make_cookie(key, salt, tick, address, client_hello_bits) # HelloVerifyRequest body is: # - 2 bytes version @@ -608,8 +609,8 @@ def _read_loop(read_fn): return b"".join(chunks) -async def handle_client_hello_untrusted(dtls, address, packet): - if dtls._listening_context is None: +async def handle_client_hello_untrusted(endpoint, address, packet): + if endpoint._listening_context is None: return try: @@ -617,16 +618,19 @@ async def handle_client_hello_untrusted(dtls, address, packet): except BadPacket: return - if not valid_cookie(cookie, address, bits): - challenge_packet = challenge_for(address, epoch_seqno, bits) + if endpoint._listening_key is None: + endpoint._listening_key = os.urandom(KEY_BYTES) + + if not valid_cookie(endpoint._listening_key, cookie, address, bits): + challenge_packet = challenge_for(endpoint._listening_key, address, epoch_seqno, bits) try: - async with dtls._send_lock: - await dtls.socket.sendto(challenge_packet, address) + async with endpoint._send_lock: + await endpoint.socket.sendto(challenge_packet, address) except (OSError, trio.ClosedResourceError): pass else: # We got a real, valid ClientHello! - stream = DTLSChannel._create(dtls, address, dtls._listening_context) + stream = DTLSChannel._create(endpoint, address, endpoint._listening_context) # Our HelloRetryRequest had some sequence number. We need our future sequence # numbers to be larger than it, so our peer knows that our future records aren't # stale/duplicates. But, we don't know what this sequence number was. What we do @@ -645,7 +649,7 @@ async def handle_client_hello_untrusted(dtls, address, packet): # after all. return # Check if we have an existing association - old_stream = dtls._streams.get(address) + old_stream = endpoint._streams.get(address) if old_stream is not None: if old_stream._client_hello == (cookie, bits): # ...This was just a duplicate of the last ClientHello, so never mind. @@ -654,14 +658,14 @@ async def handle_client_hello_untrusted(dtls, address, packet): # Ok, this *really is* a new handshake; the old stream should go away. old_stream._set_replaced() stream._client_hello = (cookie, bits) - dtls._streams[address] = stream - dtls._incoming_connections_q.s.send_nowait(stream) + endpoint._streams[address] = stream + endpoint._incoming_connections_q.s.send_nowait(stream) -async def dtls_receive_loop(dtls): - sock = dtls.socket - dtls_ref = weakref.ref(dtls) - del dtls +async def dtls_receive_loop(endpoint): + sock = endpoint.socket + endpoint_ref = weakref.ref(endpoint) + del endpoint while True: try: packet, address = await sock.recvfrom(MAX_UDP_PACKET_SIZE) @@ -678,14 +682,14 @@ async def dtls_receive_loop(dtls): # https://bobobobo.wordpress.com/2009/05/17/udp-an-existing-connection-was-forcibly-closed-by-the-remote-host/ # We'll assume that whatever it is, it's a transient problem. continue - dtls = dtls_ref() + endpoint = endpoint_ref() try: - if dtls is None: + if endpoint is None: return if is_client_hello_untrusted(packet): - await handle_client_hello_untrusted(dtls, address, packet) - elif address in dtls._streams: - stream = dtls._streams[address] + await handle_client_hello_untrusted(endpoint, address, packet) + elif address in endpoint._streams: + stream = endpoint._streams[address] if stream._did_handshake and part_of_handshake_untrusted(packet): # The peer just sent us more handshake messages, that aren't a # ClientHello, and we thought the handshake was done. Some of the @@ -707,7 +711,7 @@ async def dtls_receive_loop(dtls): # Drop packet pass finally: - del dtls + del endpoint @attr.frozen @@ -934,7 +938,7 @@ def __init__(self, socket, *, incoming_packets_buffer=10): global SSL from OpenSSL import SSL - self.socket = None # for __del__ + self.socket = None # for __del__, in case the next line raises if socket.type != trio.socket.SOCK_DGRAM: raise ValueError("DTLS requires a SOCK_DGRAM socket") @@ -948,6 +952,7 @@ def __init__(self, socket, *, incoming_packets_buffer=10): # {remote address: DTLSChannel} self._streams = weakref.WeakValueDictionary() self._listening_context = None + self._listening_key = None self._incoming_connections_q = _Queue(float("inf")) self._send_lock = trio.Lock() self._closed = False @@ -955,10 +960,11 @@ def __init__(self, socket, *, incoming_packets_buffer=10): trio.lowlevel.spawn_system_task(dtls_receive_loop, self) def __del__(self): - # Close the socket in Trio context (if our Trio context still exists), so that - # the background task gets notified about the closure and can exit. + # Do nothing if this object was never fully constructed if self.socket is None: return + # Close the socket in Trio context (if our Trio context still exists), so that + # the background task gets notified about the closure and can exit. try: self._token.run_sync_soon(self.socket.close) except RuntimeError: diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index bd1842fb25..7c8a623783 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -169,14 +169,14 @@ async def test_implicit_handshake(): server_dtls = DTLSEndpoint(server_sock) -def dtls(**kwargs): +def endpoint(**kwargs): sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) return DTLSEndpoint(sock, **kwargs) @asynccontextmanager async def dtls_echo_server(*, autocancel=True): - with dtls() as server: + with endpoint() as server: await server.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as nursery: @@ -200,7 +200,7 @@ async def echo_handler(dtls_channel): async def test_implicit_handshake(): async with dtls_echo_server() as (_, address): - with dtls() as client_endpoint: + with endpoint() as client_endpoint: client = await client_endpoint.connect(address, client_ctx) # Implicit handshake @@ -209,7 +209,7 @@ async def test_implicit_handshake(): async def test_full_duplex(): - with dtls() as server_endpoint, dtls() as client_endpoint: + with endpoint() as server_endpoint, endpoint() as client_endpoint: await server_endpoint.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as server_nursery: @@ -232,7 +232,7 @@ async def handler(channel): async def test_channel_closing(): async with dtls_echo_server() as (_, address): - with dtls() as client_endpoint: + with endpoint() as client_endpoint: client = await client_endpoint.connect(address, client_ctx) client.close() @@ -257,7 +257,7 @@ async def test_serve_exits_cleanly_on_close(): async def test_client_multiplex(): async with dtls_echo_server() as (_, address1), dtls_echo_server() as (_, address2): - with dtls() as client_endpoint: + with endpoint() as client_endpoint: client1 = await client_endpoint.connect(address1, client_ctx) client2 = await client_endpoint.connect(address2, client_ctx) @@ -294,23 +294,23 @@ async def test_double_serve(): async def null_handler(_): # pragma: no cover pass - with dtls() as endpoint: + with endpoint() as server_endpoint: async with trio.open_nursery() as nursery: - await nursery.start(endpoint.serve, server_ctx, null_handler) + await nursery.start(server_endpoint.serve, server_ctx, null_handler) with pytest.raises(trio.BusyResourceError): - await nursery.start(endpoint.serve, server_ctx, null_handler) + await nursery.start(server_endpoint.serve, server_ctx, null_handler) nursery.cancel_scope.cancel() async with trio.open_nursery() as nursery: - await nursery.start(endpoint.serve, server_ctx, null_handler) + await nursery.start(server_endpoint.serve, server_ctx, null_handler) nursery.cancel_scope.cancel() async def test_connect_to_non_server(autojump_clock): fn = FakeNet() fn.enable() - with dtls() as client1, dtls() as client2: + with endpoint() as client1, endpoint() as client2: await client1.socket.bind(("127.0.0.1", 0)) # This should just time out with trio.move_on_after(100) as cscope: @@ -323,7 +323,7 @@ async def test_incoming_buffer_overflow(autojump_clock): fn.enable() for buffer_size in [10, 20]: async with dtls_echo_server() as (_, address): - with dtls(incoming_packets_buffer=buffer_size) as client_endpoint: + with endpoint(incoming_packets_buffer=buffer_size) as client_endpoint: assert client_endpoint.incoming_packets_buffer == buffer_size client = await client_endpoint.connect(address, client_ctx) for i in range(buffer_size + 15): @@ -463,7 +463,7 @@ def route_packet(packet): async with dtls_echo_server() as (_, address): while True: - with dtls() as client: + with endpoint() as client: await client.connect(address, client_ctx) assert cscope.cancelled_caught @@ -475,7 +475,7 @@ async def test_client_cancels_handshake_and_starts_new_one(autojump_clock): fn = FakeNet() fn.enable() - with dtls() as server, dtls() as client: + with endpoint() as server, endpoint() as client: await server.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as nursery: first_time = True @@ -509,7 +509,7 @@ async def handler(channel): async def test_swap_client_server(): - with dtls() as a, dtls() as b: + with endpoint() as a, endpoint() as b: await a.socket.bind(("127.0.0.1", 0)) await b.socket.bind(("127.0.0.1", 0)) @@ -565,7 +565,7 @@ def route_packet(packet): fn.route_packet = route_packet async with dtls_echo_server() as (server_endpoint, address): - with dtls() as client_endpoint: + with endpoint() as client_endpoint: async with trio.open_nursery() as nursery: async def connecter(): @@ -608,7 +608,7 @@ def route_packet(packet): async with dtls_echo_server() as (_, address): for t in [1, 2, 4]: - with dtls() as client: + with endpoint() as client: before = trio.current_time() blackholed = True await client.connect(address, client_ctx, initial_retransmit_timeout=t) @@ -618,13 +618,20 @@ def route_packet(packet): async def test_tiny_mtu(): async with dtls_echo_server() as (server, address): - with dtls() as client: + with endpoint() as client: pass # socket closed at terrible times + # garbage collecting DTLS object without closing it -# use fakenet, send a packet to the server, then immediately drop the dtls object and -# run gc before `sock.recvfrom()` can return +# (use fakenet, send a packet to the server, then immediately drop the dtls object and +# run gc before `sock.recvfrom()` can return) # ...connect() probably shouldn't do the handshake. makes it impossible to set MTU! + +# handshake with fakenet enforcing the minimum mtu (both ipv4 and ipv6) + +# maybe handshake failure should only set _mtu, not the openssl-level mtu? +# (and rename _mtu to "_effective_handshake_mtu" or something to be clear about its +# purpose) From ca2c652ae43feeb88e54ac25655f079282f5d5c4 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 6 Sep 2021 20:55:18 -0700 Subject: [PATCH 0947/1498] Take handshake out of connect() and make it sync --- trio/_dtls.py | 24 +++-- trio/tests/test_dtls.py | 230 +++++++++++++++++++++++----------------- 2 files changed, 147 insertions(+), 107 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index c1b084519f..3c22b99c90 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -540,7 +540,9 @@ def valid_cookie(key, cookie, address, client_hello_bits): tick = _current_cookie_tick() cur_cookie = _make_cookie(key, salt, tick, address, client_hello_bits) - old_cookie = _make_cookie(key, salt, max(tick - 1, 0), address, client_hello_bits) + old_cookie = _make_cookie( + key, salt, max(tick - 1, 0), address, client_hello_bits + ) # I doubt using a short-circuiting 'or' here would leak any meaningful # information, but why risk it when '|' is just as easy. @@ -622,7 +624,9 @@ async def handle_client_hello_untrusted(endpoint, address, packet): endpoint._listening_key = os.urandom(KEY_BYTES) if not valid_cookie(endpoint._listening_key, cookie, address, bits): - challenge_packet = challenge_for(endpoint._listening_key, address, epoch_seqno, bits) + challenge_packet = challenge_for( + endpoint._listening_key, address, epoch_seqno, bits + ) try: async with endpoint._send_lock: await endpoint.socket.sendto(challenge_packet, address) @@ -1010,7 +1014,7 @@ async def handler_wrapper(stream): finally: self._listening_context = None - async def connect(self, address, ssl_context, *, initial_retransmit_timeout=1.0): + def connect(self, address, ssl_context): # it would be nice if we could detect when 'address' is our own endpoint (a # loopback connection), because that can't work # but I don't see how to do it reliably @@ -1021,11 +1025,11 @@ async def connect(self, address, ssl_context, *, initial_retransmit_timeout=1.0) if old_channel is not None: old_channel._set_replaced() self._streams[address] = channel - try: - await channel.do_handshake( - initial_retransmit_timeout=initial_retransmit_timeout - ) - except: - channel.close() - raise + # try: + # await channel.do_handshake( + # initial_retransmit_timeout=initial_retransmit_timeout + # ) + # except: + # channel.close() + # raise return channel diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 7c8a623783..4f93d3160a 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -5,12 +5,13 @@ import attr from contextlib import asynccontextmanager from itertools import count +import ipaddress import trustme from OpenSSL import SSL from trio.testing._fake_net import FakeNet -from .._core.tests.tutil import slow, can_bind_ipv6 +from .._core.tests.tutil import slow, binds_ipv6 ca = trustme.CA() server_cert = ca.issue_cert("example.com") @@ -22,51 +23,70 @@ ca.configure_trust(client_ctx) -families = [trio.socket.AF_INET] -if can_bind_ipv6: - families.append(trio.socket.AF_INET6) +parametrize_ipv6 = pytest.mark.parametrize( + "ipv6", [False, pytest.param(True, marks=binds_ipv6)], ids=["ipv4", "ipv6"] +) -@pytest.mark.parametrize("family", families) -async def test_smoke(family): - if family == trio.socket.AF_INET: - localhost = "127.0.0.1" +def endpoint(**kwargs): + ipv6 = kwargs.pop("ipv6", False) + if ipv6: + family = trio.socket.AF_INET6 else: - localhost = "::1" - server_sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) - with server_sock: - await server_sock.bind((localhost, 0)) - server_dtls = DTLSEndpoint(server_sock) - - async with trio.open_nursery() as nursery: + family = trio.socket.AF_INET + sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM, family=family) + return DTLSEndpoint(sock, **kwargs) - async def handle_client(dtls_channel): - await dtls_channel.do_handshake() - assert await dtls_channel.receive() == b"hello" - await dtls_channel.send(b"goodbye") - await nursery.start(server_dtls.serve, server_ctx, handle_client) +@asynccontextmanager +async def dtls_echo_server(*, autocancel=True, mtu=None, ipv6=False): + with endpoint(ipv6=ipv6) as server: + if ipv6: + localhost = "::1" + else: + localhost = "127.0.0.1" + await server.socket.bind((localhost, 0)) + async with trio.open_nursery() as nursery: - with trio.socket.socket( - type=trio.socket.SOCK_DGRAM, family=family - ) as client_sock: - client_dtls = DTLSEndpoint(client_sock) - client = await client_dtls.connect( - server_sock.getsockname(), client_ctx + async def echo_handler(dtls_channel): + print( + f"echo handler started: " + f"server {dtls_channel.endpoint.socket.getsockname()} " + f"client {dtls_channel.peer_address}" ) - await client.send(b"hello") - assert await client.receive() == b"goodbye" + if mtu is not None: + dtls_channel.set_ciphertext_mtu(mtu) + async for packet in dtls_channel: + print(f"echoing {packet} -> {dtls_channel.peer_address}") + await dtls_channel.send(packet) - client.set_ciphertext_mtu(1234) - cleartext_mtu_1234 = client.get_cleartext_mtu() - client.set_ciphertext_mtu(4321) - assert client.get_cleartext_mtu() > cleartext_mtu_1234 - client.set_ciphertext_mtu(1234) - assert client.get_cleartext_mtu() == cleartext_mtu_1234 + await nursery.start(server.serve, server_ctx, echo_handler) + + yield server, server.socket.getsockname() + if autocancel: nursery.cancel_scope.cancel() +@parametrize_ipv6 +async def test_smoke(ipv6): + async with dtls_echo_server(ipv6=ipv6) as (server_endpoint, address): + with endpoint(ipv6=ipv6) as client_endpoint: + client_channel = client_endpoint.connect(address, client_ctx) + await client_channel.do_handshake() + await client_channel.send(b"hello") + assert await client_channel.receive() == b"hello" + await client_channel.send(b"goodbye") + assert await client_channel.receive() == b"goodbye" + + client_channel.set_ciphertext_mtu(1234) + cleartext_mtu_1234 = client_channel.get_cleartext_mtu() + client_channel.set_ciphertext_mtu(4321) + assert client_channel.get_cleartext_mtu() > cleartext_mtu_1234 + client_channel.set_ciphertext_mtu(1234) + assert client_channel.get_cleartext_mtu() == cleartext_mtu_1234 + + @slow async def test_handshake_over_terrible_network(autojump_clock): HANDSHAKES = 1000 @@ -145,9 +165,8 @@ async def handle_client(dtls_channel): print("#" * 80) with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: client_dtls = DTLSEndpoint(client_sock) - client = await client_dtls.connect( - server_sock.getsockname(), client_ctx - ) + client = client_dtls.connect(server_sock.getsockname(), client_ctx) + await client.do_handshake() while True: data = str(next_client_idx).encode() print(f"client sending plaintext: {data}") @@ -163,45 +182,10 @@ async def handle_client(dtls_channel): nursery.cancel_scope.cancel() -async def test_implicit_handshake(): - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as server_sock: - await server_sock.bind(("127.0.0.1", 0)) - server_dtls = DTLSEndpoint(server_sock) - - -def endpoint(**kwargs): - sock = trio.socket.socket(type=trio.socket.SOCK_DGRAM) - return DTLSEndpoint(sock, **kwargs) - - -@asynccontextmanager -async def dtls_echo_server(*, autocancel=True): - with endpoint() as server: - await server.socket.bind(("127.0.0.1", 0)) - async with trio.open_nursery() as nursery: - - async def echo_handler(dtls_channel): - print( - f"echo handler started: " - f"server {dtls_channel.endpoint.socket.getsockname()} " - f"client {dtls_channel.peer_address}" - ) - async for packet in dtls_channel: - print(f"echoing {packet} -> {dtls_channel.peer_address}") - await dtls_channel.send(packet) - - await nursery.start(server.serve, server_ctx, echo_handler) - - yield server, server.socket.getsockname() - - if autocancel: - nursery.cancel_scope.cancel() - - async def test_implicit_handshake(): async with dtls_echo_server() as (_, address): with endpoint() as client_endpoint: - client = await client_endpoint.connect(address, client_ctx) + client = client_endpoint.connect(address, client_ctx) # Implicit handshake await client.send(b"xyz") @@ -209,6 +193,8 @@ async def test_implicit_handshake(): async def test_full_duplex(): + # Tests simultaneous send/receive, and also multiple methods implicitly invoking + # do_handshake simultaneously. with endpoint() as server_endpoint, endpoint() as client_endpoint: await server_endpoint.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as server_nursery: @@ -220,7 +206,7 @@ async def handler(channel): await server_nursery.start(server_endpoint.serve, server_ctx, handler) - client = await client_endpoint.connect( + client = client_endpoint.connect( server_endpoint.socket.getsockname(), client_ctx ) async with trio.open_nursery() as nursery: @@ -233,7 +219,8 @@ async def handler(channel): async def test_channel_closing(): async with dtls_echo_server() as (_, address): with endpoint() as client_endpoint: - client = await client_endpoint.connect(address, client_ctx) + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake() client.close() with pytest.raises(trio.ClosedResourceError): @@ -258,8 +245,8 @@ async def test_serve_exits_cleanly_on_close(): async def test_client_multiplex(): async with dtls_echo_server() as (_, address1), dtls_echo_server() as (_, address2): with endpoint() as client_endpoint: - client1 = await client_endpoint.connect(address1, client_ctx) - client2 = await client_endpoint.connect(address2, client_ctx) + client1 = client_endpoint.connect(address1, client_ctx) + client2 = client_endpoint.connect(address2, client_ctx) await client1.send(b"abc") await client2.send(b"xyz") @@ -273,7 +260,7 @@ async def test_client_multiplex(): with pytest.raises(trio.ClosedResourceError): await client2.receive() with pytest.raises(trio.ClosedResourceError): - await client_endpoint.connect(address1, client_ctx) + client_endpoint.connect(address1, client_ctx) async with trio.open_nursery() as nursery: with pytest.raises(trio.ClosedResourceError): @@ -314,7 +301,8 @@ async def test_connect_to_non_server(autojump_clock): await client1.socket.bind(("127.0.0.1", 0)) # This should just time out with trio.move_on_after(100) as cscope: - await client2.connect(client1.socket.getsockname(), client_ctx) + channel = client2.connect(client1.socket.getsockname(), client_ctx) + await channel.do_handshake() assert cscope.cancelled_caught @@ -325,7 +313,7 @@ async def test_incoming_buffer_overflow(autojump_clock): async with dtls_echo_server() as (_, address): with endpoint(incoming_packets_buffer=buffer_size) as client_endpoint: assert client_endpoint.incoming_packets_buffer == buffer_size - client = await client_endpoint.connect(address, client_ctx) + client = client_endpoint.connect(address, client_ctx) for i in range(buffer_size + 15): await client.send(str(i).encode()) await trio.sleep(1) @@ -464,7 +452,8 @@ def route_packet(packet): async with dtls_echo_server() as (_, address): while True: with endpoint() as client: - await client.connect(address, client_ctx) + channel = client.connect(address, client_ctx) + await channel.do_handshake() assert cscope.cancelled_caught @@ -498,11 +487,12 @@ async def handler(channel): print("client: starting first connect") with trio.CancelScope() as connect_cscope: - await client.connect(server.socket.getsockname(), client_ctx) + channel = client.connect(server.socket.getsockname(), client_ctx) + await channel.do_handshake() assert connect_cscope.cancelled_caught print("client: starting second connect") - channel = await client.connect(server.socket.getsockname(), client_ctx) + channel = client.connect(server.socket.getsockname(), client_ctx) assert await channel.receive() == b"hello" nursery.cancel_scope.cancel() @@ -525,11 +515,12 @@ async def crashing_echo_handler(channel): await nursery.start(a.serve, server_ctx, crashing_echo_handler) await nursery.start(b.serve, server_ctx, echo_handler) - b_to_a = await b.connect(a.socket.getsockname(), client_ctx) + b_to_a = b.connect(a.socket.getsockname(), client_ctx) await b_to_a.send(b"b as client") assert await b_to_a.receive() == b"b as client" - a_to_b = await a.connect(b.socket.getsockname(), client_ctx) + a_to_b = a.connect(b.socket.getsockname(), client_ctx) + await a_to_b.do_handshake() with pytest.raises(trio.BrokenResourceError): await b_to_a.send(b"association broken") await a_to_b.send(b"a as client") @@ -569,9 +560,8 @@ def route_packet(packet): async with trio.open_nursery() as nursery: async def connecter(): - client = await client_endpoint.connect( - address, client_ctx, initial_retransmit_timeout=1.5 - ) + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake(initial_retransmit_timeout=1.5) await client.send(b"hi") assert await client.receive() == b"hi" @@ -591,7 +581,7 @@ async def connecter(): # scapy.all.wrpcap("/tmp/trace.pcap", packets) -async def test_initial_retransmit_timeout(autojump_clock): +async def test_initial_retransmit_timeout_configuration(autojump_clock): fn = FakeNet() fn.enable() @@ -611,15 +601,65 @@ def route_packet(packet): with endpoint() as client: before = trio.current_time() blackholed = True - await client.connect(address, client_ctx, initial_retransmit_timeout=t) + channel = client.connect(address, client_ctx) + await channel.do_handshake(initial_retransmit_timeout=t) after = trio.current_time() assert after - before == t -async def test_tiny_mtu(): - async with dtls_echo_server() as (server, address): +async def test_setting_tiny_mtu(): + # ClientHello is ~240 bytes, and it can't be fragmented, so our mtu has to + # be larger than that. (300 is still smaller than any real network though.) + MTU = 300 + + fn = FakeNet() + fn.enable() + + def route_packet(packet): + print(f"delivering {packet}") + print(f"payload size: {len(packet.payload)}") + assert len(packet.payload) <= MTU + fn.deliver_packet(packet) + + fn.route_packet = route_packet + + async with dtls_echo_server(mtu=MTU) as (server, address): with endpoint() as client: - pass + channel = client.connect(address, client_ctx) + channel.set_ciphertext_mtu(MTU) + await channel.do_handshake() + await channel.send(b"hi") + assert await channel.receive() == b"hi" + + +@parametrize_ipv6 +async def test_tiny_network_mtu(ipv6, autojump_clock): + # Fake network that has the minimum allowable MTU for whatever protocol we're using. + fn = FakeNet() + fn.enable() + + if ipv6: + mtu = 1280 - 48 + else: + mtu = 576 - 28 + + def route_packet(packet): + if len(packet.payload) > mtu: + print(f"dropping {packet}") + else: + print(f"delivering {packet}") + fn.deliver_packet(packet) + + fn.route_packet = route_packet + + # See if we can successfully do a handshake -- some of the volleys will get dropped, + # and the retransmit logic should detect this and back off the MTU to something + # smaller until it succeeds. + async with dtls_echo_server(ipv6=ipv6) as (_, address): + with endpoint(ipv6=ipv6) as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + await client.send(b"xyz") + assert await client.receive() == b"xyz" # socket closed at terrible times @@ -628,10 +668,6 @@ async def test_tiny_mtu(): # (use fakenet, send a packet to the server, then immediately drop the dtls object and # run gc before `sock.recvfrom()` can return) -# ...connect() probably shouldn't do the handshake. makes it impossible to set MTU! - -# handshake with fakenet enforcing the minimum mtu (both ipv4 and ipv6) - # maybe handshake failure should only set _mtu, not the openssl-level mtu? # (and rename _mtu to "_effective_handshake_mtu" or something to be clear about its # purpose) From 05c3f88d21acdf90fc83eca110f3e45fcc0e9d84 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 6 Sep 2021 22:50:35 -0700 Subject: [PATCH 0948/1498] Cleanups and a few more tests --- trio/_dtls.py | 60 ++++++++--------- trio/tests/test_dtls.py | 139 ++++++++++++++++++++++++---------------- 2 files changed, 115 insertions(+), 84 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 3c22b99c90..2964a629ab 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -1,9 +1,5 @@ # https://datatracker.ietf.org/doc/html/rfc6347 -# XX: figure out what to do about the pyopenssl dependency -# Maybe the toplevel __init__.py should use __getattr__ trickery to load all -# the DTLS code lazily? - import struct import hmac import os @@ -12,6 +8,7 @@ from itertools import count import weakref import errno +import warnings import attr @@ -127,7 +124,7 @@ def is_client_hello_untrusted(packet): RECORD_HEADER = struct.Struct("!B2sQH") -hex_repr = attr.ib(repr=lambda data: data.hex()) +hex_repr = attr.ib(repr=lambda data: data.hex()) # pragma: no cover @attr.frozen @@ -143,7 +140,10 @@ def records_untrusted(packet): while i < len(packet): try: ct, version, epoch_seqno, payload_len = RECORD_HEADER.unpack_from(packet, i) - except struct.error as exc: + # Marked as no-cover because at time of writing, this code is unreachable + # (records_untrusted only gets called on packets that are either trusted or that + # have passed is_client_hello_untrusted, which filters out short packets) + except struct.error as exc: # pragma: no cover raise BadPacket("invalid record header") from exc i += RECORD_HEADER.size payload = packet[i : i + payload_len] @@ -179,7 +179,7 @@ class HandshakeFragment: msg_seq: int frag_offset: int frag_len: int - frag: bytes = attr.ib(repr=lambda f: f.hex()) + frag: bytes = hex_repr def decode_handshake_fragment_untrusted(payload): @@ -605,8 +605,6 @@ def _read_loop(read_fn): chunk = read_fn(2 ** 14) # max TLS record size except SSL.WantReadError: break - if not chunk: - break chunks.append(chunk) return b"".join(chunks) @@ -666,10 +664,11 @@ async def handle_client_hello_untrusted(endpoint, address, packet): endpoint._incoming_connections_q.s.send_nowait(stream) -async def dtls_receive_loop(endpoint): - sock = endpoint.socket - endpoint_ref = weakref.ref(endpoint) - del endpoint +async def dtls_receive_loop(endpoint_ref): + try: + sock = endpoint_ref().socket + except AttributeError: + return while True: try: packet, address = await sock.recvfrom(MAX_UDP_PACKET_SIZE) @@ -909,6 +908,8 @@ def read_volley(): async def send(self, data): if self._closed: raise trio.ClosedResourceError + if not data: + raise ValueError("openssl doesn't support sending empty DTLS packets") if not self._did_handshake: await self.do_handshake() self._check_replaced() @@ -921,18 +922,19 @@ async def send(self, data): async def receive(self): if not self._did_handshake: await self.do_handshake() + # If the packet isn't really valid, then openssl can decode it to the empty + # string (e.g. b/c it's a late-arriving handshake packet, or a duplicate copy of + # a data packet). Skip over these instead of returning them. while True: try: packet = await self._q.r.receive() except trio.EndOfChannel: assert self._replaced self._check_replaced() - # Don't return spurious empty packets because of stray handshake packets - # coming in late - if part_of_handshake_untrusted(packet): - continue self._ssl.bio_write(packet) - return _read_loop(self._ssl.read) + cleartext = _read_loop(self._ssl.read) + if cleartext: + return cleartext class DTLSEndpoint(metaclass=Final): @@ -961,7 +963,7 @@ def __init__(self, socket, *, incoming_packets_buffer=10): self._send_lock = trio.Lock() self._closed = False - trio.lowlevel.spawn_system_task(dtls_receive_loop, self) + trio.lowlevel.spawn_system_task(dtls_receive_loop, weakref.ref(self)) def __del__(self): # Do nothing if this object was never fully constructed @@ -969,10 +971,15 @@ def __del__(self): return # Close the socket in Trio context (if our Trio context still exists), so that # the background task gets notified about the closure and can exit. - try: - self._token.run_sync_soon(self.socket.close) - except RuntimeError: - pass + if not self._closed: + try: + self._token.run_sync_soon(self.close) + except RuntimeError: + pass + # Do this last, because it might raise an exception + warnings.warn( + f"unclosed DTLS endpoint {self!r}", ResourceWarning, source=self + ) def close(self): self._closed = True @@ -1025,11 +1032,4 @@ def connect(self, address, ssl_context): if old_channel is not None: old_channel._set_replaced() self._streams[address] = channel - # try: - # await channel.do_handshake( - # initial_retransmit_timeout=initial_retransmit_timeout - # ) - # except: - # channel.close() - # raise return channel diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 4f93d3160a..ad8fb82e7b 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -6,12 +6,13 @@ from contextlib import asynccontextmanager from itertools import count import ipaddress +import warnings import trustme from OpenSSL import SSL from trio.testing._fake_net import FakeNet -from .._core.tests.tutil import slow, binds_ipv6 +from .._core.tests.tutil import slow, binds_ipv6, gc_collect_harder ca = trustme.CA() server_cert = ca.issue_cert("example.com") @@ -56,9 +57,15 @@ async def echo_handler(dtls_channel): ) if mtu is not None: dtls_channel.set_ciphertext_mtu(mtu) - async for packet in dtls_channel: - print(f"echoing {packet} -> {dtls_channel.peer_address}") - await dtls_channel.send(packet) + print("server starting do_handshake") + await dtls_channel.do_handshake() + print("server finished do_handshake") + try: + async for packet in dtls_channel: + print(f"echoing {packet} -> {dtls_channel.peer_address}") + await dtls_channel.send(packet) + except trio.BrokenResourceError: + pass await nursery.start(server.serve, server_ctx, echo_handler) @@ -79,6 +86,9 @@ async def test_smoke(ipv6): await client_channel.send(b"goodbye") assert await client_channel.receive() == b"goodbye" + with pytest.raises(ValueError): + await client_channel.send(b"") + client_channel.set_ciphertext_mtu(1234) cleartext_mtu_1234 = client_channel.get_cleartext_mtu() client_channel.set_ciphertext_mtu(4321) @@ -93,7 +103,8 @@ async def test_handshake_over_terrible_network(autojump_clock): r = random.Random(0) fn = FakeNet() fn.enable() - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as server_sock: + + async with dtls_echo_server() as (_, address): async with trio.open_nursery() as nursery: async def route_packet(packet): @@ -127,60 +138,25 @@ def route_packet_wrapper(packet): fn.route_packet = route_packet_wrapper - await server_sock.bind(("1.1.1.1", 54321)) - server_dtls = DTLSEndpoint(server_sock) - - next_client_idx = 0 - next_client_msg_recvd = trio.Event() - - async def handle_client(dtls_channel): - print("handling new client") - try: - await dtls_channel.do_handshake() - while True: - data = await dtls_channel.receive() - print(f"server received plaintext: {data}") - if not data: - continue - assert int(data.decode()) == next_client_idx - next_client_msg_recvd.set() - break - except trio.BrokenResourceError: - # client might have timed out on handshake and started a new one - # so we'll let this task die and let the new task do the check - print("new handshake restarting") - pass - except: - print("server handler saw") - import traceback - - traceback.print_exc() - raise - - await nursery.start(server_dtls.serve, server_ctx, handle_client) - - for _ in range(HANDSHAKES): + for i in range(HANDSHAKES): print("#" * 80) print("#" * 80) print("#" * 80) - with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as client_sock: - client_dtls = DTLSEndpoint(client_sock) - client = client_dtls.connect(server_sock.getsockname(), client_ctx) + with endpoint() as client_endpoint: + client = client_endpoint.connect(address, client_ctx) + print("client starting do_handshake") await client.do_handshake() + print("client finished do_handshake") + msg = str(i).encode() + # Make multiple attempts to send data, because the network might + # drop it while True: - data = str(next_client_idx).encode() - print(f"client sending plaintext: {data}") - await client.send(data) with trio.move_on_after(10) as cscope: - await next_client_msg_recvd.wait() + await client.send(msg) + assert await client.receive() == msg if not cscope.cancelled_caught: break - next_client_idx += 1 - next_client_msg_recvd = trio.Event() - - nursery.cancel_scope.cancel() - async def test_implicit_handshake(): async with dtls_echo_server() as (_, address): @@ -662,11 +638,66 @@ def route_packet(packet): assert await client.receive() == b"xyz" -# socket closed at terrible times +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_system_task_cleaned_up_on_gc(): + before_tasks = trio.lowlevel.current_statistics().tasks_living + + e = endpoint() + # Give system task a chance to start up + await trio.testing.wait_all_tasks_blocked() + + during_tasks = trio.lowlevel.current_statistics().tasks_living -# garbage collecting DTLS object without closing it -# (use fakenet, send a packet to the server, then immediately drop the dtls object and -# run gc before `sock.recvfrom()` can return) + with pytest.warns(ResourceWarning): + del e + gc_collect_harder() + + await trio.testing.wait_all_tasks_blocked() + + after_tasks = trio.lowlevel.current_statistics().tasks_living + assert before_tasks < during_tasks + assert before_tasks == after_tasks + + +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_gc_before_system_task_starts(): + e = endpoint() + + with pytest.warns(ResourceWarning): + del e + gc_collect_harder() + + await trio.testing.wait_all_tasks_blocked() + + +async def test_already_closed_socket_doesnt_crash(): + with endpoint() as e: + # We close the socket before checkpointing, so the socket will already be closed + # when the system task starts up + e.socket.close() + # Now give it a chance to start up, and hopefully not crash + await trio.testing.wait_all_tasks_blocked() + + +async def test_socket_closed_while_processing_clienthello(autojump_clock): + fn = FakeNet() + fn.enable() + + # Check what happens if the socket is discovered to be closed when sending a + # HelloVerifyRequest, since that has its own sending logic + async with dtls_echo_server() as (server, address): + def route_packet(packet): + fn.deliver_packet(packet) + server.socket.close() + + fn.route_packet = route_packet + + with endpoint() as client_endpoint: + with trio.move_on_after(10): + client = client_endpoint.connect(address, client_ctx) + await client.do_handshake() + +# socket closed at terrible times # maybe handshake failure should only set _mtu, not the openssl-level mtu? # (and rename _mtu to "_effective_handshake_mtu" or something to be clear about its From 2fb2d0df2b612c101db5f35bdfe20c1164919a52 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Mon, 6 Sep 2021 23:12:34 -0700 Subject: [PATCH 0949/1498] Don't let handshake implicitly overwrite user-specified mtu --- trio/_dtls.py | 14 ++++++++------ trio/tests/test_dtls.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 2964a629ab..3cadbd5a1b 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -736,7 +736,7 @@ def __init__(self, endpoint, peer_address, ctx): # to just performing a new handshake. ctx.set_options(SSL.OP_NO_QUERY_MTU | SSL.OP_NO_RENEGOTIATION) self._ssl = SSL.Connection(ctx) - self._mtu = None + self._handshake_mtu = None # This calls self._ssl.set_ciphertext_mtu, which is important, because if you # don't call it then openssl doesn't work. self.set_ciphertext_mtu(best_guess_mtu(self.endpoint.socket)) @@ -762,7 +762,7 @@ def _check_replaced(self): ) def set_ciphertext_mtu(self, new_mtu): - self._mtu = new_mtu + self._handshake_mtu = new_mtu self._ssl.set_ciphertext_mtu(new_mtu) def get_cleartext_mtu(self): @@ -796,7 +796,9 @@ async def aclose(self): await trio.lowlevel.checkpoint() async def _send_volley(self, volley_messages): - packets = self._record_encoder.encode_volley(volley_messages, self._mtu) + packets = self._record_encoder.encode_volley( + volley_messages, self._handshake_mtu + ) for packet in packets: async with self.endpoint._send_lock: await self.endpoint.socket.sendto(packet, self.peer_address) @@ -901,8 +903,8 @@ def read_volley(): # We tried sending this twice and they both failed. Maybe our # PMTU estimate is wrong? Let's try dropping it to the minimum # and hope that helps. - self.set_ciphertext_mtu( - min(self._mtu, worst_case_mtu(self.endpoint.socket)) + self._handshake_mtu = min( + self._handshake_mtu, worst_case_mtu(self.endpoint.socket) ) async def send(self, data): @@ -947,8 +949,8 @@ def __init__(self, socket, *, incoming_packets_buffer=10): self.socket = None # for __del__, in case the next line raises if socket.type != trio.socket.SOCK_DGRAM: raise ValueError("DTLS requires a SOCK_DGRAM socket") - self.socket = socket + self.incoming_packets_buffer = incoming_packets_buffer self._token = trio.lowlevel.current_trio_token() # We don't need to track handshaking vs non-handshake connections diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index ad8fb82e7b..dce1c92954 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -583,7 +583,7 @@ def route_packet(packet): assert after - before == t -async def test_setting_tiny_mtu(): +async def test_explicit_tiny_mtu_is_respected(): # ClientHello is ~240 bytes, and it can't be fragmented, so our mtu has to # be larger than that. (300 is still smaller than any real network though.) MTU = 300 @@ -609,7 +609,7 @@ def route_packet(packet): @parametrize_ipv6 -async def test_tiny_network_mtu(ipv6, autojump_clock): +async def test_handshake_handles_minimum_network_mtu(ipv6, autojump_clock): # Fake network that has the minimum allowable MTU for whatever protocol we're using. fn = FakeNet() fn.enable() @@ -634,8 +634,13 @@ def route_packet(packet): async with dtls_echo_server(ipv6=ipv6) as (_, address): with endpoint(ipv6=ipv6) as client_endpoint: client = client_endpoint.connect(address, client_ctx) + # the handshake mtu backoff shouldn't affect the return value from + # get_cleartext_mtu, b/c that's under the user's control via + # set_ciphertext_mtu + client.set_ciphertext_mtu(9999) await client.send(b"xyz") assert await client.receive() == b"xyz" + assert client.get_cleartext_mtu() > 9000 # as vegeta said @pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") @@ -686,6 +691,7 @@ async def test_socket_closed_while_processing_clienthello(autojump_clock): # Check what happens if the socket is discovered to be closed when sending a # HelloVerifyRequest, since that has its own sending logic async with dtls_echo_server() as (server, address): + def route_packet(packet): fn.deliver_packet(packet) server.socket.close() @@ -696,9 +702,3 @@ def route_packet(packet): with trio.move_on_after(10): client = client_endpoint.connect(address, client_ctx) await client.do_handshake() - -# socket closed at terrible times - -# maybe handshake failure should only set _mtu, not the openssl-level mtu? -# (and rename _mtu to "_effective_handshake_mtu" or something to be clear about its -# purpose) From 78ecb675f2c9ead68e495db22e7eef7805375380 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 7 Sep 2021 19:21:21 -0700 Subject: [PATCH 0950/1498] More cleanup and test coverage --- trio/_dtls.py | 100 +++++++++++++++++--------------- trio/tests/test_dtls.py | 125 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 176 insertions(+), 49 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 3cadbd5a1b..3c990e1af2 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -230,7 +230,10 @@ def decode_client_hello_untrusted(packet): try: # ClientHello has to be the first record in the packet record = next(records_untrusted(packet)) - if record.content_type != ContentType.handshake: + # no-cover because at time of writing, this is unreachable: + # decode_client_hello_untrusted is only called on packets that have passed + # is_client_hello_untrusted, which confirms the content type. + if record.content_type != ContentType.handshake: # pragma: no cover raise BadPacket("not a handshake record") fragment = decode_handshake_fragment_untrusted(record.payload) if fragment.msg_type != HandshakeType.client_hello: @@ -669,52 +672,58 @@ async def dtls_receive_loop(endpoint_ref): sock = endpoint_ref().socket except AttributeError: return - while True: - try: - packet, address = await sock.recvfrom(MAX_UDP_PACKET_SIZE) - except trio.ClosedResourceError: - return - except OSError as exc: - if exc.errno in (errno.EBADF, errno.ENOTSOCK): - # Socket was closed - return - else: - # Some weird error, e.g. apparently some versions of Windows can do - # ECONNRESET here to report that some previous UDP packet got an ICMP - # Port Unreachable: - # https://bobobobo.wordpress.com/2009/05/17/udp-an-existing-connection-was-forcibly-closed-by-the-remote-host/ - # We'll assume that whatever it is, it's a transient problem. - continue - endpoint = endpoint_ref() - try: - if endpoint is None: - return - if is_client_hello_untrusted(packet): - await handle_client_hello_untrusted(endpoint, address, packet) - elif address in endpoint._streams: - stream = endpoint._streams[address] - if stream._did_handshake and part_of_handshake_untrusted(packet): - # The peer just sent us more handshake messages, that aren't a - # ClientHello, and we thought the handshake was done. Some of the - # packets that we sent to finish the handshake must have gotten - # lost. So re-send them. We do this directly here instead of just - # putting it into the queue and letting the receiver do it, because - # there's no guarantee that anyone is reading from the queue, - # because we think the handshake is done! - try: + try: + while True: + try: + packet, address = await sock.recvfrom(MAX_UDP_PACKET_SIZE) + except OSError as exc: + if exc.errno == errno.ECONNRESET: + # Windows only: "On a UDP-datagram socket [ECONNRESET] + # indicates a previous send operation resulted in an ICMP Port + # Unreachable message" -- https://docs.microsoft.com/en-us/windows/win32/api/winsock/nf-winsock-recvfrom + # + # This is totally useless -- there's nothing we can do with this + # information. So we just ignore it and retry the recv. + continue + else: + raise + endpoint = endpoint_ref() + try: + if endpoint is None: + return + if is_client_hello_untrusted(packet): + await handle_client_hello_untrusted(endpoint, address, packet) + elif address in endpoint._streams: + stream = endpoint._streams[address] + if stream._did_handshake and part_of_handshake_untrusted(packet): + # The peer just sent us more handshake messages, that aren't a + # ClientHello, and we thought the handshake was done. Some of + # the packets that we sent to finish the handshake must have + # gotten lost. So re-send them. We do this directly here instead + # of just putting it into the queue and letting the receiver do + # it, because there's no guarantee that anyone is reading from + # the queue, because we think the handshake is done! await stream._resend_final_volley() - except trio.ClosedResourceError: - return + else: + try: + stream._q.s.send_nowait(packet) + except trio.WouldBlock: + stream._packets_dropped_in_trio += 1 else: - try: - stream._q.s.send_nowait(packet) - except trio.WouldBlock: - stream._packets_dropped_in_trio += 1 - else: - # Drop packet - pass - finally: - del endpoint + # Drop packet + pass + finally: + del endpoint + except trio.ClosedResourceError: + # socket was closed + return + except OSError as exc: + if exc.errno in (errno.EBADF, errno.ENOTSOCK): + # socket was closed + return + else: # pragma: no cover + # ??? shouldn't happen + raise @attr.frozen @@ -847,6 +856,7 @@ def read_volley(): while True: # -- at this point, we need to either send or re-send a volley -- assert volley_messages + self._check_replaced() await self._send_volley(volley_messages) # -- then this is where we wait for a reply -- with trio.move_on_after(timeout) as cscope: diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index dce1c92954..2d09e469c8 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -64,8 +64,8 @@ async def echo_handler(dtls_channel): async for packet in dtls_channel: print(f"echoing {packet} -> {dtls_channel.peer_address}") await dtls_channel.send(packet) - except trio.BrokenResourceError: - pass + except trio.BrokenResourceError: # pragma: no cover + print("echo handler channel broken") await nursery.start(server.serve, server_ctx, echo_handler) @@ -111,7 +111,7 @@ async def route_packet(packet): while True: op = r.choices( ["deliver", "drop", "dupe", "delay"], - weights=[0.7, 0.1, 0.1, 0.1], + weights=[0.6, 0.1, 0.1, 0.1], )[0] print(f"{packet.source} -> {packet.destination}: {op}") if op == "drop": @@ -120,6 +120,23 @@ async def route_packet(packet): fn.send_packet(packet) elif op == "delay": await trio.sleep(r.random() * 3) + # I wanted to test random packet corruption too, but it turns out + # openssl has a bug in the following scenario: + # - client sends ClientHello + # - server sends HelloVerifyRequest with cookie -- but cookie is + # invalid b/c either the ClientHello or HelloVerifyRequest was + # corrupted + # - client re-sends ClientHello with invalid cookie + # - server replies with new HelloVerifyRequest and correct cookie + # + # At this point, the client *should* switch to the new, valid + # cookie. But OpenSSL doesn't; it stubbornly insists on re-sending + # the original, invalid cookie over and over. + # + # elif op == "distort": + # payload = bytearray(packet.payload) + # payload[r.randrange(len(payload))] ^= 1 << r.randrange(8) + # packet = attr.evolve(packet, payload=payload) else: assert op == "deliver" print( @@ -131,7 +148,7 @@ async def route_packet(packet): def route_packet_wrapper(packet): try: nursery.start_soon(route_packet, packet) - except RuntimeError: + except RuntimeError: # pragma: no cover # We're exiting the nursery, so any remaining packets can just get # dropped pass @@ -376,6 +393,25 @@ async def test_server_socket_doesnt_crash_on_garbage(autojump_clock): + b"\x00", ) ) + + handshake_empty = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=b"", + ) + ) + + client_hello_truncated_in_cookie = encode_record( + Record( + content_type=ContentType.handshake, + version=ProtocolVersion.DTLS10, + epoch_seqno=0, + payload=bytes(2 + 32 + 1) + b"\xff", + ) + ) + async with dtls_echo_server() as (_, address): with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as sock: for bad_packet in [ @@ -387,6 +423,8 @@ async def test_server_socket_doesnt_crash_on_garbage(autojump_clock): client_hello_corrupt_record_len, client_hello_fragmented, client_hello_trailing_data_in_record, + handshake_empty, + client_hello_truncated_in_cookie, ]: await sock.sendto(bad_packet, address) await trio.sleep(1) @@ -471,6 +509,8 @@ async def handler(channel): channel = client.connect(server.socket.getsockname(), client_ctx) assert await channel.receive() == b"hello" + # Give handlers a chance to finish + await trio.sleep(10) nursery.cancel_scope.cancel() @@ -675,6 +715,17 @@ async def test_gc_before_system_task_starts(): await trio.testing.wait_all_tasks_blocked() +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +def test_gc_after_trio_exits(): + async def main(): + return endpoint() + + e = trio.run(main) + with pytest.warns(ResourceWarning): + del e + gc_collect_harder() + + async def test_already_closed_socket_doesnt_crash(): with endpoint() as e: # We close the socket before checkpointing, so the socket will already be closed @@ -702,3 +753,69 @@ def route_packet(packet): with trio.move_on_after(10): client = client_endpoint.connect(address, client_ctx) await client.do_handshake() + + +async def test_association_replaced_while_handshake_running(autojump_clock): + fn = FakeNet() + fn.enable() + + blackholed = True + + def route_packet(packet): + if blackholed: + return + fn.deliver_packet(packet) + + fn.route_packet = route_packet + + async with dtls_echo_server() as (_, address): + with endpoint() as client_endpoint: + c1 = client_endpoint.connect(address, client_ctx) + async with trio.open_nursery() as nursery: + async def doomed_handshake(): + with pytest.raises(trio.BrokenResourceError): + await c1.do_handshake() + + nursery.start_soon(doomed_handshake) + + await trio.sleep(10) + + c2 = client_endpoint.connect(address, client_ctx) + + +async def test_association_replaced_before_handshake_starts(): + fn = FakeNet() + fn.enable() + + # This test shouldn't send any packets + def route_packet(packet): # pragma: no cover + assert False + + fn.route_packet = route_packet + + async with dtls_echo_server() as (_, address): + with endpoint() as client_endpoint: + c1 = client_endpoint.connect(address, client_ctx) + c2 = client_endpoint.connect(address, client_ctx) + with pytest.raises(trio.BrokenResourceError): + await c1.do_handshake() + + +async def test_send_to_closed_local_port(): + # On Windows, sending a UDP packet to a closed local port can cause a weird + # ECONNRESET error later, inside the receive task. Make sure we're handling it + # properly. + async with dtls_echo_server() as (_, address): + with endpoint() as client_endpoint: + async with trio.open_nursery() as nursery: + for i in range(1, 10): + channel = client_endpoint.connect(("127.0.0.1", i), client_ctx) + nursery.start_soon(channel.do_handshake) + channel = client_endpoint.connect(address, client_ctx) + await channel.send(b"xxx") + assert await channel.receive() == b"xxx" + nursery.cancel_scope.cancel() + +# can we work around the openssl bug with invalid cookies by rebooting the connection +# when we see a second HelloVerifyRequest? (and then enable packet corruption in the +# torture test?) From 00e5cafa3999e97c73f9253e0cbcc0322d07dc57 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 7 Sep 2021 20:57:52 -0700 Subject: [PATCH 0951/1498] comment --- trio/_dtls.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/trio/_dtls.py b/trio/_dtls.py index 3c990e1af2..e531ced810 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -1,4 +1,10 @@ +# Implementation of DTLS 1.2, using pyopenssl # https://datatracker.ietf.org/doc/html/rfc6347 +# +# OpenSSL's APIs for DTLS are extremely awkward and limited, which forces us to jump +# through a *lot* of hoops and implement important chunks of the protocol ourselves. +# Hopefully they fix this before implementing DTLS 1.3, because it's a very different +# protocol, and it's probably impossible to pull tricks like we do here. import struct import hmac From 0a2b055b2862f82d2416fc88d9d5b48c3f386baa Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 7 Sep 2021 23:39:45 -0700 Subject: [PATCH 0952/1498] A few more cleanups + add docs --- docs/source/conf.py | 1 + docs/source/reference-io.rst | 46 ++++++++ trio/_dtls.py | 223 ++++++++++++++++++++++++++++++++--- trio/tests/test_dtls.py | 49 +++++--- 4 files changed, 289 insertions(+), 30 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 6045ffd828..52872d0cb4 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -87,6 +87,7 @@ def setup(app): intersphinx_mapping = { "python": ('https://docs.python.org/3', None), "outcome": ('https://outcome.readthedocs.io/en/latest/', None), + "pyopenssl": ('https://www.pyopenssl.org/en/stable/', None), } autodoc_member_order = "bysource" diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index b18983e272..a3291ef2ae 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -258,6 +258,52 @@ you call them before the handshake completes: .. autoexception:: NeedHandshakeError +Datagram TLS support +~~~~~~~~~~~~~~~~~~~~ + +Trio also has support for Datagram TLS (DTLS), which is like TLS but +for unreliable UDP connections. This can be useful for applications +where TCP's reliable in-order delivery is problematic, like +teleconferencing, latency-sensitive games, and VPNs. + +Currently, using DTLS with Trio requires PyOpenSSL. We hope to +eventually allow the use of the stdlib `ssl` module as well, but +unfortunately that's not yet possible. + +.. warning:: Note that PyOpenSSL is in many ways lower-level than the + `ssl` module – in particular, it currently **HAS NO BUILT-IN + MECHANISM TO VALIDATE CERTIFICATES**. We *strongly* recommend that + you use the `service-identity + `__ library to validate + hostnames and certificates. + +.. autoclass:: DTLSEndpoint + + .. automethod:: connect + + .. automethod:: serve + + .. automethod:: close + +.. autoclass:: DTLSChannel + :show-inheritance: + + .. automethod:: do_handshake + + .. automethod:: send + + .. automethod:: receive + + .. automethod:: close + + .. automethod:: aclose + + .. automethod:: set_ciphertext_mtu + + .. automethod:: get_cleartext_mtu + + .. automethod:: statistics + .. module:: trio.socket Low-level networking with :mod:`trio.socket` diff --git a/trio/_dtls.py b/trio/_dtls.py index e531ced810..8289821ad0 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -673,11 +673,7 @@ async def handle_client_hello_untrusted(endpoint, address, packet): endpoint._incoming_connections_q.s.send_nowait(stream) -async def dtls_receive_loop(endpoint_ref): - try: - sock = endpoint_ref().socket - except AttributeError: - return +async def dtls_receive_loop(endpoint_ref, sock): try: while True: try: @@ -738,6 +734,20 @@ class DTLSChannelStatistics: class DTLSChannel(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): + """A DTLS connection. + + This class has no public constructor – you get instances by calling + `DTLSEndpoint.serve` or `~DTLSEndpoint.connect`. + + .. attribute:: endpoint + + The `DTLSEndpoint` that this connection is using. + + .. attribute:: peer_address + + The IP/port of the remote peer that this connection is associated with. + + """ def __init__(self, endpoint, peer_address, ctx): self.endpoint = endpoint self.peer_address = peer_address @@ -761,9 +771,6 @@ def __init__(self, endpoint, peer_address, ctx): self._handshake_lock = trio.Lock() self._record_encoder = RecordEncoder() - def statistics(self) -> DTLSChannelStatistics: - return DTLSChannelStatistics(self._packets_dropped_in_trio) - def _set_replaced(self): self._replaced = True # Any packets we already received could maybe possibly still be processed, but @@ -776,13 +783,6 @@ def _check_replaced(self): "peer tore down this connection to start a new one" ) - def set_ciphertext_mtu(self, new_mtu): - self._handshake_mtu = new_mtu - self._ssl.set_ciphertext_mtu(new_mtu) - - def get_cleartext_mtu(self): - return self._ssl.get_cleartext_mtu() - # XX on systems where we can (maybe just Linux?) take advantage of the kernel's PMTU # estimate @@ -791,6 +791,17 @@ def get_cleartext_mtu(self): # to handle receiving it properly though, which might be easier if we send it... def close(self): + """Close this connection. + + `DTLSChannel`\s don't actually own any OS-level resources – the + socket is owned by the `DTLSEndpoint`, not the individual connections. So + you don't really *have* to call this. But it will interrupt any other tasks + calling `receive` with a `ClosedResourceError`, and cause future attempts to use + this connection to fail. + + You can also use this object as a synchronous or asynchronous context manager. + + """ if self._closed: return self._closed = True @@ -807,6 +818,12 @@ def __exit__(self, *args): self.close() async def aclose(self): + """Close this connection, but asynchronously. + + This is included to satisfy the `trio.abc.Channel` contract. It's + identical to `close`, but async. + + """ self.close() await trio.lowlevel.checkpoint() @@ -822,6 +839,33 @@ async def _resend_final_volley(self): await self._send_volley(self._final_volley) async def do_handshake(self, *, initial_retransmit_timeout=1.0): + """Perform the handshake. + + Calling this is optional – if you don't, then it will be automatically called + the first time you call `send` or `receive`. But calling it explicitly can be + useful in case you want to control the retransmit timeout, use a cancel scope to + place an overall timeout on the handshake, or catch errors from the handshake + specifically. + + It's safe to call this multiple times, or call it simultaneously from multiple + tasks – the first call will perform the handshake, and the rest will be no-ops. + + Args: + + initial_retransmit_timeout (float): Since UDP is an unreliable protocol, it's + possible that some of the packets we send during the handshake will get + lost. To handle this, DTLS uses a timer to automatically retransmit + handshake packets that don't receive a response. This lets you set the + timeout we use to detect packet loss. Ideally, it should be set to ~1.5 + times the round-trip time to your peer, but 1 second is a reasonable + default. There's `some useful guidance here + `__. + + This is the *initial* timeout, because if packets keep being lost then Trio + will automatically back off to longer values, to avoid overloading the + network. + + """ async with self._handshake_lock: if self._did_handshake: return @@ -924,6 +968,10 @@ def read_volley(): ) async def send(self, data): + """Send a packet of data, securely. + + """ + if self._closed: raise trio.ClosedResourceError if not data: @@ -938,6 +986,15 @@ async def send(self, data): ) async def receive(self): + """Fetch the next packet of data from this connection's peer, waiting if + necessary. + + This is safe to call from multiple tasks simultaneously, in case you have some + reason to do that. And more importantly, it's cancellation-safe, meaning that + cancelling a call to `receive` will never cause a packet to be lost or corrupt + the underlying connection. + + """ if not self._did_handshake: await self.do_handshake() # If the packet isn't really valid, then openssl can decode it to the empty @@ -954,8 +1011,92 @@ async def receive(self): if cleartext: return cleartext + def set_ciphertext_mtu(self, new_mtu): + """Tells Trio the `largest amount of data that can be sent in a single packet to + this peer `__. + + Trio doesn't actually enforce this limit – if you pass a huge packet to `send`, + then we'll dutifully encrypt it and attempt to send it. But calling this method + does have two useful effects: + + - If called before the handshake is performed, then Trio will automatically + fragment handshake messages to fit within the given MTU. It also might + fragment them even smaller, if it detects signs of packet loss, so setting + this should never be necessary to make a successful connection. But, the + packet loss detection only happens after multiple timeouts have expired, so if + you have reason to believe that a smaller MTU is required, then you can set + this to skip those timeouts and establish the connection more quickly. + + - It changes the value returned from `get_cleartext_mtu`. So if you have some + kind of estimate of the network-level MTU, then you can use this to figure out + how much overhead DTLS will need for hashes/padding/etc., and how much space + you have left for your application data. + + The MTU here is measuring the largest UDP *payload* you think can be sent, the + amount of encrypted data that can be handed to the operating system in a single + call to `send`. It should *not* include IP/UDP headers. Note that OS estimates + of the MTU often are link-layer MTUs, so you have to subtract off 28 bytes on + IPv4 and 48 bytes on IPv6 to get the ciphertext MTU. + + By default, Trio assumes an MTU of 1472 bytes on IPv4, and 1452 bytes on IPv6, + which correspond to the common Ethernet MTU of 1500 bytes after accounting for + IP/UDP overhead. + + """ + self._handshake_mtu = new_mtu + self._ssl.set_ciphertext_mtu(new_mtu) + + def get_cleartext_mtu(self): + """Returns the largest number of bytes that you can pass in a single call to + `send` while still fitting within the network-level MTU. + + See `set_ciphertext_mtu` for more details. + + """ + if not self._did_handshake: + raise trio.NeedHandshakeError + return self._ssl.get_cleartext_mtu() + + def statistics(self): + """Returns an object with statistics about this connection. + + Currently this has only one attribute: + + - ``incoming_packets_dropped_in_trio`` (``int``): Gives a count of the number of + incoming packets from this peer that Trio successfully received from the + network, but then got dropped because the internal channel buffer was full. If + this is non-zero, then you might want to call ``receive`` more often, or use a + larger ``incoming_packets_buffer``, or just not worry about it because your + UDP-based protocol should be able to handle the occasional lost packet, right? + + """ + return DTLSChannelStatistics(self._packets_dropped_in_trio) + class DTLSEndpoint(metaclass=Final): + """A DTLS endpoint. + + A single UDP socket can handle arbitrarily many DTLS connections simultaneously, + acting as a client or server as needed. A `DTLSEndpoint` object holds a UDP socket + and manages these connections, which are represented as `DTLSChannel` objects. + + Args: + socket: (trio.socket.SocketType): A ``SOCK_DGRAM`` socket. If you want to accept + incoming connections in server mode, then you should probably bind the socket to + some known port. + incoming_packets_buffer (int): Each `DTLSChannel` using this socket has its own + buffer that holds incoming packets until you call `~DTLSChannel.receive` to read + them. This lets you adjust the size of this buffer. `~DTLSChannel.statistics` + lets you check if the buffer has overflowed. + + .. attribute:: socket + incoming_packets_buffer + + Both constructor arguments are also exposed as attributes, in case you need to + access them later. + + """ + def __init__(self, socket, *, incoming_packets_buffer=10): # We do this lazily on first construction, so only people who actually use DTLS # have to install PyOpenSSL. @@ -981,7 +1122,7 @@ def __init__(self, socket, *, incoming_packets_buffer=10): self._send_lock = trio.Lock() self._closed = False - trio.lowlevel.spawn_system_task(dtls_receive_loop, weakref.ref(self)) + trio.lowlevel.spawn_system_task(dtls_receive_loop, weakref.ref(self), self.socket) def __del__(self): # Do nothing if this object was never fully constructed @@ -1000,6 +1141,11 @@ def __del__(self): ) def close(self): + """Close this socket, and all associated DTLS connections. + + This object can also be used as a context manager. + + """ self._closed = True self.socket.close() for stream in list(self._streams.values()): @@ -1019,6 +1165,34 @@ def _check_closed(self): async def serve( self, ssl_context, async_fn, *args, task_status=trio.TASK_STATUS_IGNORED ): + """Listen for incoming connections, and spawn a handler for each using an + internal nursery. + + Similar to `~trio.serve_tcp`, this function never returns until cancelled, or + the `DTLSEndpoint` is closed and all handlers have exited. + + Usage commonly looks like:: + + async def handler(dtls_channel): + ... + + async with trio.open_nursery() as nursery: + await nursery.start(dtls_endpoint.serve, ssl_context, handler) + # ... do other things here ... + + The ``dtls_channel`` passed into the handler function has already performed the + "cookie exchange" part of the DTLS handshake, so the peer address is + trustworthy. But the actual cryptographic handshake doesn't happen until you + start using it, giving you a chance for any last minute configuration, and the + option to catch and handle handshake errors. + + Args: + ssl_context (OpenSSL.SSL.Context): The PyOpenSSL context object to use for + incoming connections. + async_fn: The handler function that will be invoked for each incoming + connection. + + """ self._check_closed() if self._listening_context is not None: raise trio.BusyResourceError("another task is already listening") @@ -1040,6 +1214,23 @@ async def handler_wrapper(stream): self._listening_context = None def connect(self, address, ssl_context): + """Initiate an outgoing DTLS connection. + + Notice that this is a synchronous method. That's because it doesn't actually + initiate any I/O – it just sets up a `DTLSChannel` object. The actual handshake + doesn't occur until you start using the `DTLSChannel`. This gives you a chance + to do further configuration first, like setting MTU etc. + + Args: + address: The address to connect to. Usually a (host, port) tuple, like + ``("127.0.0.1", 12345)``. + ssl_context (OpenSSL.SSL.Context): The PyOpenSSL context object to use for + this connection. + + Returns: + DTLSChannel + + """ # it would be nice if we could detect when 'address' is our own endpoint (a # loopback connection), because that can't work # but I don't see how to do it reliably diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 2d09e469c8..c889d353aa 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -57,10 +57,10 @@ async def echo_handler(dtls_channel): ) if mtu is not None: dtls_channel.set_ciphertext_mtu(mtu) - print("server starting do_handshake") - await dtls_channel.do_handshake() - print("server finished do_handshake") try: + print("server starting do_handshake") + await dtls_channel.do_handshake() + print("server finished do_handshake") async for packet in dtls_channel: print(f"echoing {packet} -> {dtls_channel.peer_address}") await dtls_channel.send(packet) @@ -80,6 +80,9 @@ async def test_smoke(ipv6): async with dtls_echo_server(ipv6=ipv6) as (server_endpoint, address): with endpoint(ipv6=ipv6) as client_endpoint: client_channel = client_endpoint.connect(address, client_ctx) + with pytest.raises(trio.NeedHandshakeError): + client_channel.get_cleartext_mtu() + await client_channel.do_handshake() await client_channel.send(b"hello") assert await client_channel.receive() == b"hello" @@ -111,7 +114,7 @@ async def route_packet(packet): while True: op = r.choices( ["deliver", "drop", "dupe", "delay"], - weights=[0.6, 0.1, 0.1, 0.1], + weights=[0.7, 0.1, 0.1, 0.1], )[0] print(f"{packet.source} -> {packet.destination}: {op}") if op == "drop": @@ -122,6 +125,7 @@ async def route_packet(packet): await trio.sleep(r.random() * 3) # I wanted to test random packet corruption too, but it turns out # openssl has a bug in the following scenario: + # # - client sends ClientHello # - server sends HelloVerifyRequest with cookie -- but cookie is # invalid b/c either the ClientHello or HelloVerifyRequest was @@ -131,7 +135,12 @@ async def route_packet(packet): # # At this point, the client *should* switch to the new, valid # cookie. But OpenSSL doesn't; it stubbornly insists on re-sending - # the original, invalid cookie over and over. + # the original, invalid cookie over and over. In theory we could + # work around this by detecting cookie changes and starting over + # with a whole new SSL object, but (a) it doesn't seem worth it, (b) + # when I tried then I ran into another issue where OpenSSL got stuck + # in an infinite loop sending alerts over and over, which I didn't + # dig into because see (a). # # elif op == "distort": # payload = bytearray(packet.payload) @@ -715,6 +724,26 @@ async def test_gc_before_system_task_starts(): await trio.testing.wait_all_tasks_blocked() +@pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") +async def test_gc_as_packet_received(): + fn = FakeNet() + fn.enable() + + e = endpoint() + await e.socket.bind(("127.0.0.1", 0)) + + await trio.testing.wait_all_tasks_blocked() + + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: + await s.sendto(b"xxx", e.socket.getsockname()) + # At this point, the endpoint's receive loop has been marked runnable because it + # just received a packet; closing the endpoint socket won't interrupt that. But by + # the time it wakes up to process the packet, the endpoint will be gone. + with pytest.warns(ResourceWarning): + del e + gc_collect_harder() + + @pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") def test_gc_after_trio_exits(): async def main(): @@ -759,12 +788,8 @@ async def test_association_replaced_while_handshake_running(autojump_clock): fn = FakeNet() fn.enable() - blackholed = True - def route_packet(packet): - if blackholed: - return - fn.deliver_packet(packet) + pass fn.route_packet = route_packet @@ -815,7 +840,3 @@ async def test_send_to_closed_local_port(): await channel.send(b"xxx") assert await channel.receive() == b"xxx" nursery.cancel_scope.cancel() - -# can we work around the openssl bug with invalid cookies by rebooting the connection -# when we see a second HelloVerifyRequest? (and then enable packet corruption in the -# torture test?) From 419c962c21e9aba95d038e0430c4af0748f154ea Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 30 Sep 2021 08:05:26 -0700 Subject: [PATCH 0953/1498] Quote literal backslash in string --- trio/_dtls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 8289821ad0..58b24abaa8 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -793,7 +793,7 @@ def _check_replaced(self): def close(self): """Close this connection. - `DTLSChannel`\s don't actually own any OS-level resources – the + `DTLSChannel`\\s don't actually own any OS-level resources – the socket is owned by the `DTLSEndpoint`, not the individual connections. So you don't really *have* to call this. But it will interrupt any other tasks calling `receive` with a `ClosedResourceError`, and cause future attempts to use From 717e46f7a90c52873203a1ec133fe1a98506f885 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Thu, 30 Sep 2021 08:15:01 -0700 Subject: [PATCH 0954/1498] Defer starting the DTLS receive task until we actually need to receive This works around a problem on windows where recvfrom on an unbound socket errors out. --- trio/_dtls.py | 15 ++++++++++++++- trio/tests/test_dtls.py | 17 +++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 58b24abaa8..766a3268e1 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -909,6 +909,7 @@ def read_volley(): self._check_replaced() await self._send_volley(volley_messages) # -- then this is where we wait for a reply -- + self.endpoint._ensure_receive_loop() with trio.move_on_after(timeout) as cscope: async for packet in self._q.r: self._ssl.bio_write(packet) @@ -1121,8 +1122,15 @@ def __init__(self, socket, *, incoming_packets_buffer=10): self._incoming_connections_q = _Queue(float("inf")) self._send_lock = trio.Lock() self._closed = False + self._receive_loop_spawned = False - trio.lowlevel.spawn_system_task(dtls_receive_loop, weakref.ref(self), self.socket) + def _ensure_receive_loop(self): + # We have to spawn this lazily, because on Windows it will immediately error out + # if the socket isn't already bound -- which for clients might not happen until + # after we send our first packet. + if not self._receive_loop_spawned: + trio.lowlevel.spawn_system_task(dtls_receive_loop, weakref.ref(self), self.socket) + self._receive_loop_spawned = True def __del__(self): # Do nothing if this object was never fully constructed @@ -1196,6 +1204,11 @@ async def handler(dtls_channel): self._check_closed() if self._listening_context is not None: raise trio.BusyResourceError("another task is already listening") + try: + self.socket.getsockname() + except OSError: + raise RuntimeError("DTLS socket must be bound before it can serve") + self._ensure_receive_loop() # We do cookie verification ourselves, so tell OpenSSL not to worry about it. # (See also _inject_client_hello_untrusted.) ssl_context.set_cookie_verify_callback(lambda *_: True) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index c889d353aa..44f967ca06 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -284,6 +284,7 @@ async def null_handler(_): # pragma: no cover pass with endpoint() as server_endpoint: + await server_endpoint.socket.bind(("127.0.0.1", 0)) async with trio.open_nursery() as nursery: await nursery.start(server_endpoint.serve, server_ctx, null_handler) with pytest.raises(trio.BusyResourceError): @@ -696,9 +697,21 @@ def route_packet(packet): async def test_system_task_cleaned_up_on_gc(): before_tasks = trio.lowlevel.current_statistics().tasks_living + e = endpoint() - # Give system task a chance to start up - await trio.testing.wait_all_tasks_blocked() + + async def force_receive_loop_to_start(): + # This connection/handshake attempt can't succeed. The only purpose is to force + # the endpoint to set up a receive loop. + with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: + await s.bind(("127.0.0.1", 0)) + c = e.connect(s.getsockname(), client_ctx) + async with trio.open_nursery() as nursery: + nursery.start_soon(c.do_handshake) + await trio.testing.wait_all_tasks_blocked() + nursery.cancel_scope.cancel() + + await force_receive_loop_to_start() during_tasks = trio.lowlevel.current_statistics().tasks_living From 08686f67da8e6750fa9279c140087ad787d0d7fb Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 2 Nov 2021 21:20:33 -0700 Subject: [PATCH 0955/1498] Work around bug in Ubuntu 18.04's OpenSSL --- trio/_dtls.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/trio/_dtls.py b/trio/_dtls.py index 766a3268e1..57b2437b78 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -659,6 +659,24 @@ async def handle_client_hello_untrusted(endpoint, address, packet): # ...OpenSSL didn't like it, so I guess we didn't have a valid ClientHello # after all. return + + # Some old versions of OpenSSL have a bug with memory BIOs, where DTLSv1_listen + # consumes the ClientHello out of the BIO, but then do_handshake expects the + # ClientHello to still be in there (but not the one that ships with Ubuntu + # 20.04). In particular, this is known to affect the OpenSSL v1.1.1 that ships + # with Ubuntu 18.04. To work around this, we deliver a second copy of the + # ClientHello after DTLSv1_listen has completed. This is safe to do + # unconditionally, because on newer versions of OpenSSL, the second ClientHello + # is treated as a duplicate packet, which is a normal thing that can happen over + # UDP. For more details, see: + # + # https://github.com/pyca/pyopenssl/blob/e84e7b57d1838de70ab7a27089fbee78ce0d2106/tests/test_ssl.py#L4226-L4293 + # + # This was fixed in v1.1.1a, and all later versions. So maybe in 2024 or so we + # can delete this. The fix landed in OpenSSL master as 079ef6bd534d2, and then + # was backported to the 1.1.1 branch as d1bfd8076e28. + stream._ssl.bio_write(packet) + # Check if we have an existing association old_stream = endpoint._streams.get(address) if old_stream is not None: From e5a4d0d37ced86869e09f74f8bc868940efb3f74 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 2 Nov 2021 21:21:13 -0700 Subject: [PATCH 0956/1498] Clean up test Noticed while testing on Ubuntu 18.04's weird old OpenSSL that this test was getting stuck in an infinite loop, even though it passed with more recent OpenSSL. I'm not sure why it was different, exactly, but on closer examination, writing the test this way makes more sense anyway, and now the tests pass on Ubuntu 18.04 too. --- trio/tests/test_dtls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 44f967ca06..305c4c2a19 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -448,7 +448,9 @@ async def test_invalid_cookie_rejected(autojump_clock): with trio.CancelScope() as cscope: - offset_to_corrupt = count() + # the first 11 bytes of ClientHello aren't protected by the cookie, so only test + # corrupting bytes after that. + offset_to_corrupt = count(11) def route_packet(packet): try: From 2ffd8927c0d5ff1f3ef8b87a7739a9bf3520ad49 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 2 Nov 2021 21:25:43 -0700 Subject: [PATCH 0957/1498] Temporarily switch branch to pull dev version of openssl, to let CI run --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index aa32adab96..a2f050a033 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pygments==2.10.0 # via ipython pylint==2.12.2 # via -r test-requirements.in -pyopenssl==21.0.0 +pyopenssl @ git+https://github.com/pyca/pyopenssl # via -r test-requirements.in pyparsing==3.0.7 # via packaging From f335dce06e4e8543526055626800af540d5288ed Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 2 Nov 2021 21:49:17 -0700 Subject: [PATCH 0958/1498] blacken --- trio/_dtls.py | 9 +++++---- trio/tests/test_dtls.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 57b2437b78..dc653f19ff 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -766,6 +766,7 @@ class DTLSChannel(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): The IP/port of the remote peer that this connection is associated with. """ + def __init__(self, endpoint, peer_address, ctx): self.endpoint = endpoint self.peer_address = peer_address @@ -987,9 +988,7 @@ def read_volley(): ) async def send(self, data): - """Send a packet of data, securely. - - """ + """Send a packet of data, securely.""" if self._closed: raise trio.ClosedResourceError @@ -1147,7 +1146,9 @@ def _ensure_receive_loop(self): # if the socket isn't already bound -- which for clients might not happen until # after we send our first packet. if not self._receive_loop_spawned: - trio.lowlevel.spawn_system_task(dtls_receive_loop, weakref.ref(self), self.socket) + trio.lowlevel.spawn_system_task( + dtls_receive_loop, weakref.ref(self), self.socket + ) self._receive_loop_spawned = True def __del__(self): diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 305c4c2a19..eb18d7a1a9 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -699,7 +699,6 @@ def route_packet(packet): async def test_system_task_cleaned_up_on_gc(): before_tasks = trio.lowlevel.current_statistics().tasks_living - e = endpoint() async def force_receive_loop_to_start(): @@ -812,6 +811,7 @@ def route_packet(packet): with endpoint() as client_endpoint: c1 = client_endpoint.connect(address, client_ctx) async with trio.open_nursery() as nursery: + async def doomed_handshake(): with pytest.raises(trio.BrokenResourceError): await c1.do_handshake() From acc0eacdd84c57f8f0b0a14cb50aec27153bbe70 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 3 Nov 2021 21:28:22 -0700 Subject: [PATCH 0959/1498] Restore py36 compatibility --- trio/tests/test_dtls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index eb18d7a1a9..0cadad8148 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -3,7 +3,7 @@ from trio import DTLSEndpoint import random import attr -from contextlib import asynccontextmanager +from async_generator import asynccontextmanager from itertools import count import ipaddress import warnings From e3fb2d823f84210a3ab5adb5ede577ad2c1f2a15 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 3 Nov 2021 21:29:01 -0700 Subject: [PATCH 0960/1498] Maybe this will work? --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a2f050a033..1747ebe512 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pygments==2.10.0 # via ipython pylint==2.12.2 # via -r test-requirements.in -pyopenssl @ git+https://github.com/pyca/pyopenssl +pyopenssl @ https://github.com/pyca/pyopenssl/archive/refs/heads/main.zip # via -r test-requirements.in pyparsing==3.0.7 # via packaging From 78634986cabb747af99a3579a715cc648b9a7c01 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 3 Nov 2021 21:38:26 -0700 Subject: [PATCH 0961/1498] Pacify flake8 --- trio/_dtls.py | 2 +- trio/tests/test_dtls.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index dc653f19ff..5bd0cc7872 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -936,7 +936,7 @@ def read_volley(): self._ssl.do_handshake() # We ignore generic SSL.Error here, because you can get those # from random invalid packets - except (SSL.WantReadError, SSL.Error) as exc: + except (SSL.WantReadError, SSL.Error): pass else: # No exception -> the handshake is done, and we can diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 0cadad8148..68107fe86d 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -820,7 +820,7 @@ async def doomed_handshake(): await trio.sleep(10) - c2 = client_endpoint.connect(address, client_ctx) + client_endpoint.connect(address, client_ctx) async def test_association_replaced_before_handshake_starts(): @@ -836,7 +836,7 @@ def route_packet(packet): # pragma: no cover async with dtls_echo_server() as (_, address): with endpoint() as client_endpoint: c1 = client_endpoint.connect(address, client_ctx) - c2 = client_endpoint.connect(address, client_ctx) + client_endpoint.connect(address, client_ctx) with pytest.raises(trio.BrokenResourceError): await c1.do_handshake() From 6e3aca08fb0e290b19080502ac2aa8b36e669f64 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 3 Nov 2021 21:47:32 -0700 Subject: [PATCH 0962/1498] make mypy happy --- test-requirements.in | 1 + test-requirements.txt | 13 ++++--------- trio/_dtls.py | 17 +++++++++-------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 4d15de8e4f..2c75770524 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -12,6 +12,7 @@ cryptography>=36.0.0 # 35.0.0 is transitive but fails # Tools black; implementation_name == "cpython" mypy; implementation_name == "cpython" +types-pyOpenSSL; implementation_name == "cpython" flake8 astor # code generation diff --git a/test-requirements.txt b/test-requirements.txt index 1747ebe512..e932052c31 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file=test-requirements.txt test-requirements.in +# pip-compile --output-file test-requirements.txt test-requirements.in # astor==0.8.1 # via -r test-requirements.in @@ -25,9 +25,8 @@ click==8.0.3 # via black coverage[toml]==6.0.2 # via pytest-cov -cryptography==36.0.1 +cryptography==35.0.0 # via - # -r test-requirements.in # pyopenssl # trustme decorator==5.1.1 @@ -127,8 +126,6 @@ traitlets==5.1.1 # ipython # matplotlib-inline trustme==0.9.0 - # via -r test-requirements.in -typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in # black @@ -137,6 +134,4 @@ wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +types-pyOpenSSL diff --git a/trio/_dtls.py b/trio/_dtls.py index 5bd0cc7872..fc3d111931 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -130,15 +130,16 @@ def is_client_hello_untrusted(packet): RECORD_HEADER = struct.Struct("!B2sQH") -hex_repr = attr.ib(repr=lambda data: data.hex()) # pragma: no cover +def to_hex(data: bytes) -> str: # pragma: no cover + return data.hex() @attr.frozen class Record: content_type: int - version: bytes = hex_repr + version: bytes = attr.ib(repr=to_hex) epoch_seqno: int - payload: bytes = hex_repr + payload: bytes = attr.ib(repr=to_hex) def records_untrusted(packet): @@ -185,7 +186,7 @@ class HandshakeFragment: msg_seq: int frag_offset: int frag_len: int - frag: bytes = hex_repr + frag: bytes = attr.ib(repr=to_hex) def decode_handshake_fragment_untrusted(payload): @@ -300,19 +301,19 @@ def decode_client_hello_untrusted(packet): @attr.frozen class HandshakeMessage: - record_version: bytes = hex_repr + record_version: bytes = attr.ib(repr=to_hex) msg_type: HandshakeType msg_seq: int - body: bytearray = hex_repr + body: bytearray = attr.ib(repr=to_hex) # ChangeCipherSpec is part of the handshake, but it's not a "handshake # message" and can't be fragmented the same way. Sigh. @attr.frozen class PseudoHandshakeMessage: - record_version: bytes = hex_repr + record_version: bytes = attr.ib(repr=to_hex) content_type: int - payload: bytes = hex_repr + payload: bytes = attr.ib(repr=to_hex) # The final record in a handshake is Finished, which is encrypted, can't be fragmented From 1559b56407050a268b1df5b5dddab3700fe73a3c Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 3 Nov 2021 22:10:09 -0700 Subject: [PATCH 0963/1498] more py36 --- trio/_dtls.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index fc3d111931..d5c5e66d94 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -42,6 +42,14 @@ def best_guess_mtu(sock): return 1500 - packet_header_overhead(sock) +try: + from hmac import digest +except ImportError: + # python 3.6 + def digest(key, msg, algorithm): + return hmac.new(key, msg, algorithm).digest() + + # There are a bunch of different RFCs that define these codes, so for a # comprehensive collection look here: # https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml @@ -540,7 +548,7 @@ def _make_cookie(key, salt, tick, address, client_hello_bits): client_hello_bits, ) - return (salt + hmac.digest(key, signable_data, COOKIE_HASH))[:COOKIE_LENGTH] + return (salt + digest(key, signable_data, COOKIE_HASH))[:COOKIE_LENGTH] def valid_cookie(key, cookie, address, client_hello_bits): From 86ab14d45b653e6ad0397218a8204716216015d5 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Wed, 3 Nov 2021 22:26:26 -0700 Subject: [PATCH 0964/1498] shut up mypy --- trio/_dtls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index d5c5e66d94..09dd4bf4b2 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -46,7 +46,7 @@ def best_guess_mtu(sock): from hmac import digest except ImportError: # python 3.6 - def digest(key, msg, algorithm): + def digest(key, msg, algorithm): # type: ignore return hmac.new(key, msg, algorithm).digest() From 1714e73c5b711f3ec6e3b3b9e9f2da85c826a319 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Fri, 28 Jan 2022 17:25:33 -0800 Subject: [PATCH 0965/1498] remove unneeded import --- trio/_dtls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 09dd4bf4b2..66e19a0367 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -9,7 +9,6 @@ import struct import hmac import os -import io import enum from itertools import count import weakref From 1fc7847a8992d396e29a132e4707e01ac3255218 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 29 Jan 2022 12:29:39 -0800 Subject: [PATCH 0966/1498] pyopenssl has released! --- test-requirements.in | 4 ++-- test-requirements.txt | 27 ++++++++++++++++++++------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 2c75770524..6d12a658e2 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -3,8 +3,8 @@ pytest >= 5.0 # for faulthandler in core pytest-cov >= 2.6.0 # ipython 7.x is the last major version supporting Python 3.7 ipython ~= 7.31 # for the IPython traceback integration tests -pyOpenSSL # for the ssl tests -trustme # for the ssl tests +pyOpenSSL >= 22.0.0 # for the ssl + DTLS tests +trustme # for the ssl + DTLS tests pylint # for pylint finding all symbols tests jedi # for jedi code completion tests cryptography>=36.0.0 # 35.0.0 is transitive but fails diff --git a/test-requirements.txt b/test-requirements.txt index e932052c31..54d4add08b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --output-file test-requirements.txt test-requirements.in +# pip-compile test-requirements.in # astor==0.8.1 # via -r test-requirements.in @@ -25,8 +25,9 @@ click==8.0.3 # via black coverage[toml]==6.0.2 # via pytest-cov -cryptography==35.0.0 +cryptography==36.0.1 # via + # -r test-requirements.in # pyopenssl # trustme decorator==5.1.1 @@ -96,7 +97,7 @@ pygments==2.10.0 # via ipython pylint==2.12.2 # via -r test-requirements.in -pyopenssl @ https://github.com/pyca/pyopenssl/archive/refs/heads/main.zip +pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.7 # via packaging @@ -106,8 +107,6 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -six==1.16.0 - # via pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.4.0 @@ -126,12 +125,26 @@ traitlets==5.1.1 # ipython # matplotlib-inline trustme==0.9.0 + # via -r test-requirements.in +types-cryptography==3.3.14 + # via types-pyopenssl +types-enum34==1.1.8 + # via types-cryptography +types-ipaddress==1.0.7 + # via types-cryptography +types-pyopenssl==21.0.3 ; implementation_name == "cpython" + # via -r test-requirements.in +typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid # black # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.13.3 # via astroid -types-pyOpenSSL + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 10f450645d94e27a39513d6da2171e2da7e6a6e7 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 29 Jan 2022 12:30:13 -0800 Subject: [PATCH 0967/1498] Work around pypy gc quirks --- trio/tests/test_dtls.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 68107fe86d..8968d9a601 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -1,12 +1,11 @@ import pytest import trio +import trio.testing from trio import DTLSEndpoint import random import attr from async_generator import asynccontextmanager from itertools import count -import ipaddress -import warnings import trustme from OpenSSL import SSL @@ -699,9 +698,13 @@ def route_packet(packet): async def test_system_task_cleaned_up_on_gc(): before_tasks = trio.lowlevel.current_statistics().tasks_living - e = endpoint() + # We put this into a sub-function so that everything automatically becomes garbage + # when the frame exits. For some reason just doing 'del e' wasn't enough on pypy + # with coverage enabled -- I think we were hitting this bug: + # https://foss.heptapod.net/pypy/pypy/-/issues/3656 + async def start_and_forget_endpoint(): + e = endpoint() - async def force_receive_loop_to_start(): # This connection/handshake attempt can't succeed. The only purpose is to force # the endpoint to set up a receive loop. with trio.socket.socket(type=trio.socket.SOCK_DGRAM) as s: @@ -712,12 +715,12 @@ async def force_receive_loop_to_start(): await trio.testing.wait_all_tasks_blocked() nursery.cancel_scope.cancel() - await force_receive_loop_to_start() - - during_tasks = trio.lowlevel.current_statistics().tasks_living + during_tasks = trio.lowlevel.current_statistics().tasks_living + return during_tasks with pytest.warns(ResourceWarning): - del e + during_tasks = await start_and_forget_endpoint() + await trio.testing.wait_all_tasks_blocked() gc_collect_harder() await trio.testing.wait_all_tasks_blocked() @@ -745,6 +748,7 @@ async def test_gc_as_packet_received(): e = endpoint() await e.socket.bind(("127.0.0.1", 0)) + e._ensure_receive_loop() await trio.testing.wait_all_tasks_blocked() @@ -761,6 +765,12 @@ async def test_gc_as_packet_received(): @pytest.mark.filterwarnings("always:unclosed DTLS:ResourceWarning") def test_gc_after_trio_exits(): async def main(): + # We use fakenet just to make sure no real sockets can leak out of the test + # case - on pypy somehow the socket was outliving the gc_collect_harder call + # below. Since the test is just making sure DTLSEndpoint.__del__ doesn't explode + # when called after trio exits, it doesn't need a real socket. + fn = FakeNet() + fn.enable() return endpoint() e = trio.run(main) From 284cf2d9be44f5647c5df3a1662493fbacd6df84 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 29 Jan 2022 12:45:24 -0800 Subject: [PATCH 0968/1498] Drop py36 support --- trio/_dtls.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 66e19a0367..4cd4392fbc 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -41,14 +41,6 @@ def best_guess_mtu(sock): return 1500 - packet_header_overhead(sock) -try: - from hmac import digest -except ImportError: - # python 3.6 - def digest(key, msg, algorithm): # type: ignore - return hmac.new(key, msg, algorithm).digest() - - # There are a bunch of different RFCs that define these codes, so for a # comprehensive collection look here: # https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml @@ -547,7 +539,7 @@ def _make_cookie(key, salt, tick, address, client_hello_bits): client_hello_bits, ) - return (salt + digest(key, signable_data, COOKIE_HASH))[:COOKIE_LENGTH] + return (salt + hmac.digest(key, signable_data, COOKIE_HASH))[:COOKIE_LENGTH] def valid_cookie(key, cookie, address, client_hello_bits): From 2f2e37cc55436d30c656147eca6325133fc4a05e Mon Sep 17 00:00:00 2001 From: Robie Basak Date: Mon, 15 Nov 2021 21:52:59 +0000 Subject: [PATCH 0969/1498] Fix test_close_at_bad_time_for_send_all on ppc64el On ppc64el, PIPE_BUF is 8192 but make_clogged_pipe() ends up writing a total of 1048576 bytes before the pipe is full, and then a subsequent receive_some(10000) isn't sufficient for orig_wait_writable() to return for our subsequent aclose() call. It's necessary to empty the pipe further before this happens. So we must loop until the pipe is empty to make sure that the sender wakes up even in this case. Otherwise patched_wait_writable() never gets to the aclose(), so expect_closedresourceerror() never returns, the nursery never finishes all tasks and this test hangs. --- trio/tests/test_unix_pipes.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/trio/tests/test_unix_pipes.py b/trio/tests/test_unix_pipes.py index 55dd4e3734..79d0bde57e 100644 --- a/trio/tests/test_unix_pipes.py +++ b/trio/tests/test_unix_pipes.py @@ -229,8 +229,19 @@ async def patched_wait_writable(*args, **kwargs): async with _core.open_nursery() as nursery: nursery.start_soon(expect_closedresourceerror) await wait_all_tasks_blocked() - # Trigger everything by waking up the sender - await r.receive_some(10000) + # Trigger everything by waking up the sender. On ppc64el, PIPE_BUF + # is 8192 but make_clogged_pipe() ends up writing a total of + # 1048576 bytes before the pipe is full, and then a subsequent + # receive_some(10000) isn't sufficient for orig_wait_writable() to + # return for our subsequent aclose() call. It's necessary to empty + # the pipe further before this happens. So we loop here until the + # pipe is empty to make sure that the sender wakes up even in this + # case. Otherwise patched_wait_writable() never gets to the + # aclose(), so expect_closedresourceerror() never returns, the + # nursery never finishes all tasks and this test hangs. + received_data = await r.receive_some(10000) + while received_data: + received_data = await r.receive_some(10000) # On FreeBSD, directories are readable, and we haven't found any other trick From 1e9af84b52b7c8799aee9b7c7e08ab560a808c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 30 Jan 2022 16:38:51 +0100 Subject: [PATCH 0970/1498] =?UTF-8?q?=F0=9F=94=A5=20Remove=20context=20par?= =?UTF-8?q?ameter=20from=20nursery.start=5Fsoon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit as requested in code review --- trio/_core/_run.py | 11 ++--------- trio/tests/test_contextvars.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 847651d705..ca8a0b07fd 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -957,7 +957,7 @@ def aborted(raise_cancel): # (see test_nursery_cancel_doesnt_create_cyclic_garbage) del self._pending_excs - def start_soon(self, async_fn, *args, name=None, context=None): + def start_soon(self, async_fn, *args, name=None): """Creates a child task, scheduling ``await async_fn(*args)``. This and :meth:`start` are the two fundamental methods for @@ -992,20 +992,13 @@ def start_soon(self, async_fn, *args, name=None, context=None): before spawning a new task, you might pass the original function as the ``name=`` to make debugging easier. - context: An optional ``contextvars.Context`` object with context - variables to use for this task. You would normally get a copy - of the current context with - ``context = contextvars.copy_context()`` and then you would - pass that ``context`` object here. Raises: RuntimeError: If this nursery is no longer open (i.e. its ``async with`` block has exited). """ - GLOBAL_RUN_CONTEXT.runner.spawn_impl( - async_fn, args, self, name, context=context - ) + GLOBAL_RUN_CONTEXT.runner.spawn_impl(async_fn, args, self, name) async def start(self, async_fn, *args, name=None): r"""Creates and initializes a child task. diff --git a/trio/tests/test_contextvars.py b/trio/tests/test_contextvars.py index 375e155394..63853f5171 100644 --- a/trio/tests/test_contextvars.py +++ b/trio/tests/test_contextvars.py @@ -45,7 +45,7 @@ async def child(): record.append(value) async with _core.open_nursery() as nursery: - nursery.start_soon(child, context=context) + context.run(nursery.start_soon, child) nursery.start_soon(child) value = trio_testing_contextvar.get() assert set(record) == {"main", "second_main"} From 53ae1a1c4d03e1db1955e0194565d3c1ee141641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 30 Jan 2022 16:42:31 +0100 Subject: [PATCH 0971/1498] =?UTF-8?q?=F0=9F=94=A5=20Remove=20mention=20in?= =?UTF-8?q?=20docs=20to=20removed=20start=5Fsoon(context=3Dcontext)=20para?= =?UTF-8?q?meter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/reference-core.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 720bfbf590..58d583a39d 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -1023,16 +1023,6 @@ Example output (yours may differ slightly): For more information, read the `contextvars docs `__. -.. note:: - - In advanced use cases you can also pass a custom `contextvars` context to - ``nursery.start_soon(child, context=context)``. - - That's an advanced use case that you probably won't need, but now you - know why that keyword argument exists! - - You will also learn about how to use these `contextvars` with threads later here - in the docs. .. _synchronization: From 0a388bb83bf766324bb4dcea4892fa4173e1ef90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 30 Jan 2022 16:44:39 +0100 Subject: [PATCH 0972/1498] =?UTF-8?q?=F0=9F=94=A5=20Remove=20mention=20to?= =?UTF-8?q?=20discarded=20nursery.start=5Fsoon(context=3Dcontext)=20in=20n?= =?UTF-8?q?ewsfragment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- newsfragments/2160.feature.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst index 1bca734ed1..4193d1ca13 100644 --- a/newsfragments/2160.feature.rst +++ b/newsfragments/2160.feature.rst @@ -2,5 +2,4 @@ Now context variables set with `contextvars` are preserved when running function in a worker thread with `trio.to_thread.run_sync`, or when running functions from the worker thread in the parent Trio thread with `trio.from_thread.run`, and `trio.from_thread.run_sync`. -This is done by automatically copying the `contextvars` context. You can also pass a -custom `contextvars` context to ``nursery.start_soon(child, context=context)``. +This is done by automatically copying the `contextvars` context. From e6093296e2a76f1c9d8551948d0b51b7ddbf2c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 30 Jan 2022 19:47:06 +0100 Subject: [PATCH 0973/1498] =?UTF-8?q?=F0=9F=93=9D=20Update=20newsfragment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- newsfragments/2160.feature.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst index 4193d1ca13..dfe51d5b5c 100644 --- a/newsfragments/2160.feature.rst +++ b/newsfragments/2160.feature.rst @@ -3,3 +3,4 @@ in a worker thread with `trio.to_thread.run_sync`, or when running functions from the worker thread in the parent Trio thread with `trio.from_thread.run`, and `trio.from_thread.run_sync`. This is done by automatically copying the `contextvars` context. +`trio.lowlevel.spawn_system_task` now also receives an optional `context` argument. From 02966f11e434bb448775062d72c36a7a8062a377 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sun, 30 Jan 2022 13:54:40 -0500 Subject: [PATCH 0974/1498] fix rtd with double backticks --- newsfragments/2160.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst index dfe51d5b5c..00cb09a975 100644 --- a/newsfragments/2160.feature.rst +++ b/newsfragments/2160.feature.rst @@ -3,4 +3,4 @@ in a worker thread with `trio.to_thread.run_sync`, or when running functions from the worker thread in the parent Trio thread with `trio.from_thread.run`, and `trio.from_thread.run_sync`. This is done by automatically copying the `contextvars` context. -`trio.lowlevel.spawn_system_task` now also receives an optional `context` argument. +`trio.lowlevel.spawn_system_task` now also receives an optional ``context`` argument. From e2fbec01f2a5a3a28d09f8f005016368944f9d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 30 Jan 2022 23:54:09 +0200 Subject: [PATCH 0975/1498] Updated an example to refer to BaseExceptionGroup --- docs/source/tutorial/echo-server.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/source/tutorial/echo-server.py b/docs/source/tutorial/echo-server.py index ffa63ab5f4..3751cadd73 100644 --- a/docs/source/tutorial/echo-server.py +++ b/docs/source/tutorial/echo-server.py @@ -11,6 +11,7 @@ CONNECTION_COUNTER = count() + async def echo_server(server_stream): # Assign each connection a unique number to make our debug prints easier # to understand when there are multiple simultaneous connections. @@ -21,18 +22,20 @@ async def echo_server(server_stream): print(f"echo_server {ident}: received data {data!r}") await server_stream.send_all(data) print(f"echo_server {ident}: connection closed") - # FIXME: add discussion of MultiErrors to the tutorial, and use - # MultiError.catch here. (Not important in this case, but important if the - # server code uses nurseries internally.) + # FIXME: add discussion of (Base)ExceptionGroup to the tutorial, and use + # exceptiongroup.catch() here. (Not important in this case, but important + # if the server code uses nurseries internally.) except Exception as exc: # Unhandled exceptions will propagate into our parent and take # down the whole program. If the exception is KeyboardInterrupt, # that's what we want, but otherwise maybe not... print(f"echo_server {ident}: crashed: {exc!r}") + async def main(): await trio.serve_tcp(echo_server, PORT) + # We could also just write 'trio.run(trio.serve_tcp, echo_server, PORT)', but real # programs almost always end up doing other stuff too and then we'd have to go # back and factor it out into a separate function anyway. So it's simplest to From 95a7b9eff089730624ecd5135871951ef426fbcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 31 Jan 2022 00:55:01 +0200 Subject: [PATCH 0976/1498] Improved the docs regarding exception groups --- docs/source/conf.py | 3 ++- docs/source/reference-core.rst | 45 +++++++++++++++++++++++++++++---- newsfragments/2211.breaking.rst | 7 +++++ newsfragments/2211.removal.rst | 4 --- 4 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 newsfragments/2211.breaking.rst delete mode 100644 newsfragments/2211.removal.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index 6045ffd828..760f8837a2 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -84,8 +84,9 @@ def setup(app): 'local_customization', ] +# FIXME: change the "python" link back to /3 when Python 3.11 is released intersphinx_mapping = { - "python": ('https://docs.python.org/3', None), + "python": ('https://docs.python.org/3.11', None), "outcome": ('https://outcome.readthedocs.io/en/latest/', None), } diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 658e14d08e..2cc9c0e3f9 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -687,6 +687,8 @@ You might wonder why Trio can't just remember "this task should be cancelled in If you want a timeout to apply to one task but not another, then you need to put the cancel scope in that individual task's function -- ``child()``, in this example. +.. _exceptiongroups: + Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -709,13 +711,46 @@ limitation. Consider code like:: ``broken1`` raises ``KeyError``. ``broken2`` raises ``IndexError``. Obviously ``parent`` should raise some error, but -what? In some sense, the answer should be "both of these at once", but -in Python there can only be one exception at a time. +what? The answer is that both exceptions are grouped in an `ExceptionGroup`. +The `ExceptionGroup` and its parent class `BaseExceptionGroup` are used to encapsulate +multiple exceptions being raised at once. + +To catch individual exceptions encapsulated in an exception group, the ``except*`` +clause was introduced in Python 3.11 (:pep:`654`). Here's how it works:: + + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(broken1) + nursery.start_soon(broken2) + except* KeyError: + ... # handle each KeyError + except* IndexError: + ... # handle each IndexError + +But what if you can't use ``except*`` just yet? Well, for that there is the handy +exceptiongroup_ library which lets you approximate this behavior with exception handler +callbacks:: + + from exceptiongroup import catch + + def handle_keyerror(exc): + ... # handle each KeyError + + def handle_indexerror(exc): + ... # handle each IndexError + + with catch({ + KeyError: handle_keyerror, + IndexError: handle_indexerror + }): + async with trio.open_nursery() as nursery: + nursery.start_soon(broken1) + nursery.start_soon(broken2) -Trio's answer is that it raises a ``BaseExceptionGroup`` object. This is a -special exception which encapsulates multiple exception objects – -either regular exceptions or nested ``BaseExceptionGroup``\s. +.. hint:: If your code, written using ``except*``, would set local variables, you can do + the same with handler callbacks as long as you declare those variables ``nonlocal``. +.. _exceptiongroup: https://pypi.org/project/exceptiongroup/ Spawning tasks without becoming a parent ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/newsfragments/2211.breaking.rst b/newsfragments/2211.breaking.rst new file mode 100644 index 0000000000..84ec5427cb --- /dev/null +++ b/newsfragments/2211.breaking.rst @@ -0,0 +1,7 @@ +``trio.MultiError`` has been removed in favor of the built-in :exc:`BaseExceptionGroup` +(and its derivative :exc:`ExceptionGroup`), falling back to the backport_ on +Python < 3.11. +See the :ref:`updated documentation ` on how to deal with exception +groups. + +.. _backport: https://pypi.org/project/exceptiongroup/ diff --git a/newsfragments/2211.removal.rst b/newsfragments/2211.removal.rst deleted file mode 100644 index 728ac2f2ff..0000000000 --- a/newsfragments/2211.removal.rst +++ /dev/null @@ -1,4 +0,0 @@ -``trio.MultiError`` has been removed in favor of the built-in ``BaseExceptionGroup`` -(and its derivative ``ExceptionGroup``), falling back to the backport_ on Python < 3.11. - -.. _backport: https://pypi.org/project/exceptiongroup/ From 745f3cf798cb111fd2c21c542a2bb1aab8ad58c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 31 Jan 2022 01:11:18 +0200 Subject: [PATCH 0977/1498] Added Python 3.11 to the CI matrix --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65fa899179..6dbf6beafa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] arch: ['x86', 'x64'] lsp: [''] lsp_extract_file: [''] @@ -72,7 +72,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.7', 'pypy-3.8', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] + python: ['pypy-3.7', 'pypy-3.8', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev', '3.11-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] @@ -114,7 +114,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] include: - python: '3.8' # <- not actually used arch: 'x64' From bc043aaa63cb5cd82ada7cc9b7bc1e3b0beb552f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 31 Jan 2022 01:46:27 +0200 Subject: [PATCH 0978/1498] Attempt at fixing Python 3.11 test runs --- test-requirements.in | 1 + test-requirements.txt | 37 +++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 024ea35b21..3c5079838b 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -9,6 +9,7 @@ pylint # for pylint finding all symbols tests jedi # for jedi code completion tests cryptography>=36.0.0 # 35.0.0 is transitive but fails exceptiongroup # for catch() +wrapt @ git+https://github.com/grayjk/wrapt.git@issue-196 # fixes Python 3.11 compat for pylint # Tools black; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index e15470a0df..ed45c4655c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.7 # To update, run: # -# pip-compile --output-file=test-requirements.txt test-requirements.in +# pip-compile test-requirements.in # astor==0.8.1 # via -r test-requirements.in @@ -17,13 +17,13 @@ attrs==21.4.0 # pytest backcall==0.2.0 # via ipython -black==21.12b0 ; implementation_name == "cpython" +black==22.1.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography click==8.0.3 # via black -coverage[toml]==6.0.2 +coverage[toml]==6.3 # via pytest-cov cryptography==36.0.1 # via @@ -40,6 +40,12 @@ idna==3.3 # via # -r test-requirements.in # trustme +importlib-metadata==4.2.0 + # via + # click + # flake8 + # pluggy + # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 @@ -95,11 +101,11 @@ pycparser==2.21 # via cffi pyflakes==2.4.0 # via flake8 -pygments==2.10.0 +pygments==2.11.2 # via ipython pylint==2.12.2 # via -r test-requirements.in -pyopenssl==21.0.0 +pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.7 # via packaging @@ -109,8 +115,6 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -six==1.16.0 - # via pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.4.0 @@ -119,7 +123,7 @@ toml==0.10.2 # via # pylint # pytest -tomli==1.2.3 +tomli==2.0.0 # via # black # coverage @@ -130,15 +134,28 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in +typed-ast==1.5.2 ; implementation_name == "cpython" and python_version < "3.8" + # via + # -r test-requirements.in + # astroid + # black + # mypy typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid # black + # importlib-metadata # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.13.3 - # via astroid +wrapt @ git+https://github.com/grayjk/wrapt.git@issue-196 + # via + # -r test-requirements.in + # astroid +zipp==3.7.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 797c234cc3b384f526432895a8d1351683082f8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 31 Jan 2022 01:49:52 +0200 Subject: [PATCH 0979/1498] Removed Windows and macOS py3.11 test jobs These platforms don't work with Python 3.11 on GitHub Actions yet. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dbf6beafa..3c7fa94e1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] + python: ['3.7', '3.8', '3.9', '3.10'] arch: ['x86', 'x64'] lsp: [''] lsp_extract_file: [''] @@ -114,7 +114,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] + python: ['3.7', '3.8', '3.9', '3.10'] include: - python: '3.8' # <- not actually used arch: 'x64' From 8a0977876febeceb47a676989a710dbebbcbeb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 31 Jan 2022 01:52:46 +0200 Subject: [PATCH 0980/1498] Skip test_coroutine_or_error() on Python 3.11 --- trio/tests/test_util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 2d57e0ebfc..bf2f606f00 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -1,4 +1,6 @@ import signal +import sys + import pytest import trio @@ -93,6 +95,9 @@ def not_main_thread(): # @coroutine is deprecated since python 3.8, which is fine with us. @pytest.mark.filterwarnings("ignore:.*@coroutine.*:DeprecationWarning") +@pytest.mark.skipif( + sys.version_info >= (3, 11), reason="asyncio.coroutine was removed in Python 3.11" +) def test_coroutine_or_error(): class Deferred: "Just kidding" From b60424cd18c7201b78780eadb7f02e10498cc53a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 10:18:19 +0000 Subject: [PATCH 0981/1498] Bump charset-normalizer from 2.0.10 to 2.0.11 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.10 to 2.0.11. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.10...2.0.11) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index dfa024dd99..c178c771a1 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.10.8 # via requests -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via requests click==8.0.3 # via From 98d02723fd3d710fbc1b36c78ff435500fb7e6ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 10:19:15 +0000 Subject: [PATCH 0982/1498] Bump pyopenssl from 21.0.0 to 22.0.0 Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 21.0.0 to 22.0.0. - [Release notes](https://github.com/pyca/pyopenssl/releases) - [Changelog](https://github.com/pyca/pyopenssl/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/pyopenssl/compare/21.0.0...22.0.0) --- updated-dependencies: - dependency-name: pyopenssl dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index aa32adab96..5394492aef 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pygments==2.10.0 # via ipython pylint==2.12.2 # via -r test-requirements.in -pyopenssl==21.0.0 +pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.7 # via packaging @@ -107,8 +107,6 @@ pytest==6.2.5 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -six==1.16.0 - # via pyopenssl sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.4.0 From 2a7096ea1aff1b85a39f19bd49561e5c4c597c91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Feb 2022 01:05:35 +0200 Subject: [PATCH 0983/1498] Bump black from 21.12b0 to 22.1.0 (#2241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump black from 21.12b0 to 22.1.0 Bumps [black](https://github.com/psf/black) from 21.12b0 to 22.1.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits/22.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Applied black formatting changes Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Grönholm --- test-requirements.txt | 3 +-- trio/_core/_io_windows.py | 2 +- trio/_core/_wakeup_socketpair.py | 2 +- trio/_highlevel_socket.py | 2 +- trio/testing/_check_streams.py | 4 ++-- trio/tests/test_subprocess.py | 2 +- trio/tests/test_unix_pipes.py | 2 +- trio/tests/test_windows_pipes.py | 2 +- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5394492aef..fffde3ef6d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==21.4.0 # pytest backcall==0.2.0 # via ipython -black==21.12b0 ; implementation_name == "cpython" +black==22.1.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography @@ -129,7 +129,6 @@ trustme==0.9.0 typing-extensions==4.0.1 ; implementation_name == "cpython" # via # -r test-requirements.in - # black # mypy wcwidth==0.2.5 # via prompt-toolkit diff --git a/trio/_core/_io_windows.py b/trio/_core/_io_windows.py index 6d3994499f..9b5ebfc268 100644 --- a/trio/_core/_io_windows.py +++ b/trio/_core/_io_windows.py @@ -639,7 +639,7 @@ def _refresh_afd(self, base_handle): lpOverlapped = ffi.new("LPOVERLAPPED") poll_info = ffi.new("AFD_POLL_INFO *") - poll_info.Timeout = 2 ** 63 - 1 # INT64_MAX + poll_info.Timeout = 2**63 - 1 # INT64_MAX poll_info.NumberOfHandles = 1 poll_info.Exclusive = 0 poll_info.Handles[0].Handle = base_handle diff --git a/trio/_core/_wakeup_socketpair.py b/trio/_core/_wakeup_socketpair.py index 8d51419ecc..c084403eaa 100644 --- a/trio/_core/_wakeup_socketpair.py +++ b/trio/_core/_wakeup_socketpair.py @@ -42,7 +42,7 @@ async def wait_woken(self): def drain(self): try: while True: - self.wakeup_sock.recv(2 ** 16) + self.wakeup_sock.recv(2**16) except BlockingIOError: pass diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index 0d9dbc0e92..0de76aa54f 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -89,7 +89,7 @@ def __init__(self, socket): # http://devstreaming.apple.com/videos/wwdc/2015/719ui2k57m/719/719_your_app_and_next_generation_networks.pdf?dl=1 # ). The theory is that you want it to be bandwidth * # rescheduling interval. - self.setsockopt(tsocket.IPPROTO_TCP, tsocket.TCP_NOTSENT_LOWAT, 2 ** 14) + self.setsockopt(tsocket.IPPROTO_TCP, tsocket.TCP_NOTSENT_LOWAT, 2**14) except OSError: pass diff --git a/trio/testing/_check_streams.py b/trio/testing/_check_streams.py index 884e3f15cd..de1f5fe687 100644 --- a/trio/testing/_check_streams.py +++ b/trio/testing/_check_streams.py @@ -408,8 +408,8 @@ async def flipped_clogged_stream_maker(): assert isinstance(s2, Stream) # Duplex can be a bit tricky, might as well check it as well - DUPLEX_TEST_SIZE = 2 ** 20 - CHUNK_SIZE_MAX = 2 ** 14 + DUPLEX_TEST_SIZE = 2**20 + CHUNK_SIZE_MAX = 2**14 r = random.Random(0) i = r.getrandbits(8 * DUPLEX_TEST_SIZE) diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 65abe13e50..061a715104 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -268,7 +268,7 @@ async def drain_one(stream, count, digit): async def test_run(): - data = bytes(random.randint(0, 255) for _ in range(2 ** 18)) + data = bytes(random.randint(0, 255) for _ in range(2**18)) result = await run_process( CAT, stdin=data, capture_stdout=True, capture_stderr=True diff --git a/trio/tests/test_unix_pipes.py b/trio/tests/test_unix_pipes.py index 79d0bde57e..cf98942ea4 100644 --- a/trio/tests/test_unix_pipes.py +++ b/trio/tests/test_unix_pipes.py @@ -74,7 +74,7 @@ async def test_receive_pipe(): async def test_pipes_combined(): write, read = await make_pipe() - count = 2 ** 20 + count = 2**20 async def sender(): big = bytearray(count) diff --git a/trio/tests/test_windows_pipes.py b/trio/tests/test_windows_pipes.py index 361cd64ce2..0a6b3516b0 100644 --- a/trio/tests/test_windows_pipes.py +++ b/trio/tests/test_windows_pipes.py @@ -51,7 +51,7 @@ async def test_pipe_error_on_close(): async def test_pipes_combined(): write, read = await make_pipe() - count = 2 ** 20 + count = 2**20 replicas = 3 async def sender(): From 30358d1f21e3edd86811eebdde336dec476296f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 1 Feb 2022 01:34:43 +0200 Subject: [PATCH 0984/1498] Changed case to be consistent in BEG messages --- trio/_highlevel_open_tcp_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 5987a23562..d530386193 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -121,7 +121,7 @@ def close_all(): if len(errs) == 1: raise errs[0] elif errs: - raise BaseExceptionGroup("Multiple close operations failed", errs) + raise BaseExceptionGroup("multiple close operations failed", errs) def reorder_for_rfc_6555_section_5_4(targets): From 9d5ff749dea5bf98b2cb0c98225eb79076e3f22a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Feb 2022 10:20:22 +0000 Subject: [PATCH 0985/1498] Bump tomli from 1.2.3 to 2.0.0 Bumps [tomli](https://github.com/hukkin/tomli) from 1.2.3 to 2.0.0. - [Release notes](https://github.com/hukkin/tomli/releases) - [Changelog](https://github.com/hukkin/tomli/blob/master/CHANGELOG.md) - [Commits](https://github.com/hukkin/tomli/compare/1.2.3...2.0.0) --- updated-dependencies: - dependency-name: tomli dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fffde3ef6d..28bb99f7ed 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -115,7 +115,7 @@ toml==0.10.2 # via # pylint # pytest -tomli==1.2.3 +tomli==2.0.0 # via # black # coverage From 54eb0cebf3dcd8a03684189001a60a26ec9c16d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Thu, 3 Feb 2022 22:23:28 +0200 Subject: [PATCH 0986/1498] Restored (and deprecated) MultiError It now inherits from BaseExceptionGroup. --- newsfragments/2211.breaking.rst | 7 - newsfragments/2211.deprecated.rst | 12 + trio/__init__.py | 8 + trio/_core/_multierror.py | 378 ++++++++++++++++ trio/_core/_run.py | 5 +- trio/_core/tests/test_multierror.py | 409 ++++++++++++++++++ trio/_core/tests/test_run.py | 52 ++- trio/_highlevel_open_tcp_listeners.py | 8 +- trio/_highlevel_open_tcp_stream.py | 14 +- .../test_highlevel_open_tcp_listeners.py | 11 +- 10 files changed, 853 insertions(+), 51 deletions(-) delete mode 100644 newsfragments/2211.breaking.rst create mode 100644 newsfragments/2211.deprecated.rst create mode 100644 trio/_core/_multierror.py create mode 100644 trio/_core/tests/test_multierror.py diff --git a/newsfragments/2211.breaking.rst b/newsfragments/2211.breaking.rst deleted file mode 100644 index 84ec5427cb..0000000000 --- a/newsfragments/2211.breaking.rst +++ /dev/null @@ -1,7 +0,0 @@ -``trio.MultiError`` has been removed in favor of the built-in :exc:`BaseExceptionGroup` -(and its derivative :exc:`ExceptionGroup`), falling back to the backport_ on -Python < 3.11. -See the :ref:`updated documentation ` on how to deal with exception -groups. - -.. _backport: https://pypi.org/project/exceptiongroup/ diff --git a/newsfragments/2211.deprecated.rst b/newsfragments/2211.deprecated.rst new file mode 100644 index 0000000000..6becff6b98 --- /dev/null +++ b/newsfragments/2211.deprecated.rst @@ -0,0 +1,12 @@ +``MultiError`` has been deprecated in favor of the standard :exc:`BaseExceptionGroup` +(introduced in :pep:`654`). On Python < 3.11, this exception and its derivative +:exc:`ExceptionGroup` are provided by the backport_. Trio still raises ``MultiError``, +but it has been refactored into a subclass of :exc:`BaseExceptionGroup` which users +should catch instead of ``MultiError``. Uses of the ``filter()`` class method should be +replaced with :meth:`BaseExceptionGroup.split`. Uses of the ``catch()`` class method +should be replaced with either ``except*`` clauses (on Python 3.11+) or the ``catch()`` +context manager provided by the backport_. + +See the :ref:`updated documentation ` for details. + +.. _backport: https://pypi.org/project/exceptiongroup/ diff --git a/trio/__init__.py b/trio/__init__.py index 528ea15f08..ec8b58e616 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -87,6 +87,8 @@ serve_ssl_over_tcp, ) +from ._core._multierror import MultiError as _MultiError + from ._deprecate import TrioDeprecationWarning # Submodules imported by default @@ -112,6 +114,12 @@ issue=1104, instead="trio.lowlevel.open_process", ), + "MultiError": _deprecate.DeprecatedAttribute( + value=_MultiError, + version="0.20.0", + issue=2211, + instead="BaseExceptionGroup (py3.11+) or exceptiongroup.BaseExceptionGroup", + ), } # Having the public path in .__module__ attributes is important for: diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py new file mode 100644 index 0000000000..2e3d0f0ab7 --- /dev/null +++ b/trio/_core/_multierror.py @@ -0,0 +1,378 @@ +import sys +import warnings +from typing import Sequence + +import attr +from exceptiongroup._exceptions import EBase, T + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + +################################################################ +# MultiError +################################################################ + + +def _filter_impl(handler, root_exc): + # We have a tree of MultiError's, like: + # + # MultiError([ + # ValueError, + # MultiError([ + # KeyError, + # ValueError, + # ]), + # ]) + # + # or similar. + # + # We want to + # 1) apply the filter to each of the leaf exceptions -- each leaf + # might stay the same, be replaced (with the original exception + # potentially sticking around as __context__ or __cause__), or + # disappear altogether. + # 2) simplify the resulting tree -- remove empty nodes, and replace + # singleton MultiError's with their contents, e.g.: + # MultiError([KeyError]) -> KeyError + # (This can happen recursively, e.g. if the two ValueErrors above + # get caught then we'll just be left with a bare KeyError.) + # 3) preserve sensible tracebacks + # + # It's the tracebacks that are most confusing. As a MultiError + # propagates through the stack, it accumulates traceback frames, but + # the exceptions inside it don't. Semantically, the traceback for a + # leaf exception is the concatenation the tracebacks of all the + # exceptions you see when traversing the exception tree from the root + # to that leaf. Our correctness invariant is that this concatenated + # traceback should be the same before and after. + # + # The easy way to do that would be to, at the beginning of this + # function, "push" all tracebacks down to the leafs, so all the + # MultiErrors have __traceback__=None, and all the leafs have complete + # tracebacks. But whenever possible, we'd actually prefer to keep + # tracebacks as high up in the tree as possible, because this lets us + # keep only a single copy of the common parts of these exception's + # tracebacks. This is cheaper (in memory + time -- tracebacks are + # unpleasantly quadratic-ish to work with, and this might matter if + # you have thousands of exceptions, which can happen e.g. after + # cancelling a large task pool, and no-one will ever look at their + # tracebacks!), and more importantly, factoring out redundant parts of + # the tracebacks makes them more readable if/when users do see them. + # + # So instead our strategy is: + # - first go through and construct the new tree, preserving any + # unchanged subtrees + # - then go through the original tree (!) and push tracebacks down + # until either we hit a leaf, or we hit a subtree which was + # preserved in the new tree. + + # This used to also support async handler functions. But that runs into: + # https://bugs.python.org/issue29600 + # which is difficult to fix on our end. + + # Filters a subtree, ignoring tracebacks, while keeping a record of + # which MultiErrors were preserved unchanged + def filter_tree(exc, preserved): + if isinstance(exc, MultiError): + new_exceptions = [] + changed = False + for child_exc in exc.exceptions: + new_child_exc = filter_tree(child_exc, preserved) + if new_child_exc is not child_exc: + changed = True + if new_child_exc is not None: + new_exceptions.append(new_child_exc) + if not new_exceptions: + return None + elif changed: + return MultiError(new_exceptions) + else: + preserved.add(id(exc)) + return exc + else: + new_exc = handler(exc) + # Our version of implicit exception chaining + if new_exc is not None and new_exc is not exc: + new_exc.__context__ = exc + return new_exc + + def push_tb_down(tb, exc, preserved): + if id(exc) in preserved: + return + new_tb = concat_tb(tb, exc.__traceback__) + if isinstance(exc, MultiError): + for child_exc in exc.exceptions: + push_tb_down(new_tb, child_exc, preserved) + exc.__traceback__ = None + else: + exc.__traceback__ = new_tb + + preserved = set() + new_root_exc = filter_tree(root_exc, preserved) + push_tb_down(None, root_exc, preserved) + # Delete the local functions to avoid a reference cycle (see + # test_simple_cancel_scope_usage_doesnt_create_cyclic_garbage) + del filter_tree, push_tb_down + return new_root_exc + + +# Normally I'm a big fan of (a)contextmanager, but in this case I found it +# easier to use the raw context manager protocol, because it makes it a lot +# easier to reason about how we're mutating the traceback as we go. (End +# result: if the exception gets modified, then the 'raise' here makes this +# frame show up in the traceback; otherwise, we leave no trace.) +@attr.s(frozen=True) +class MultiErrorCatcher: + _handler = attr.ib() + + def __enter__(self): + pass + + def __exit__(self, etype, exc, tb): + if exc is not None: + filtered_exc = _filter_impl(self._handler, exc) + + if filtered_exc is exc: + # Let the interpreter re-raise it + return False + if filtered_exc is None: + # Swallow the exception + return True + # When we raise filtered_exc, Python will unconditionally blow + # away its __context__ attribute and replace it with the original + # exc we caught. So after we raise it, we have to pause it while + # it's in flight to put the correct __context__ back. + old_context = filtered_exc.__context__ + try: + raise filtered_exc + finally: + _, value, _ = sys.exc_info() + assert value is filtered_exc + value.__context__ = old_context + # delete references from locals to avoid creating cycles + # see test_MultiError_catch_doesnt_create_cyclic_garbage + del _, filtered_exc, value + + +class MultiError(BaseExceptionGroup): + """An exception that contains other exceptions; also known as an + "inception". + + It's main use is to represent the situation when multiple child tasks all + raise errors "in parallel". + + Args: + exceptions (list): The exceptions + + Returns: + If ``len(exceptions) == 1``, returns that exception. This means that a + call to ``MultiError(...)`` is not guaranteed to return a + :exc:`MultiError` object! + + Otherwise, returns a new :exc:`MultiError` object. + + Raises: + TypeError: if any of the passed in objects are not instances of + :exc:`BaseException`. + + """ + + def __init__(self, exceptions, *, _collapse=True): + # Avoid recursion when exceptions[0] returned by __new__() happens + # to be a MultiError and subsequently __init__() is called. + if _collapse and hasattr(self, "_exceptions"): + # __init__ was already called on this object + assert len(exceptions) == 1 and exceptions[0] is self + return + + super().__init__("multiple tasks failed", exceptions) + + def __new__(cls, exceptions, *, _collapse=True): + exceptions = list(exceptions) + for exc in exceptions: + if not isinstance(exc, BaseException): + raise TypeError(f"Expected an exception object, not {exc!r}") + if _collapse and len(exceptions) == 1: + # If this lone object happens to itself be a MultiError, then + # Python will implicitly call our __init__ on it again. See + # special handling in __init__. + return exceptions[0] + else: + # The base class __new__() implicitly invokes our __init__, which + # is what we want. + # + # In an earlier version of the code, we didn't define __init__ and + # simply set the `exceptions` attribute directly on the new object. + # However, linters expect attributes to be initialized in __init__. + return super().__new__(cls, "multiple tasks failed", exceptions) + + def __str__(self): + return ", ".join(repr(exc) for exc in self.exceptions) + + def __repr__(self): + return "".format(self) + + def derive(self: T, __excs: Sequence[EBase]) -> T: + return MultiError(__excs, _collapse=False) + + @classmethod + def filter(cls, handler, root_exc): + """Apply the given ``handler`` to all the exceptions in ``root_exc``. + + Args: + handler: A callable that takes an atomic (non-MultiError) exception + as input, and returns either a new exception object or None. + root_exc: An exception, often (though not necessarily) a + :exc:`MultiError`. + + Returns: + A new exception object in which each component exception ``exc`` has + been replaced by the result of running ``handler(exc)`` – or, if + ``handler`` returned None for all the inputs, returns None. + + """ + warnings.warn( + "MultiError.filter() has been deprecated. " + "Use the .split() method instead.", + DeprecationWarning, + ) + + return _filter_impl(handler, root_exc) + + @classmethod + def catch(cls, handler): + """Return a context manager that catches and re-throws exceptions + after running :meth:`filter` on them. + + Args: + handler: as for :meth:`filter` + + """ + warnings.warn( + "MultiError.catch() has been deprecated. " + "Use except* or exceptiongroup.catch() instead.", + DeprecationWarning, + ) + + return MultiErrorCatcher(handler) + + +# Clean up exception printing: +MultiError.__module__ = "trio" + +################################################################ +# concat_tb +################################################################ + +# We need to compute a new traceback that is the concatenation of two existing +# tracebacks. This requires copying the entries in 'head' and then pointing +# the final tb_next to 'tail'. +# +# NB: 'tail' might be None, which requires some special handling in the ctypes +# version. +# +# The complication here is that Python doesn't actually support copying or +# modifying traceback objects, so we have to get creative... +# +# On CPython, we use ctypes. On PyPy, we use "transparent proxies". +# +# Jinja2 is a useful source of inspiration: +# https://github.com/pallets/jinja/blob/master/jinja2/debug.py + +try: + import tputil +except ImportError: + have_tproxy = False +else: + have_tproxy = True + +if have_tproxy: + # http://doc.pypy.org/en/latest/objspace-proxies.html + def copy_tb(base_tb, tb_next): + def controller(operation): + # Rationale for pragma: I looked fairly carefully and tried a few + # things, and AFAICT it's not actually possible to get any + # 'opname' that isn't __getattr__ or __getattribute__. So there's + # no missing test we could add, and no value in coverage nagging + # us about adding one. + if operation.opname in [ + "__getattribute__", + "__getattr__", + ]: # pragma: no cover + if operation.args[0] == "tb_next": + return tb_next + return operation.delegate() + + return tputil.make_proxy(controller, type(base_tb), base_tb) + +else: + # ctypes it is + import ctypes + + # How to handle refcounting? I don't want to use ctypes.py_object because + # I don't understand or trust it, and I don't want to use + # ctypes.pythonapi.Py_{Inc,Dec}Ref because we might clash with user code + # that also tries to use them but with different types. So private _ctypes + # APIs it is! + import _ctypes + + class CTraceback(ctypes.Structure): + _fields_ = [ + ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), + ("tb_next", ctypes.c_void_p), + ("tb_frame", ctypes.c_void_p), + ("tb_lasti", ctypes.c_int), + ("tb_lineno", ctypes.c_int), + ] + + def copy_tb(base_tb, tb_next): + # TracebackType has no public constructor, so allocate one the hard way + try: + raise ValueError + except ValueError as exc: + new_tb = exc.__traceback__ + c_new_tb = CTraceback.from_address(id(new_tb)) + + # At the C level, tb_next either pointer to the next traceback or is + # NULL. c_void_p and the .tb_next accessor both convert NULL to None, + # but we shouldn't DECREF None just because we assigned to a NULL + # pointer! Here we know that our new traceback has only 1 frame in it, + # so we can assume the tb_next field is NULL. + assert c_new_tb.tb_next is None + # If tb_next is None, then we want to set c_new_tb.tb_next to NULL, + # which it already is, so we're done. Otherwise, we have to actually + # do some work: + if tb_next is not None: + _ctypes.Py_INCREF(tb_next) + c_new_tb.tb_next = id(tb_next) + + assert c_new_tb.tb_frame is not None + _ctypes.Py_INCREF(base_tb.tb_frame) + old_tb_frame = new_tb.tb_frame + c_new_tb.tb_frame = id(base_tb.tb_frame) + _ctypes.Py_DECREF(old_tb_frame) + + c_new_tb.tb_lasti = base_tb.tb_lasti + c_new_tb.tb_lineno = base_tb.tb_lineno + + try: + return new_tb + finally: + # delete references from locals to avoid creating cycles + # see test_MultiError_catch_doesnt_create_cyclic_garbage + del new_tb, old_tb_frame + + +def concat_tb(head, tail): + # We have to use an iterative algorithm here, because in the worst case + # this might be a RecursionError stack that is by definition too deep to + # process by recursion! + head_tbs = [] + pointer = head + while pointer is not None: + head_tbs.append(pointer) + pointer = pointer.tb_next + current_head = tail + for head_tb in reversed(head_tbs): + current_head = copy_tb(head_tb, tb_next=current_head) + return current_head diff --git a/trio/_core/_run.py b/trio/_core/_run.py index f6fa853179..9ff7f890f5 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -28,6 +28,7 @@ KIManager, enable_ki_protection, ) +from ._multierror import MultiError from ._traps import ( Abort, wait_task_rescheduled, @@ -939,7 +940,7 @@ def _child_finished(self, task, outcome): async def _nested_child_finished(self, nested_child_exc): """ - Returns BaseExceptionGroup instance if there are pending exceptions. + Returns MultiError instance if there are pending exceptions. """ if nested_child_exc is not None: self._add_exc(nested_child_exc) @@ -969,7 +970,7 @@ def aborted(raise_cancel): assert popped is self if self._pending_excs: try: - return BaseExceptionGroup("multiple tasks failed", self._pending_excs) + return MultiError(self._pending_excs) finally: # avoid a garbage cycle # (see test_nursery_cancel_doesnt_create_cyclic_garbage) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py new file mode 100644 index 0000000000..304b18558b --- /dev/null +++ b/trio/_core/tests/test_multierror.py @@ -0,0 +1,409 @@ +import gc +import logging +import pytest + +from traceback import ( + extract_tb, + print_exception, + format_exception, +) +from traceback import _cause_message # type: ignore +import sys +import re + +from .._multierror import MultiError, concat_tb +from ..._core import open_nursery + + +class NotHashableException(Exception): + code = None + + def __init__(self, code): + super().__init__() + self.code = code + + def __eq__(self, other): + if not isinstance(other, NotHashableException): + return False + return self.code == other.code + + +async def raise_nothashable(code): + raise NotHashableException(code) + + +def raiser1(): + raiser1_2() + + +def raiser1_2(): + raiser1_3() + + +def raiser1_3(): + raise ValueError("raiser1_string") + + +def raiser2(): + raiser2_2() + + +def raiser2_2(): + raise KeyError("raiser2_string") + + +def raiser3(): + raise NameError + + +def get_exc(raiser): + try: + raiser() + except Exception as exc: + return exc + + +def get_tb(raiser): + return get_exc(raiser).__traceback__ + + +def einfo(exc): + return (type(exc), exc, exc.__traceback__) + + +def test_concat_tb(): + + tb1 = get_tb(raiser1) + tb2 = get_tb(raiser2) + + # These return a list of (filename, lineno, fn name, text) tuples + # https://docs.python.org/3/library/traceback.html#traceback.extract_tb + entries1 = extract_tb(tb1) + entries2 = extract_tb(tb2) + + tb12 = concat_tb(tb1, tb2) + assert extract_tb(tb12) == entries1 + entries2 + + tb21 = concat_tb(tb2, tb1) + assert extract_tb(tb21) == entries2 + entries1 + + # Check degenerate cases + assert extract_tb(concat_tb(None, tb1)) == entries1 + assert extract_tb(concat_tb(tb1, None)) == entries1 + assert concat_tb(None, None) is None + + # Make sure the original tracebacks didn't get mutated by mistake + assert extract_tb(get_tb(raiser1)) == entries1 + assert extract_tb(get_tb(raiser2)) == entries2 + + +def test_MultiError(): + exc1 = get_exc(raiser1) + exc2 = get_exc(raiser2) + + assert MultiError([exc1]) is exc1 + m = MultiError([exc1, exc2]) + assert m.exceptions == (exc1, exc2) + assert "ValueError" in str(m) + assert "ValueError" in repr(m) + + with pytest.raises(TypeError): + MultiError(object()) + with pytest.raises(TypeError): + MultiError([KeyError(), ValueError]) + + +def test_MultiErrorOfSingleMultiError(): + # For MultiError([MultiError]), ensure there is no bad recursion by the + # constructor where __init__ is called if __new__ returns a bare MultiError. + exceptions = (KeyError(), ValueError()) + a = MultiError(exceptions) + b = MultiError([a]) + assert b == a + assert b.exceptions == exceptions + + +async def test_MultiErrorNotHashable(): + exc1 = NotHashableException(42) + exc2 = NotHashableException(4242) + exc3 = ValueError() + assert exc1 != exc2 + assert exc1 != exc3 + + with pytest.raises(MultiError): + async with open_nursery() as nursery: + nursery.start_soon(raise_nothashable, 42) + nursery.start_soon(raise_nothashable, 4242) + + +def test_MultiError_filter_NotHashable(): + excs = MultiError([NotHashableException(42), ValueError()]) + + def handle_ValueError(exc): + if isinstance(exc, ValueError): + return None + else: + return exc + + filtered_excs = pytest.deprecated_call(MultiError.filter, handle_ValueError, excs) + assert isinstance(filtered_excs, NotHashableException) + + +def test_traceback_recursion(): + exc1 = RuntimeError() + exc2 = KeyError() + exc3 = NotHashableException(42) + # Note how this creates a loop, where exc1 refers to exc1 + # This could trigger an infinite recursion; the 'seen' set is supposed to prevent + # this. + exc1.__cause__ = MultiError([exc1, exc2, exc3]) + format_exception(*einfo(exc1)) + + +def make_tree(): + # Returns an object like: + # MultiError([ + # MultiError([ + # ValueError, + # KeyError, + # ]), + # NameError, + # ]) + # where all exceptions except the root have a non-trivial traceback. + exc1 = get_exc(raiser1) + exc2 = get_exc(raiser2) + exc3 = get_exc(raiser3) + + # Give m12 a non-trivial traceback + try: + raise MultiError([exc1, exc2]) + except BaseException as m12: + return MultiError([m12, exc3]) + + +def assert_tree_eq(m1, m2): + if m1 is None or m2 is None: + assert m1 is m2 + return + assert type(m1) is type(m2) + assert extract_tb(m1.__traceback__) == extract_tb(m2.__traceback__) + assert_tree_eq(m1.__cause__, m2.__cause__) + assert_tree_eq(m1.__context__, m2.__context__) + if isinstance(m1, MultiError): + assert len(m1.exceptions) == len(m2.exceptions) + for e1, e2 in zip(m1.exceptions, m2.exceptions): + assert_tree_eq(e1, e2) + + +def test_MultiError_filter(): + def null_handler(exc): + return exc + + m = make_tree() + assert_tree_eq(m, m) + assert pytest.deprecated_call(MultiError.filter, null_handler, m) is m + assert_tree_eq(m, make_tree()) + + # Make sure we don't pick up any detritus if run in a context where + # implicit exception chaining would like to kick in + m = make_tree() + try: + raise ValueError + except ValueError: + assert pytest.deprecated_call(MultiError.filter, null_handler, m) is m + assert_tree_eq(m, make_tree()) + + def simple_filter(exc): + if isinstance(exc, ValueError): + return None + if isinstance(exc, KeyError): + return RuntimeError() + return exc + + new_m = pytest.deprecated_call(MultiError.filter, simple_filter, make_tree()) + assert isinstance(new_m, MultiError) + assert len(new_m.exceptions) == 2 + # was: [[ValueError, KeyError], NameError] + # ValueError disappeared & KeyError became RuntimeError, so now: + assert isinstance(new_m.exceptions[0], RuntimeError) + assert isinstance(new_m.exceptions[1], NameError) + + # implicit chaining: + assert isinstance(new_m.exceptions[0].__context__, KeyError) + + # also, the traceback on the KeyError incorporates what used to be the + # traceback on its parent MultiError + orig = make_tree() + # make sure we have the right path + assert isinstance(orig.exceptions[0].exceptions[1], KeyError) + # get original traceback summary + orig_extracted = ( + extract_tb(orig.__traceback__) + + extract_tb(orig.exceptions[0].__traceback__) + + extract_tb(orig.exceptions[0].exceptions[1].__traceback__) + ) + + def p(exc): + print_exception(type(exc), exc, exc.__traceback__) + + p(orig) + p(orig.exceptions[0]) + p(orig.exceptions[0].exceptions[1]) + p(new_m.exceptions[0].__context__) + # compare to the new path + assert new_m.__traceback__ is None + new_extracted = extract_tb(new_m.exceptions[0].__context__.__traceback__) + assert orig_extracted == new_extracted + + # check preserving partial tree + def filter_NameError(exc): + if isinstance(exc, NameError): + return None + return exc + + m = make_tree() + new_m = pytest.deprecated_call(MultiError.filter, filter_NameError, m) + # with the NameError gone, the other branch gets promoted + assert new_m is m.exceptions[0] + + # check fully handling everything + def filter_all(exc): + return None + + assert pytest.deprecated_call(MultiError.filter, filter_all, make_tree()) is None + + +def test_MultiError_catch(): + # No exception to catch + + def noop(_): + pass # pragma: no cover + + with pytest.deprecated_call(MultiError.catch, noop): + pass + + # Simple pass-through of all exceptions + m = make_tree() + with pytest.raises(MultiError) as excinfo: + with pytest.deprecated_call(MultiError.catch, lambda exc: exc): + raise m + assert excinfo.value is m + # Should be unchanged, except that we added a traceback frame by raising + # it here + assert m.__traceback__ is not None + assert m.__traceback__.tb_frame.f_code.co_name == "test_MultiError_catch" + assert m.__traceback__.tb_next is None + m.__traceback__ = None + assert_tree_eq(m, make_tree()) + + # Swallows everything + with pytest.deprecated_call(MultiError.catch, lambda _: None): + raise make_tree() + + def simple_filter(exc): + if isinstance(exc, ValueError): + return None + if isinstance(exc, KeyError): + return RuntimeError() + return exc + + with pytest.raises(MultiError) as excinfo: + with pytest.deprecated_call(MultiError.catch, simple_filter): + raise make_tree() + new_m = excinfo.value + assert isinstance(new_m, MultiError) + assert len(new_m.exceptions) == 2 + # was: [[ValueError, KeyError], NameError] + # ValueError disappeared & KeyError became RuntimeError, so now: + assert isinstance(new_m.exceptions[0], RuntimeError) + assert isinstance(new_m.exceptions[1], NameError) + # Make sure that Python did not successfully attach the old MultiError to + # our new MultiError's __context__ + assert not new_m.__suppress_context__ + assert new_m.__context__ is None + + # check preservation of __cause__ and __context__ + v = ValueError() + v.__cause__ = KeyError() + with pytest.raises(ValueError) as excinfo: + with pytest.deprecated_call(MultiError.catch, lambda exc: exc): + raise v + assert isinstance(excinfo.value.__cause__, KeyError) + + v = ValueError() + context = KeyError() + v.__context__ = context + with pytest.raises(ValueError) as excinfo: + with pytest.deprecated_call(MultiError.catch, lambda exc: exc): + raise v + assert excinfo.value.__context__ is context + assert not excinfo.value.__suppress_context__ + + for suppress_context in [True, False]: + v = ValueError() + context = KeyError() + v.__context__ = context + v.__suppress_context__ = suppress_context + distractor = RuntimeError() + with pytest.raises(ValueError) as excinfo: + + def catch_RuntimeError(exc): + if isinstance(exc, RuntimeError): + return None + else: + return exc + + with pytest.deprecated_call(MultiError.catch, catch_RuntimeError): + raise MultiError([v, distractor]) + assert excinfo.value.__context__ is context + assert excinfo.value.__suppress_context__ == suppress_context + + +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="Only makes sense with refcounting GC" +) +def test_MultiError_catch_doesnt_create_cyclic_garbage(): + # https://github.com/python-trio/trio/pull/2063 + gc.collect() + old_flags = gc.get_debug() + + def make_multi(): + # make_tree creates cycles itself, so a simple + raise MultiError([get_exc(raiser1), get_exc(raiser2)]) + + def simple_filter(exc): + if isinstance(exc, ValueError): + return Exception() + if isinstance(exc, KeyError): + return RuntimeError() + assert False, "only ValueError and KeyError should exist" # pragma: no cover + + try: + gc.set_debug(gc.DEBUG_SAVEALL) + with pytest.raises(MultiError): + # covers MultiErrorCatcher.__exit__ and _multierror.copy_tb + with pytest.deprecated_call(MultiError.catch, simple_filter): + raise make_multi() + gc.collect() + assert not gc.garbage + finally: + gc.set_debug(old_flags) + gc.garbage.clear() + + +def assert_match_in_seq(pattern_list, string): + offset = 0 + print("looking for pattern matches...") + for pattern in pattern_list: + print("checking pattern:", pattern) + reobj = re.compile(pattern) + match = reobj.search(string, offset) + assert match is not None + offset = match.end() + + +def test_assert_match_in_seq(): + assert_match_in_seq(["a", "b"], "xx a xx b xx") + assert_match_in_seq(["b", "a"], "xx b xx a xx") + with pytest.raises(AssertionError): + assert_match_in_seq(["a", "b"], "xx b xx a xx") diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 18dd4b2311..2dfacc3a55 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -29,6 +29,7 @@ ) from ... import _core +from ..._core._multierror import MultiError from .._run import DEADLINE_HEAP_MIN_PRUNE_THRESHOLD from ..._threads import to_thread_run_sync from ..._timeouts import sleep, fail_after @@ -171,8 +172,8 @@ async def main(): def test_main_and_task_both_crash(): - # If main crashes and there's also a task crash, then we get both in an - # ExceptionGroup + # If main crashes and there's also a task crash, then we get both in a + # MultiError async def crasher(): raise ValueError @@ -181,7 +182,7 @@ async def main(): nursery.start_soon(crasher) raise KeyError - with pytest.raises(ExceptionGroup) as excinfo: + with pytest.raises(MultiError) as excinfo: _core.run(main) print(excinfo.value) assert {type(exc) for exc in excinfo.value.exceptions} == { @@ -199,7 +200,7 @@ async def main(): nursery.start_soon(crasher, KeyError) nursery.start_soon(crasher, ValueError) - with pytest.raises(ExceptionGroup) as excinfo: + with pytest.raises(MultiError) as excinfo: _core.run(main) assert {type(exc) for exc in excinfo.value.exceptions} == { ValueError, @@ -436,7 +437,7 @@ async def crasher(): # And one that raises a different error nursery.start_soon(crasher) # t4 # and then our __aexit__ also receives an outer Cancelled - except BaseExceptionGroup as multi_exc: + except MultiError as multi_exc: # Since the outer scope became cancelled before the # nursery block exited, all cancellations inside the # nursery block continue propagating to reach the @@ -775,7 +776,7 @@ async def task2(): with pytest.raises(RuntimeError) as exc_info: await nursery_mgr.__aexit__(*sys.exc_info()) assert "which had already been exited" in str(exc_info.value) - assert type(exc_info.value.__context__) is ExceptionGroup + assert type(exc_info.value.__context__) is MultiError assert len(exc_info.value.__context__.exceptions) == 3 cancelled_in_context = False for exc in exc_info.value.__context__.exceptions: @@ -916,7 +917,7 @@ async def main(): _core.run(main) -def test_system_task_crash_ExceptionGroup(): +def test_system_task_crash_MultiError(): async def crasher1(): raise KeyError @@ -936,7 +937,7 @@ async def main(): _core.run(main) me = excinfo.value.__cause__ - assert isinstance(me, ExceptionGroup) + assert isinstance(me, MultiError) assert len(me.exceptions) == 2 for exc in me.exceptions: assert isinstance(exc, (KeyError, ValueError)) @@ -944,7 +945,7 @@ async def main(): def test_system_task_crash_plus_Cancelled(): # Set up a situation where a system task crashes with a - # BaseExceptionGroup([Cancelled, ValueError]) + # MultiError([Cancelled, ValueError]) async def crasher(): try: await sleep_forever() @@ -1115,11 +1116,11 @@ async def test_nursery_exception_chaining_doesnt_make_context_loops(): async def crasher(): raise KeyError - with pytest.raises(ExceptionGroup) as excinfo: + with pytest.raises(MultiError) as excinfo: async with _core.open_nursery() as nursery: nursery.start_soon(crasher) raise ValueError - # the ExceptionGroup should not have the KeyError or ValueError as context + # the MultiError should not have the KeyError or ValueError as context assert excinfo.value.__context__ is None @@ -1619,7 +1620,7 @@ async def test_trivial_yields(): with _core.CancelScope() as cancel_scope: cancel_scope.cancel() - with pytest.raises(BaseExceptionGroup) as excinfo: + with pytest.raises(MultiError) as excinfo: async with _core.open_nursery(): raise KeyError assert len(excinfo.value.exceptions) == 2 @@ -1709,7 +1710,7 @@ async def raise_keyerror_after_started(task_status=_core.TASK_STATUS_IGNORED): async with _core.open_nursery() as nursery: with _core.CancelScope() as cs: cs.cancel() - with pytest.raises(BaseExceptionGroup) as excinfo: + with pytest.raises(MultiError) as excinfo: await nursery.start(raise_keyerror_after_started) assert {type(e) for e in excinfo.value.exceptions} == { _core.Cancelled, @@ -1824,7 +1825,7 @@ async def fail(): async with _core.open_nursery() as nursery: nursery.start_soon(fail) raise StopIteration - except ExceptionGroup as e: + except MultiError as e: assert tuple(map(type, e.exceptions)) == (StopIteration, ValueError) @@ -1859,7 +1860,11 @@ async def __anext__(self): def handle(exc): nonlocal got_stop - got_stop = True + if isinstance(exc, StopAsyncIteration): + got_stop = True + return None + else: # pragma: no cover + return exc with catch({StopAsyncIteration: handle}): async with _core.open_nursery() as nursery: @@ -1886,7 +1891,7 @@ async def fail(): async with _core.open_nursery() as nursery: nursery.start_soon(fail) raise StopIteration - except ExceptionGroup as e: + except MultiError as e: assert tuple(map(type, e.exceptions)) == (StopIteration, ValueError) @@ -1896,14 +1901,14 @@ async def my_child_task(): try: # Trick: For now cancel/nursery scopes still leave a bunch of tb gunk - # behind. But if there's an ExceptionGroup, they leave it on the - # ExceptionGroup, which lets us get a clean look at the KeyError - # itself. Someday I guess this will always be an ExceptionGroup (#611), - # but for now we can force it by raising two exceptions. + # behind. But if there's a MultiError, they leave it on the MultiError, + # which lets us get a clean look at the KeyError itself. Someday I + # guess this will always be a MultiError (#611), but for now we can + # force it by raising two exceptions. async with _core.open_nursery() as nursery: nursery.start_soon(my_child_task) nursery.start_soon(my_child_task) - except ExceptionGroup as exc: + except MultiError as exc: first_exc = exc.exceptions[0] assert isinstance(first_exc, KeyError) # The top frame in the exception traceback should be inside the child @@ -2277,9 +2282,8 @@ async def crasher(): outer.cancel() # And one that raises a different error nursery.start_soon(crasher) - # so that outer filters a Cancelled from the BaseExceptionGroup - # and covers CancelScope.__exit__ - # (and NurseryManager.__aexit__) + # so that outer filters a Cancelled from the MultiError and + # covers CancelScope.__exit__ (and NurseryManager.__aexit__) # (See https://github.com/python-trio/trio/pull/2063) gc.collect() diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index 955b38ffe3..cce6f1d602 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -4,9 +4,7 @@ import trio from . import socket as tsocket - -if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup +from ._core._multierror import MultiError # Default backlog size: @@ -141,9 +139,7 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): errno.EAFNOSUPPORT, "This system doesn't support any of the kinds of " "socket that that address could use", - ) from ExceptionGroup( - "All socket creation attempts failed", unsupported_address_families - ) + ) from MultiError(unsupported_address_families) return listeners diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index d530386193..382a5889c9 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -1,12 +1,10 @@ -import sys +import warnings from contextlib import contextmanager import trio +from trio._core._multierror import MultiError from trio.socket import getaddrinfo, SOCK_STREAM, socket -if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup - # Implementation of RFC 6555 "Happy eyeballs" # https://tools.ietf.org/html/rfc6555 # @@ -121,7 +119,9 @@ def close_all(): if len(errs) == 1: raise errs[0] elif errs: - raise BaseExceptionGroup("multiple close operations failed", errs) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + raise MultiError(errs) def reorder_for_rfc_6555_section_5_4(targets): @@ -370,9 +370,7 @@ async def attempt_connect(socket_args, sockaddr, attempt_failed): msg = "all attempts to connect to {} failed".format( format_host_port(host, port) ) - raise OSError(msg) from BaseExceptionGroup( - "multiple connection attempts failed", oserrors - ) + raise OSError(msg) from MultiError(oserrors) else: stream = trio.SocketStream(winning_socket) open_sockets.remove(winning_socket) diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index e86d108d2b..103984739e 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -14,7 +14,7 @@ from .._core.tests.tutil import slow, creates_ipv6, binds_ipv6 if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup + from exceptiongroup import BaseExceptionGroup async def test_open_tcp_listeners_basic(): @@ -244,9 +244,12 @@ async def test_open_tcp_listeners_some_address_families_unavailable( await open_tcp_listeners(80, host="example.org") assert "This system doesn't support" in str(exc_info.value) - assert isinstance(exc_info.value.__cause__, ExceptionGroup) - for subexc in exc_info.value.__cause__.exceptions: - assert "nope" in str(subexc) + if isinstance(exc_info.value.__cause__, BaseExceptionGroup): + for subexc in exc_info.value.__cause__.exceptions: + assert "nope" in str(subexc) + else: + assert isinstance(exc_info.value.__cause__, OSError) + assert "nope" in str(exc_info.value.__cause__) else: listeners = await open_tcp_listeners(80) for listener in listeners: From f1cbcf750706c0ccd2e45aacddaebaf337869048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Fri, 4 Feb 2022 01:05:57 +0200 Subject: [PATCH 0987/1498] Removed unwarranted type annotations --- trio/_core/_multierror.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 2e3d0f0ab7..ac6e3bd2b2 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -3,7 +3,6 @@ from typing import Sequence import attr -from exceptiongroup._exceptions import EBase, T if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup @@ -212,7 +211,7 @@ def __str__(self): def __repr__(self): return "".format(self) - def derive(self: T, __excs: Sequence[EBase]) -> T: + def derive(self, __excs): return MultiError(__excs, _collapse=False) @classmethod From 0f5dc7978a242a3f9e852b7dc767679c97529e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Fri, 4 Feb 2022 01:13:05 +0200 Subject: [PATCH 0988/1498] Removed useless failing test --- trio/_core/tests/test_multierror.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 304b18558b..a2a238800f 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -149,17 +149,6 @@ def handle_ValueError(exc): assert isinstance(filtered_excs, NotHashableException) -def test_traceback_recursion(): - exc1 = RuntimeError() - exc2 = KeyError() - exc3 = NotHashableException(42) - # Note how this creates a loop, where exc1 refers to exc1 - # This could trigger an infinite recursion; the 'seen' set is supposed to prevent - # this. - exc1.__cause__ = MultiError([exc1, exc2, exc3]) - format_exception(*einfo(exc1)) - - def make_tree(): # Returns an object like: # MultiError([ From 36e0ebae12c8bb1bda331bd38c5f5da9d7bb4c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Fri, 4 Feb 2022 01:26:25 +0200 Subject: [PATCH 0989/1498] Removed unused function --- trio/_core/tests/test_multierror.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index a2a238800f..01e6253d07 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -67,10 +67,6 @@ def get_tb(raiser): return get_exc(raiser).__traceback__ -def einfo(exc): - return (type(exc), exc, exc.__traceback__) - - def test_concat_tb(): tb1 = get_tb(raiser1) From 5a2f4d5f76a398be1d67f0e9a9d2cd529951ecd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Feb 2022 10:23:57 +0000 Subject: [PATCH 0990/1498] Bump pytest from 6.2.5 to 7.0.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.5 to 7.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.5...7.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 28bb99f7ed..fae1ed3e4d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -101,7 +101,7 @@ pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.7 # via packaging -pytest==6.2.5 +pytest==7.0.0 # via # -r test-requirements.in # pytest-cov @@ -112,14 +112,13 @@ sniffio==1.2.0 sortedcontainers==2.4.0 # via -r test-requirements.in toml==0.10.2 - # via - # pylint - # pytest + # via pylint tomli==2.0.0 # via # black # coverage # mypy + # pytest traitlets==5.1.1 # via # ipython From 8bd7f47b96f2a6c41d28fb9b343156afdae93943 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 7 Feb 2022 15:59:47 +0400 Subject: [PATCH 0991/1498] Fix pytest 7 warning --- trio/_core/tests/test_guest_mode.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index 564b65d88a..a5d6d78e18 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -10,6 +10,7 @@ import socket import threading import time +import warnings import trio import trio.testing @@ -167,19 +168,16 @@ async def trio_main(in_host): assert signal.set_wakeup_fd(-1) == a.fileno() # Don't warn if there isn't already a wakeup fd - with pytest.warns(None) as record: + with warnings.catch_warnings(): + warnings.simplefilter("error") assert trivial_guest_run(trio_main) == "ok" - # Apparently this is how you assert 'there were no RuntimeWarnings' - with pytest.raises(AssertionError): - record.pop(RuntimeWarning) - with pytest.warns(None) as record: + with warnings.catch_warnings(): + warnings.simplefilter("error") assert ( trivial_guest_run(trio_main, host_uses_signal_set_wakeup_fd=True) == "ok" ) - with pytest.raises(AssertionError): - record.pop(RuntimeWarning) # If there's already a wakeup fd, but we've been told to trust it, # then it's left alone and there's no warning @@ -192,7 +190,8 @@ async def trio_check_wakeup_fd_unaltered(in_host): signal.set_wakeup_fd(fd) return "ok" - with pytest.warns(None) as record: + with warnings.catch_warnings(): + warnings.simplefilter("error") assert ( trivial_guest_run( trio_check_wakeup_fd_unaltered, @@ -200,8 +199,6 @@ async def trio_check_wakeup_fd_unaltered(in_host): ) == "ok" ) - with pytest.raises(AssertionError): - record.pop(RuntimeWarning) finally: assert signal.set_wakeup_fd(-1) == a.fileno() From 20df5d85a4f50b171079b0e9f0e42897c136b33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 8 Feb 2022 11:15:53 +0200 Subject: [PATCH 0992/1498] Update docs/source/reference-core.rst Co-authored-by: Joshua Oreman --- docs/source/reference-core.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 76561e9b5a..9639287d71 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -641,7 +641,7 @@ crucial things to keep in mind: * Any unhandled exceptions are re-raised inside the parent task. If there are multiple exceptions, then they're collected up into a - single ``BaseExceptionGroup`` or ``ExceptionGroup`` exception. + single :exc:`BaseExceptionGroup` or :exc:`ExceptionGroup` exception. Since all tasks are descendents of the initial task, one consequence of this is that :func:`run` can't finish until all tasks have From 2dc1c641d7cef6155b7bd9da983ec59fe63f470e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 8 Feb 2022 11:30:22 +0200 Subject: [PATCH 0993/1498] Update newsfragments/2211.deprecated.rst Co-authored-by: Joshua Oreman --- newsfragments/2211.deprecated.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/2211.deprecated.rst b/newsfragments/2211.deprecated.rst index 6becff6b98..b7e8e85442 100644 --- a/newsfragments/2211.deprecated.rst +++ b/newsfragments/2211.deprecated.rst @@ -1,5 +1,5 @@ ``MultiError`` has been deprecated in favor of the standard :exc:`BaseExceptionGroup` -(introduced in :pep:`654`). On Python < 3.11, this exception and its derivative +(introduced in :pep:`654`). On Python versions below 3.11, this exception and its derivative :exc:`ExceptionGroup` are provided by the backport_. Trio still raises ``MultiError``, but it has been refactored into a subclass of :exc:`BaseExceptionGroup` which users should catch instead of ``MultiError``. Uses of the ``filter()`` class method should be From e7faa467783f6a0ddc9960b92aa45870d1b0785f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 8 Feb 2022 11:31:01 +0200 Subject: [PATCH 0994/1498] Update docs/source/tutorial/echo-server.py Co-authored-by: Joshua Oreman --- docs/source/tutorial/echo-server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/tutorial/echo-server.py b/docs/source/tutorial/echo-server.py index 3751cadd73..d37e509af4 100644 --- a/docs/source/tutorial/echo-server.py +++ b/docs/source/tutorial/echo-server.py @@ -11,7 +11,6 @@ CONNECTION_COUNTER = count() - async def echo_server(server_stream): # Assign each connection a unique number to make our debug prints easier # to understand when there are multiple simultaneous connections. From c41517af5b2fde0a0f7d69f27dd64cf08ab8f712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 8 Feb 2022 11:38:11 +0200 Subject: [PATCH 0995/1498] Update trio/_highlevel_open_tcp_stream.py --- trio/_highlevel_open_tcp_stream.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 382a5889c9..c57d421043 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -119,9 +119,7 @@ def close_all(): if len(errs) == 1: raise errs[0] elif errs: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - raise MultiError(errs) + raise MultiError(errs) def reorder_for_rfc_6555_section_5_4(targets): From 0ee78a52e3aa999f35a7fc36e86c0b08f631b6cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:19:57 +0000 Subject: [PATCH 0996/1498] Bump prompt-toolkit from 3.0.26 to 3.0.27 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.26 to 3.0.27. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.26...3.0.27) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fae1ed3e4d..3eb627ee65 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -81,7 +81,7 @@ platformdirs==2.4.1 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.27 # via ipython ptyprocess==0.7.0 # via pexpect From 50963c6cf9bf316b91f65989d8c6f4ad5da95c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 8 Feb 2022 13:03:40 +0200 Subject: [PATCH 0997/1498] Don't skip the entire test_coroutine_or_error test on py3.11 --- trio/tests/test_util.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index bf2f606f00..40743f0cfa 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -95,9 +95,6 @@ def not_main_thread(): # @coroutine is deprecated since python 3.8, which is fine with us. @pytest.mark.filterwarnings("ignore:.*@coroutine.*:DeprecationWarning") -@pytest.mark.skipif( - sys.version_info >= (3, 11), reason="asyncio.coroutine was removed in Python 3.11" -) def test_coroutine_or_error(): class Deferred: "Just kidding" @@ -113,9 +110,11 @@ async def f(): # pragma: no cover import asyncio - @asyncio.coroutine - def generator_based_coro(): # pragma: no cover - yield from asyncio.sleep(1) + if sys.version_info < (3, 11): + + @asyncio.coroutine + def generator_based_coro(): # pragma: no cover + yield from asyncio.sleep(1) with pytest.raises(TypeError) as excinfo: coroutine_or_error(generator_based_coro()) From ba17fbbfc9b396cd1524b17c9b617be7317e84dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 8 Feb 2022 13:05:20 +0200 Subject: [PATCH 0998/1498] Removed weird test code --- trio/_core/tests/test_run.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 2dfacc3a55..8fcf8c8b78 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1860,11 +1860,7 @@ async def __anext__(self): def handle(exc): nonlocal got_stop - if isinstance(exc, StopAsyncIteration): - got_stop = True - return None - else: # pragma: no cover - return exc + got_stop = True with catch({StopAsyncIteration: handle}): async with _core.open_nursery() as nursery: From 8911e0a59c2e66e2c4598cf60f0d9d8d3a85793d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 8 Feb 2022 13:49:44 +0200 Subject: [PATCH 0999/1498] Update newsfragments/2211.deprecated.rst --- newsfragments/2211.deprecated.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newsfragments/2211.deprecated.rst b/newsfragments/2211.deprecated.rst index b7e8e85442..ed7e9e1f29 100644 --- a/newsfragments/2211.deprecated.rst +++ b/newsfragments/2211.deprecated.rst @@ -4,8 +4,8 @@ but it has been refactored into a subclass of :exc:`BaseExceptionGroup` which users should catch instead of ``MultiError``. Uses of the ``filter()`` class method should be replaced with :meth:`BaseExceptionGroup.split`. Uses of the ``catch()`` class method -should be replaced with either ``except*`` clauses (on Python 3.11+) or the ``catch()`` -context manager provided by the backport_. +should be replaced with either ``except*`` clauses (on Python 3.11+) or the +``exceptiongroup.catch()`` context manager provided by the backport_. See the :ref:`updated documentation ` for details. From 8877822d8871d4ed95b8ae3c486c907b412bdcc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 10:22:27 +0000 Subject: [PATCH 1000/1498] Bump tomli from 2.0.0 to 2.0.1 Bumps [tomli](https://github.com/hukkin/tomli) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/hukkin/tomli/releases) - [Changelog](https://github.com/hukkin/tomli/blob/master/CHANGELOG.md) - [Commits](https://github.com/hukkin/tomli/compare/2.0.0...2.0.1) --- updated-dependencies: - dependency-name: tomli dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3eb627ee65..84cbe8d629 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -113,7 +113,7 @@ sortedcontainers==2.4.0 # via -r test-requirements.in toml==0.10.2 # via pylint -tomli==2.0.0 +tomli==2.0.1 # via # black # coverage From 3357f19d14929098ba07ef199dbf8197bce6bd2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Feb 2022 11:06:23 +0000 Subject: [PATCH 1001/1498] Bump platformdirs from 2.4.1 to 2.5.0 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.4.1 to 2.5.0. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.4.1...2.5.0) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 84cbe8d629..b57d1070a9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -75,7 +75,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 +platformdirs==2.5.0 # via # black # pylint From 2143af6d09032f8ab93a040f76f1f00d86910b88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Feb 2022 11:56:11 +0000 Subject: [PATCH 1002/1498] Bump prompt-toolkit from 3.0.27 to 3.0.28 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.27 to 3.0.28. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.27...3.0.28) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b57d1070a9..12983a29b9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -81,7 +81,7 @@ platformdirs==2.5.0 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.27 +prompt-toolkit==3.0.28 # via ipython ptyprocess==0.7.0 # via pexpect From 9478f4da2ee6bf3c11cd30e085e372386e9dfb4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Feb 2022 11:56:37 +0000 Subject: [PATCH 1003/1498] Bump towncrier from 21.3.0 to 21.9.0 Bumps [towncrier](https://github.com/hawkowl/towncrier) from 21.3.0 to 21.9.0. - [Release notes](https://github.com/hawkowl/towncrier/releases) - [Changelog](https://github.com/twisted/towncrier/blob/master/NEWS.rst) - [Commits](https://github.com/hawkowl/towncrier/compare/21.3.0...21.9.0) --- updated-dependencies: - dependency-name: towncrier dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c178c771a1..346df2c7e4 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -83,9 +83,9 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -toml==0.10.2 +tomli==2.0.1 # via towncrier -towncrier==21.3.0 +towncrier==21.9.0 # via -r docs-requirements.in urllib3==1.26.8 # via requests From 7c701546e8e47c706872da60245189e0ca326b27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:23:24 +0000 Subject: [PATCH 1004/1498] Bump typing-extensions from 4.0.1 to 4.1.1 Bumps [typing-extensions](https://github.com/python/typing) from 4.0.1 to 4.1.1. - [Release notes](https://github.com/python/typing/releases) - [Changelog](https://github.com/python/typing/blob/master/typing_extensions/CHANGELOG) - [Commits](https://github.com/python/typing/compare/4.0.1...4.1.1) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 12983a29b9..f1da8b4167 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -125,7 +125,7 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typing-extensions==4.0.1 ; implementation_name == "cpython" +typing-extensions==4.1.1 ; implementation_name == "cpython" # via # -r test-requirements.in # mypy From bd3180ce57174cecbd8ac79994d97be776fa5646 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:23:52 +0000 Subject: [PATCH 1005/1498] Bump pytest from 7.0.0 to 7.0.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.0.0...7.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 12983a29b9..60ecb4a4b7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -101,7 +101,7 @@ pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.7 # via packaging -pytest==7.0.0 +pytest==7.0.1 # via # -r test-requirements.in # pytest-cov From f6e0c8280b9920c5c3bd1f6e11e11615457ef0bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Feb 2022 10:17:29 +0000 Subject: [PATCH 1006/1498] Bump markupsafe from 2.0.1 to 2.1.0 Bumps [markupsafe](https://github.com/pallets/markupsafe) from 2.0.1 to 2.1.0. - [Release notes](https://github.com/pallets/markupsafe/releases) - [Changelog](https://github.com/pallets/markupsafe/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/markupsafe/compare/2.0.1...2.1.0) --- updated-dependencies: - dependency-name: markupsafe dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 346df2c7e4..1d1146a8e1 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -42,7 +42,7 @@ jinja2==3.0.3 # via # sphinx # towncrier -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in From 4e87c29d1dd60b517d4659216d5aff660684eead Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 10:20:02 +0000 Subject: [PATCH 1007/1498] Bump platformdirs from 2.5.0 to 2.5.1 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.5.0 to 2.5.1. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.5.0...2.5.1) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 12983a29b9..bf1e1bed37 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -75,7 +75,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.0 +platformdirs==2.5.1 # via # black # pylint From a0840a6415ad6bd0f47d38b4b95619c1e1051aea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 10:22:10 +0000 Subject: [PATCH 1008/1498] Bump click from 8.0.3 to 8.0.4 Bumps [click](https://github.com/pallets/click) from 8.0.3 to 8.0.4. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.0.3...8.0.4) --- updated-dependencies: - dependency-name: click dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 1d1146a8e1..947dfc66e6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.11 # via requests -click==8.0.3 +click==8.0.4 # via # click-default-group # towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 12983a29b9..b637cff284 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ black==22.1.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography -click==8.0.3 +click==8.0.4 # via black coverage[toml]==6.0.2 # via pytest-cov From d6fa9a5f4257ee3b83d3b4b84dc0db4b55aca9f5 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 15:26:03 +0400 Subject: [PATCH 1009/1498] Tickle sr.ht CI From 5ee3c59855f46909257baac716f5f39876f71875 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 11:26:23 +0000 Subject: [PATCH 1010/1498] Bump charset-normalizer from 2.0.11 to 2.0.12 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.11 to 2.0.12. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.11...2.0.12) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 947dfc66e6..058b5e0a49 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.10.8 # via requests -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests click==8.0.4 # via From d56a7ddbf66a640495081b40f67174e2e9c71605 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 15:26:59 +0400 Subject: [PATCH 1011/1498] Tickle sr.ht CI From 02311e767c409bd34e3431f0afa2cafa9138e879 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 15:46:39 +0400 Subject: [PATCH 1012/1498] Merge two competing newsfragments It turns out towncrier will ignore the removal and only consider the feature, which means we have to merge them. --- newsfragments/1104.feature.rst | 6 ++++++ newsfragments/1104.removal.rst | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 newsfragments/1104.removal.rst diff --git a/newsfragments/1104.feature.rst b/newsfragments/1104.feature.rst index b2d4ceaa11..0ca2021025 100644 --- a/newsfragments/1104.feature.rst +++ b/newsfragments/1104.feature.rst @@ -1,3 +1,9 @@ You can now conveniently spawn a child process in a background task and interact it with on the fly using ``process = await nursery.start(run_process, ...)``. See `run_process` for more details. +We recommend most users switch to this new API. Also note that: + +- ``trio.open_process`` has been deprecated in favor of + `trio.lowlevel.open_process`, +- The ``aclose`` method on `Process` has been deprecated along with + ``async with process_obj``. diff --git a/newsfragments/1104.removal.rst b/newsfragments/1104.removal.rst deleted file mode 100644 index f2a5a0a997..0000000000 --- a/newsfragments/1104.removal.rst +++ /dev/null @@ -1,5 +0,0 @@ -``trio.open_process`` has been renamed to -`trio.lowlevel.open_process`, and the ``aclose`` method on `Process` -has been deprecated, along with ``async with process_obj``. We -recommend most users switch to the new -``nursery.start(trio.run_process, ...)`` API instead. From 6947901ef0da402d83d967a01351df3df95e6a00 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 16:31:57 +0400 Subject: [PATCH 1013/1498] Bump version to 0.20.0 --- docs/source/history.rst | 39 ++++++++++++++++++++++++++++++++++ newsfragments/1104.feature.rst | 9 -------- newsfragments/2063.bugfix.rst | 3 --- newsfragments/2160.feature.rst | 6 ------ newsfragments/2193.bugfix.rst | 3 --- newsfragments/2203.bugfix.rst | 2 -- newsfragments/2209.bugfix.rst | 3 --- trio/_version.py | 2 +- 8 files changed, 40 insertions(+), 27 deletions(-) delete mode 100644 newsfragments/1104.feature.rst delete mode 100644 newsfragments/2063.bugfix.rst delete mode 100644 newsfragments/2160.feature.rst delete mode 100644 newsfragments/2193.bugfix.rst delete mode 100644 newsfragments/2203.bugfix.rst delete mode 100644 newsfragments/2209.bugfix.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 3936df867e..3fb51f346a 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,45 @@ Release history .. towncrier release notes start +Trio 0.20.0 (2022-02-21) +------------------------ + +Features +~~~~~~~~ + +- You can now conveniently spawn a child process in a background task + and interact it with on the fly using ``process = await + nursery.start(run_process, ...)``. See `run_process` for more details. + We recommend most users switch to this new API. Also note that: + + - ``trio.open_process`` has been deprecated in favor of + `trio.lowlevel.open_process`, + - The ``aclose`` method on `Process` has been deprecated along with + ``async with process_obj``. (`#1104 `__) +- Now context variables set with `contextvars` are preserved when running functions + in a worker thread with `trio.to_thread.run_sync`, or when running + functions from the worker thread in the parent Trio thread with + `trio.from_thread.run`, and `trio.from_thread.run_sync`. + This is done by automatically copying the `contextvars` context. + `trio.lowlevel.spawn_system_task` now also receives an optional ``context`` argument. (`#2160 `__) + + +Bugfixes +~~~~~~~~ + +- Trio now avoids creating cyclic garbage when a `MultiError` is generated and filtered, + including invisibly within the cancellation system. This means errors raised + through nurseries and cancel scopes should result in less GC latency. (`#2063 `__) +- Trio now deterministically cleans up file descriptors that were opened before + subprocess creation fails. Previously, they would remain open until the next run of + the garbage collector. (`#2193 `__) +- Add compatibility with OpenSSL 3.0 on newer Python and PyPy versions by working + around ``SSLEOFError`` not being raised properly. (`#2203 `__) +- Fix a bug that could cause `Process.wait` to hang on Linux systems using pidfds, if + another task were to access `Process.returncode` after the process exited but before + ``wait`` woke up (`#2209 `__) + + Trio 0.19.0 (2021-06-15) ------------------------ diff --git a/newsfragments/1104.feature.rst b/newsfragments/1104.feature.rst deleted file mode 100644 index 0ca2021025..0000000000 --- a/newsfragments/1104.feature.rst +++ /dev/null @@ -1,9 +0,0 @@ -You can now conveniently spawn a child process in a background task -and interact it with on the fly using ``process = await -nursery.start(run_process, ...)``. See `run_process` for more details. -We recommend most users switch to this new API. Also note that: - -- ``trio.open_process`` has been deprecated in favor of - `trio.lowlevel.open_process`, -- The ``aclose`` method on `Process` has been deprecated along with - ``async with process_obj``. diff --git a/newsfragments/2063.bugfix.rst b/newsfragments/2063.bugfix.rst deleted file mode 100644 index 85fe7aa52b..0000000000 --- a/newsfragments/2063.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Trio now avoids creating cyclic garbage when a `MultiError` is generated and filtered, -including invisibly within the cancellation system. This means errors raised -through nurseries and cancel scopes should result in less GC latency. \ No newline at end of file diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst deleted file mode 100644 index 00cb09a975..0000000000 --- a/newsfragments/2160.feature.rst +++ /dev/null @@ -1,6 +0,0 @@ -Now context variables set with `contextvars` are preserved when running functions -in a worker thread with `trio.to_thread.run_sync`, or when running -functions from the worker thread in the parent Trio thread with -`trio.from_thread.run`, and `trio.from_thread.run_sync`. -This is done by automatically copying the `contextvars` context. -`trio.lowlevel.spawn_system_task` now also receives an optional ``context`` argument. diff --git a/newsfragments/2193.bugfix.rst b/newsfragments/2193.bugfix.rst deleted file mode 100644 index 979dec111b..0000000000 --- a/newsfragments/2193.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Trio now deterministically cleans up file descriptors that were opened before -subprocess creation fails. Previously, they would remain open until the next run of -the garbage collector. diff --git a/newsfragments/2203.bugfix.rst b/newsfragments/2203.bugfix.rst deleted file mode 100644 index 55a01d737e..0000000000 --- a/newsfragments/2203.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add compatibility with OpenSSL 3.0 on newer Python and PyPy versions by working -around ``SSLEOFError`` not being raised properly. \ No newline at end of file diff --git a/newsfragments/2209.bugfix.rst b/newsfragments/2209.bugfix.rst deleted file mode 100644 index 8ea6094f15..0000000000 --- a/newsfragments/2209.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug that could cause `Process.wait` to hang on Linux systems using pidfds, if -another task were to access `Process.returncode` after the process exited but before -``wait`` woke up diff --git a/trio/_version.py b/trio/_version.py index c99aa04a7e..f1b741b96a 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.19.0+dev" +__version__ = "0.20.0" From a60054a6474adc1221949a8fce838fe7d89439cf Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 17:29:53 +0400 Subject: [PATCH 1014/1498] Bump version to 0.20.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index f1b741b96a..5a02c2e464 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.20.0" +__version__ = "0.20.0+dev" From dfd084cc3ffd8d4ca6009e70157dfad5cff1aa36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:19:57 +0000 Subject: [PATCH 1015/1498] Bump prompt-toolkit from 3.0.26 to 3.0.27 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.26 to 3.0.27. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.26...3.0.27) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index fa6243e7d0..19a6a1c9cf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -89,7 +89,7 @@ platformdirs==2.4.1 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.27 # via ipython ptyprocess==0.7.0 # via pexpect From b3c79c8e09647e1f194cb3f24dea6944bdb1e42d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 10:22:27 +0000 Subject: [PATCH 1016/1498] Bump tomli from 2.0.0 to 2.0.1 Bumps [tomli](https://github.com/hukkin/tomli) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/hukkin/tomli/releases) - [Changelog](https://github.com/hukkin/tomli/blob/master/CHANGELOG.md) - [Commits](https://github.com/hukkin/tomli/compare/2.0.0...2.0.1) --- updated-dependencies: - dependency-name: tomli dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 19a6a1c9cf..943e880243 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -121,7 +121,7 @@ sortedcontainers==2.4.0 # via -r test-requirements.in toml==0.10.2 # via pylint -tomli==2.0.0 +tomli==2.0.1 # via # black # coverage From 81feff2f26de031d1fde906ba6198c4125388b13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Feb 2022 11:06:23 +0000 Subject: [PATCH 1017/1498] Bump platformdirs from 2.4.1 to 2.5.0 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.4.1 to 2.5.0. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.4.1...2.5.0) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 943e880243..a500b4d5ba 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -83,7 +83,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 +platformdirs==2.5.0 # via # black # pylint From bcb442b949ffc069d34358e8363005fee0f04063 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Feb 2022 11:56:37 +0000 Subject: [PATCH 1018/1498] Bump towncrier from 21.3.0 to 21.9.0 Bumps [towncrier](https://github.com/hawkowl/towncrier) from 21.3.0 to 21.9.0. - [Release notes](https://github.com/hawkowl/towncrier/releases) - [Changelog](https://github.com/twisted/towncrier/blob/master/NEWS.rst) - [Commits](https://github.com/hawkowl/towncrier/compare/21.3.0...21.9.0) --- updated-dependencies: - dependency-name: towncrier dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 57c3106ad7..38cd45cd1c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -87,9 +87,9 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -toml==0.10.2 +tomli==2.0.1 # via towncrier -towncrier==21.3.0 +towncrier==21.9.0 # via -r docs-requirements.in typing-extensions==4.0.1 # via From ab78e027c988b147e048f4f41b397d2999a69edc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Feb 2022 11:56:11 +0000 Subject: [PATCH 1019/1498] Bump prompt-toolkit from 3.0.27 to 3.0.28 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.27 to 3.0.28. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.27...3.0.28) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a500b4d5ba..04b2bb28c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -89,7 +89,7 @@ platformdirs==2.5.0 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.27 +prompt-toolkit==3.0.28 # via ipython ptyprocess==0.7.0 # via pexpect From 9abddfad23eea51ed26fa3c6c23b51d706fd761f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Feb 2022 10:17:29 +0000 Subject: [PATCH 1020/1498] Bump markupsafe from 2.0.1 to 2.1.0 Bumps [markupsafe](https://github.com/pallets/markupsafe) from 2.0.1 to 2.1.0. - [Release notes](https://github.com/pallets/markupsafe/releases) - [Changelog](https://github.com/pallets/markupsafe/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/markupsafe/compare/2.0.1...2.1.0) --- updated-dependencies: - dependency-name: markupsafe dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 38cd45cd1c..c999fbf67f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -46,7 +46,7 @@ jinja2==3.0.3 # via # sphinx # towncrier -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in From efc8b0158b8becb2e2482aa6cd4b41d4cc9588c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 10:22:10 +0000 Subject: [PATCH 1021/1498] Bump click from 8.0.3 to 8.0.4 Bumps [click](https://github.com/pallets/click) from 8.0.3 to 8.0.4. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.0.3...8.0.4) --- updated-dependencies: - dependency-name: click dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c999fbf67f..6103f81d2f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.11 # via requests -click==8.0.3 +click==8.0.4 # via # click-default-group # towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 04b2bb28c7..d9953a6226 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ black==22.1.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography -click==8.0.3 +click==8.0.4 # via black coverage[toml]==6.3 # via pytest-cov From 9a4f5123c759cc4288e9ac9839c7eeff3c4384b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 10:20:02 +0000 Subject: [PATCH 1022/1498] Bump platformdirs from 2.5.0 to 2.5.1 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.5.0 to 2.5.1. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.5.0...2.5.1) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d9953a6226..b463eb5269 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -83,7 +83,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.0 +platformdirs==2.5.1 # via # black # pylint From 6cb5ad97be463799145fa385e56e8e22a7cf8bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 9 Mar 2022 21:21:15 +0200 Subject: [PATCH 1023/1498] Merged test requirements --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b463eb5269..994a3aea93 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -139,7 +139,7 @@ typed-ast==1.5.2 ; implementation_name == "cpython" and python_version < "3.8" # astroid # black # mypy -typing-extensions==4.0.1 ; implementation_name == "cpython" +typing-extensions==4.1.1 ; implementation_name == "cpython" # via # -r test-requirements.in # astroid From ee245a049bce4999ae7ef0c4d064e218790472fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:23:52 +0000 Subject: [PATCH 1024/1498] Bump pytest from 7.0.0 to 7.0.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.0.0 to 7.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.0.0...7.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 994a3aea93..a3c556fcba 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -109,7 +109,7 @@ pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.7 # via packaging -pytest==7.0.0 +pytest==7.0.1 # via # -r test-requirements.in # pytest-cov From 1e36f8c8c7a3226a73cb4852fb2a2c6a444cd39c Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 15:26:03 +0400 Subject: [PATCH 1025/1498] Tickle sr.ht CI From 034bd326a829e19912e05bf04edbf428510c15b4 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 15:46:39 +0400 Subject: [PATCH 1026/1498] Merge two competing newsfragments It turns out towncrier will ignore the removal and only consider the feature, which means we have to merge them. --- newsfragments/1104.feature.rst | 6 ++++++ newsfragments/1104.removal.rst | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 newsfragments/1104.removal.rst diff --git a/newsfragments/1104.feature.rst b/newsfragments/1104.feature.rst index b2d4ceaa11..0ca2021025 100644 --- a/newsfragments/1104.feature.rst +++ b/newsfragments/1104.feature.rst @@ -1,3 +1,9 @@ You can now conveniently spawn a child process in a background task and interact it with on the fly using ``process = await nursery.start(run_process, ...)``. See `run_process` for more details. +We recommend most users switch to this new API. Also note that: + +- ``trio.open_process`` has been deprecated in favor of + `trio.lowlevel.open_process`, +- The ``aclose`` method on `Process` has been deprecated along with + ``async with process_obj``. diff --git a/newsfragments/1104.removal.rst b/newsfragments/1104.removal.rst deleted file mode 100644 index f2a5a0a997..0000000000 --- a/newsfragments/1104.removal.rst +++ /dev/null @@ -1,5 +0,0 @@ -``trio.open_process`` has been renamed to -`trio.lowlevel.open_process`, and the ``aclose`` method on `Process` -has been deprecated, along with ``async with process_obj``. We -recommend most users switch to the new -``nursery.start(trio.run_process, ...)`` API instead. From 80a751f996540b5ba05118cbf771dd3b0d435f7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 11:26:23 +0000 Subject: [PATCH 1027/1498] Bump charset-normalizer from 2.0.11 to 2.0.12 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.11 to 2.0.12. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.11...2.0.12) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 6103f81d2f..fa9bca2fc6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.9.1 # via sphinx certifi==2021.10.8 # via requests -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests click==8.0.4 # via From 9eb2d1c046511b6a2729174c18f865266c3cb1b3 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 15:26:59 +0400 Subject: [PATCH 1028/1498] Tickle sr.ht CI From fd56492958c34008f0dc55894660dfd2258bc2ff Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 16:31:57 +0400 Subject: [PATCH 1029/1498] Bump version to 0.20.0 --- docs/source/history.rst | 39 ++++++++++++++++++++++++++++++++++ newsfragments/1104.feature.rst | 9 -------- newsfragments/2160.feature.rst | 6 ------ newsfragments/2193.bugfix.rst | 3 --- newsfragments/2203.bugfix.rst | 2 -- newsfragments/2209.bugfix.rst | 3 --- trio/_version.py | 2 +- 7 files changed, 40 insertions(+), 24 deletions(-) delete mode 100644 newsfragments/1104.feature.rst delete mode 100644 newsfragments/2160.feature.rst delete mode 100644 newsfragments/2193.bugfix.rst delete mode 100644 newsfragments/2203.bugfix.rst delete mode 100644 newsfragments/2209.bugfix.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 450817bb77..e2576fa64d 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,45 @@ Release history .. towncrier release notes start +Trio 0.20.0 (2022-02-21) +------------------------ + +Features +~~~~~~~~ + +- You can now conveniently spawn a child process in a background task + and interact it with on the fly using ``process = await + nursery.start(run_process, ...)``. See `run_process` for more details. + We recommend most users switch to this new API. Also note that: + + - ``trio.open_process`` has been deprecated in favor of + `trio.lowlevel.open_process`, + - The ``aclose`` method on `Process` has been deprecated along with + ``async with process_obj``. (`#1104 `__) +- Now context variables set with `contextvars` are preserved when running functions + in a worker thread with `trio.to_thread.run_sync`, or when running + functions from the worker thread in the parent Trio thread with + `trio.from_thread.run`, and `trio.from_thread.run_sync`. + This is done by automatically copying the `contextvars` context. + `trio.lowlevel.spawn_system_task` now also receives an optional ``context`` argument. (`#2160 `__) + + +Bugfixes +~~~~~~~~ + +- Trio now avoids creating cyclic garbage when a `MultiError` is generated and filtered, + including invisibly within the cancellation system. This means errors raised + through nurseries and cancel scopes should result in less GC latency. (`#2063 `__) +- Trio now deterministically cleans up file descriptors that were opened before + subprocess creation fails. Previously, they would remain open until the next run of + the garbage collector. (`#2193 `__) +- Add compatibility with OpenSSL 3.0 on newer Python and PyPy versions by working + around ``SSLEOFError`` not being raised properly. (`#2203 `__) +- Fix a bug that could cause `Process.wait` to hang on Linux systems using pidfds, if + another task were to access `Process.returncode` after the process exited but before + ``wait`` woke up (`#2209 `__) + + Trio 0.19.0 (2021-06-15) ------------------------ diff --git a/newsfragments/1104.feature.rst b/newsfragments/1104.feature.rst deleted file mode 100644 index 0ca2021025..0000000000 --- a/newsfragments/1104.feature.rst +++ /dev/null @@ -1,9 +0,0 @@ -You can now conveniently spawn a child process in a background task -and interact it with on the fly using ``process = await -nursery.start(run_process, ...)``. See `run_process` for more details. -We recommend most users switch to this new API. Also note that: - -- ``trio.open_process`` has been deprecated in favor of - `trio.lowlevel.open_process`, -- The ``aclose`` method on `Process` has been deprecated along with - ``async with process_obj``. diff --git a/newsfragments/2160.feature.rst b/newsfragments/2160.feature.rst deleted file mode 100644 index 00cb09a975..0000000000 --- a/newsfragments/2160.feature.rst +++ /dev/null @@ -1,6 +0,0 @@ -Now context variables set with `contextvars` are preserved when running functions -in a worker thread with `trio.to_thread.run_sync`, or when running -functions from the worker thread in the parent Trio thread with -`trio.from_thread.run`, and `trio.from_thread.run_sync`. -This is done by automatically copying the `contextvars` context. -`trio.lowlevel.spawn_system_task` now also receives an optional ``context`` argument. diff --git a/newsfragments/2193.bugfix.rst b/newsfragments/2193.bugfix.rst deleted file mode 100644 index 979dec111b..0000000000 --- a/newsfragments/2193.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Trio now deterministically cleans up file descriptors that were opened before -subprocess creation fails. Previously, they would remain open until the next run of -the garbage collector. diff --git a/newsfragments/2203.bugfix.rst b/newsfragments/2203.bugfix.rst deleted file mode 100644 index 55a01d737e..0000000000 --- a/newsfragments/2203.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add compatibility with OpenSSL 3.0 on newer Python and PyPy versions by working -around ``SSLEOFError`` not being raised properly. \ No newline at end of file diff --git a/newsfragments/2209.bugfix.rst b/newsfragments/2209.bugfix.rst deleted file mode 100644 index 8ea6094f15..0000000000 --- a/newsfragments/2209.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug that could cause `Process.wait` to hang on Linux systems using pidfds, if -another task were to access `Process.returncode` after the process exited but before -``wait`` woke up diff --git a/trio/_version.py b/trio/_version.py index c99aa04a7e..f1b741b96a 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.19.0+dev" +__version__ = "0.20.0" From af6233b86a3073968cf786212e294ebe9f8e0349 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 21 Feb 2022 17:29:53 +0400 Subject: [PATCH 1030/1498] Bump version to 0.20.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index f1b741b96a..5a02c2e464 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.20.0" +__version__ = "0.20.0+dev" From a4876bc5b09fdd2b468da4efd0580fd37f57f186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 9 Mar 2022 21:19:07 +0200 Subject: [PATCH 1031/1498] Renamed the news fragment --- newsfragments/{2211.deprecated.rst => 2211.headline.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{2211.deprecated.rst => 2211.headline.rst} (100%) diff --git a/newsfragments/2211.deprecated.rst b/newsfragments/2211.headline.rst similarity index 100% rename from newsfragments/2211.deprecated.rst rename to newsfragments/2211.headline.rst From 5c09f4470f45d0080f2eee4391ffeb5e9a10fc30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 9 Mar 2022 21:29:28 +0200 Subject: [PATCH 1032/1498] Updated wrapt in test requirements --- test-requirements.in | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 3c5079838b..35a87028e4 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -9,7 +9,7 @@ pylint # for pylint finding all symbols tests jedi # for jedi code completion tests cryptography>=36.0.0 # 35.0.0 is transitive but fails exceptiongroup # for catch() -wrapt @ git+https://github.com/grayjk/wrapt.git@issue-196 # fixes Python 3.11 compat for pylint +wrapt @ git+https://github.com/GrahamDumpleton/wrapt.git@8f180bf981fc7a92094cfecfd7a9e5f591d4bd4b # fixes Python 3.11 compat for pylint # Tools black; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index a3c556fcba..69b4896d72 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -149,7 +149,7 @@ typing-extensions==4.1.1 ; implementation_name == "cpython" # pylint wcwidth==0.2.5 # via prompt-toolkit -wrapt @ git+https://github.com/grayjk/wrapt.git@issue-196 +wrapt @ git+https://github.com/GrahamDumpleton/wrapt.git@8f180bf981fc7a92094cfecfd7a9e5f591d4bd4b # via # -r test-requirements.in # astroid From c5af0a4f193adf7643de5ef50a07e6b82775c483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 9 Mar 2022 21:34:20 +0200 Subject: [PATCH 1033/1498] Changed wrapt target git hash so it's compatible with astroid --- test-requirements.in | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index 35a87028e4..fd0d482378 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -9,7 +9,7 @@ pylint # for pylint finding all symbols tests jedi # for jedi code completion tests cryptography>=36.0.0 # 35.0.0 is transitive but fails exceptiongroup # for catch() -wrapt @ git+https://github.com/GrahamDumpleton/wrapt.git@8f180bf981fc7a92094cfecfd7a9e5f591d4bd4b # fixes Python 3.11 compat for pylint +wrapt @ git+https://github.com/GrahamDumpleton/wrapt.git@a1a06529f41cffcc9fd5132fcf1923ea538f3d3c # fixes Python 3.11 compat for pylint # Tools black; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index 69b4896d72..5653f6b293 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -149,7 +149,7 @@ typing-extensions==4.1.1 ; implementation_name == "cpython" # pylint wcwidth==0.2.5 # via prompt-toolkit -wrapt @ git+https://github.com/GrahamDumpleton/wrapt.git@8f180bf981fc7a92094cfecfd7a9e5f591d4bd4b +wrapt @ git+https://github.com/GrahamDumpleton/wrapt.git@a1a06529f41cffcc9fd5132fcf1923ea538f3d3c # via # -r test-requirements.in # astroid From fb581b98b179fd9bd48a1e54b015e694ab289001 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Mar 2022 10:22:43 +0000 Subject: [PATCH 1034/1498] Bump mypy from 0.931 to 0.941 Bumps [mypy](https://github.com/python/mypy) from 0.931 to 0.941. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.931...v0.941) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ae9670bbad..4b4f11160c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -56,7 +56,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.931 ; implementation_name == "cpython" +mypy==0.941 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From 7d9fe3d1aa165f19221fb623a2b31f0ec7ca07a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Mar 2022 10:19:31 +0000 Subject: [PATCH 1035/1498] Bump cryptography from 36.0.1 to 36.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 36.0.1 to 36.0.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/36.0.1...36.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ae9670bbad..9b159932d5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.0.4 # via black coverage[toml]==6.0.2 # via pytest-cov -cryptography==36.0.1 +cryptography==36.0.2 # via # -r test-requirements.in # pyopenssl From da7ed4ac054f86a6801a1f6de93f4576c05b4ce8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Mar 2022 10:21:00 +0000 Subject: [PATCH 1036/1498] Bump markupsafe from 2.1.0 to 2.1.1 Bumps [markupsafe](https://github.com/pallets/markupsafe) from 2.1.0 to 2.1.1. - [Release notes](https://github.com/pallets/markupsafe/releases) - [Changelog](https://github.com/pallets/markupsafe/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/markupsafe/compare/2.1.0...2.1.1) --- updated-dependencies: - dependency-name: markupsafe dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 058b5e0a49..9775816229 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -42,7 +42,7 @@ jinja2==3.0.3 # via # sphinx # towncrier -markupsafe==2.1.0 +markupsafe==2.1.1 # via jinja2 outcome==1.1.0 # via -r docs-requirements.in From feb9cbc515f8291c32183c78551664e9c96053b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Mar 2022 10:22:54 +0000 Subject: [PATCH 1037/1498] Bump urllib3 from 1.26.8 to 1.26.9 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.8 to 1.26.9. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.9/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.8...1.26.9) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 058b5e0a49..5816e694dc 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -87,5 +87,5 @@ tomli==2.0.1 # via towncrier towncrier==21.9.0 # via -r docs-requirements.in -urllib3==1.26.8 +urllib3==1.26.9 # via requests From 6d37564747e15dfb839566916d71d1a9e24e304c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 10:51:24 +0000 Subject: [PATCH 1038/1498] Bump pytz from 2021.3 to 2022.1 Bumps [pytz](https://github.com/stub42/pytz) from 2021.3 to 2022.1. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2021.3...release_2022.1) --- updated-dependencies: - dependency-name: pytz dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 058b5e0a49..af70ebf1de 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -52,7 +52,7 @@ pygments==2.10.0 # via sphinx pyparsing==3.0.7 # via packaging -pytz==2021.3 +pytz==2022.1 # via babel requests==2.27.1 # via sphinx From 06faa998d5d316034da4802e72f352268efd3d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 24 Mar 2022 13:28:43 +0100 Subject: [PATCH 1039/1498] :green_heart: Make Fedora build use Python 3.10 cffi manylinux wheels for 3.10 are available Reverts 27612eaf1517282909701cea1847d021ae76ccc4 --- .builds/fedora.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.builds/fedora.yml b/.builds/fedora.yml index e30f7706ce..eddb2368a0 100644 --- a/.builds/fedora.yml +++ b/.builds/fedora.yml @@ -1,14 +1,12 @@ image: fedora/rawhide packages: - # Comment out to avoid Python 3.10 until cffi has built wheels for Python 3.10 - # - python3-devel - - python3.9 + - python3-devel - python3-pip sources: - https://github.com/python-trio/trio tasks: - test: | - python3.9 -m venv venv + python3 -m venv venv source venv/bin/activate cd trio CI_BUILD_ID=$JOB_ID CI_BUILD_URL=$JOB_URL ./ci.sh From 033d8741889bda37168525259cf7bbf66aaef94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 24 Mar 2022 13:33:27 +0100 Subject: [PATCH 1040/1498] Python 3.11: traceback_exception_init() has new keyword arguments This avoids: ============================= test session starts ============================== platform linux -- Python 3.11.0a6, pytest-7.1.1, pluggy-1.0.0 rootdir: .../trio, configfile: pyproject.toml collected 213 items trio/_core/tests/test_asyncgen.py . INTERNALERROR> Traceback (most recent call last): INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/main.py", line 268, in wrap_session INTERNALERROR> session.exitstatus = doit(config, session) or 0 INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/main.py", line 322, in _main INTERNALERROR> config.hook.pytest_runtestloop(session=session) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_hooks.py", line 265, in __call__ INTERNALERROR> return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_manager.py", line 80, in _hookexec INTERNALERROR> return self._inner_hookexec(hook_name, methods, kwargs, firstresult) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_callers.py", line 60, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_result.py", line 60, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_callers.py", line 39, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/main.py", line 347, in pytest_runtestloop INTERNALERROR> item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_hooks.py", line 265, in __call__ INTERNALERROR> return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_manager.py", line 80, in _hookexec INTERNALERROR> return self._inner_hookexec(hook_name, methods, kwargs, firstresult) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_callers.py", line 60, in _multicall INTERNALERROR> return outcome.get_result() INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_result.py", line 60, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_callers.py", line 39, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/runner.py", line 111, in pytest_runtest_protocol INTERNALERROR> runtestprotocol(item, nextitem=nextitem) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/runner.py", line 130, in runtestprotocol INTERNALERROR> reports.append(call_and_report(item, "call", log)) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/runner.py", line 221, in call_and_report INTERNALERROR> report: TestReport = hook.pytest_runtest_makereport(item=item, call=call) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_hooks.py", line 265, in __call__ INTERNALERROR> return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_manager.py", line 80, in _hookexec INTERNALERROR> return self._inner_hookexec(hook_name, methods, kwargs, firstresult) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_callers.py", line 55, in _multicall INTERNALERROR> gen.send(outcome) INTERNALERROR> ^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/skipping.py", line 265, in pytest_runtest_makereport INTERNALERROR> rep = outcome.get_result() INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_result.py", line 60, in get_result INTERNALERROR> raise ex[1].with_traceback(ex[2]) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/pluggy/_callers.py", line 39, in _multicall INTERNALERROR> res = hook_impl.function(*args) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/runner.py", line 365, in pytest_runtest_makereport INTERNALERROR> return TestReport.from_item_and_call(item, call) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/reports.py", line 345, in from_item_and_call INTERNALERROR> longrepr = item.repr_failure(excinfo) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/python.py", line 1795, in repr_failure INTERNALERROR> return self._repr_failure_py(excinfo, style=style) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/nodes.py", line 475, in _repr_failure_py INTERNALERROR> return excinfo.getrepr( INTERNALERROR> ^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/_code/code.py", line 666, in getrepr INTERNALERROR> return fmt.repr_excinfo(self) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/_code/code.py", line 926, in repr_excinfo INTERNALERROR> reprtraceback = self.repr_traceback(excinfo_) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/_code/code.py", line 867, in repr_traceback INTERNALERROR> reprentry = self.repr_traceback_entry(entry, einfo) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/_code/code.py", line 818, in repr_traceback_entry INTERNALERROR> s = self.get_source(source, line_index, excinfo, short=short) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/_code/code.py", line 756, in get_source INTERNALERROR> lines.extend(self.get_exconly(excinfo, indent=indent, markall=True)) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/_code/code.py", line 768, in get_exconly INTERNALERROR> exlines = excinfo.exconly(tryshort=True).split("\n") INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/__venv__/lib64/python3.11/site-packages/_pytest/_code/code.py", line 585, in exconly INTERNALERROR> lines = format_exception_only(self.type, self.value) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File "/usr/lib64/python3.11/traceback.py", line 159, in format_exception_only INTERNALERROR> te = TracebackException(type(value), value, None, compact=True) INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File ".../trio/trio/_core/_multierror.py", line 393, in traceback_exception_init INTERNALERROR> traceback_exception_original_init( INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ INTERNALERROR> File "/usr/lib64/python3.11/traceback.py", line 718, in __init__ INTERNALERROR> cause = TracebackException( INTERNALERROR> ^^^^^^^^^^^^^^^^^^^ INTERNALERROR> TypeError: traceback_exception_init() got an unexpected keyword argument 'max_group_width' --- trio/_core/_multierror.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 6dfdaa7a52..95399e5366 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -383,11 +383,10 @@ def traceback_exception_init( capture_locals=False, compact=False, _seen=None, + **kwargs ): if sys.version_info >= (3, 10): - kwargs = {"compact": compact} - else: - kwargs = {} + kwargs["compact"] = compact # Capture the original exception and its cause and context as TracebackExceptions traceback_exception_original_init( From 363b719fb282e48ca1002b15ae32aa408b429c73 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 24 Mar 2022 18:15:34 +0400 Subject: [PATCH 1041/1498] Add IPPROTO_MPTCP Needed on Fedora Rawhide on Python 3.10. --- trio/socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/socket.py b/trio/socket.py index 613375ef41..416a2ba9b9 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -38,7 +38,7 @@ IPPROTO_ND, IPPROTO_TP, IPPROTO_ROUTING, IPPROTO_FRAGMENT, IPPROTO_RSVP, IPPROTO_GRE, IPPROTO_ESP, IPPROTO_AH, IPPROTO_ICMPV6, IPPROTO_NONE, IPPROTO_DSTOPTS, IPPROTO_XTP, IPPROTO_EON, IPPROTO_PIM, - IPPROTO_IPCOMP, IPPROTO_SCTP, IPPROTO_RAW, IPPROTO_MAX, + IPPROTO_IPCOMP, IPPROTO_SCTP, IPPROTO_RAW, IPPROTO_MAX, IPPROTO_MPTCP, SYSPROTO_CONTROL, IPPORT_RESERVED, IPPORT_USERRESERVED, INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_UNSPEC_GROUP, INADDR_ALLHOSTS_GROUP, INADDR_MAX_LOCAL_GROUP, INADDR_NONE, IP_OPTIONS, From c1ed76a4dd0fafd282e30b23b94d118226803b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 24 Mar 2022 16:02:45 +0100 Subject: [PATCH 1042/1498] fixup! Python 3.11: traceback_exception_init() has new keyword arguments --- trio/_core/_multierror.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 95399e5366..78c828ebe2 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -420,6 +420,7 @@ def traceback_exception_init( # copy the set of _seen exceptions so that duplicates # shared between sub-exceptions are not omitted _seen=None if seen_was_none else set(_seen), + **kwargs ) ) self.embedded = embedded From 69496a91258ffcbd11901a61624301aa48c9d17e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Mar 2022 10:22:27 +0000 Subject: [PATCH 1043/1498] Bump pylint from 2.12.2 to 2.13.0 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.12.2 to 2.13.0. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.12.2...v2.13.0) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1b409c6625..f51abf97ab 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.9.3 +astroid==2.11.1 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -32,6 +32,8 @@ cryptography==36.0.2 # trustme decorator==5.1.1 # via ipython +dill==0.3.4 + # via pylint flake8==4.0.1 # via -r test-requirements.in idna==3.3 @@ -95,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.12.2 +pylint==2.13.0 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in @@ -111,13 +113,12 @@ sniffio==1.2.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -toml==0.10.2 - # via pylint tomli==2.0.1 # via # black # coverage # mypy + # pylint # pytest traitlets==5.1.1 # via From 97e520b89f3a0ea0f49e1ad340282f4e0beca8a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Mar 2022 06:59:44 +0000 Subject: [PATCH 1044/1498] Bump mypy from 0.941 to 0.942 Bumps [mypy](https://github.com/python/mypy) from 0.941 to 0.942. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.941...v0.942) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f51abf97ab..0e229d2554 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -58,7 +58,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.941 ; implementation_name == "cpython" +mypy==0.942 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From 37a1bcfd1d2b84d6a81167714ac77d43a01c4fa0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Mar 2022 06:59:59 +0000 Subject: [PATCH 1045/1498] Bump pytest from 7.0.1 to 7.1.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.0.1 to 7.1.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.0.1...7.1.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f51abf97ab..ad52a78249 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -103,7 +103,7 @@ pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.7 # via packaging -pytest==7.0.1 +pytest==7.1.1 # via # -r test-requirements.in # pytest-cov From bd773866515a5634088269038f72fe44769b8bb2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 10:23:42 +0000 Subject: [PATCH 1046/1498] Bump wrapt from 1.13.3 to 1.14.0 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.13.3 to 1.14.0. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.13.3...1.14.0) --- updated-dependencies: - dependency-name: wrapt dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6d32a3b52b..6b56fda88c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -132,7 +132,7 @@ typing-extensions==4.1.1 ; implementation_name == "cpython" # mypy wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.13.3 +wrapt==1.14.0 # via astroid # The following packages are considered to be unsafe in a requirements file: From 8912d1d9e6df3fdf2376c0fd03b8bc3e511fb670 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 10:26:30 +0000 Subject: [PATCH 1047/1498] Bump immutables from 0.16 to 0.17 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.16 to 0.17. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.16...v0.17) --- updated-dependencies: - dependency-name: immutables dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 8fc9f28d8d..2a98edb1ff 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -34,7 +34,7 @@ idna==3.3 # requests imagesize==1.3.0 # via sphinx -immutables==0.16 +immutables==0.17 # via -r docs-requirements.in incremental==21.3.0 # via towncrier From 7f442974dd363264b411cb64ede49200ce56a254 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 10:27:22 +0000 Subject: [PATCH 1048/1498] Bump pylint from 2.13.0 to 2.13.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.13.0 to 2.13.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.13.0...v2.13.2) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6d32a3b52b..0f43b14b5a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.11.1 +astroid==2.11.2 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.13.0 +pylint==2.13.2 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From 5c4b6942720967306dd6ef0d8c70ef69659dd9e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 31 Mar 2022 10:26:36 +0000 Subject: [PATCH 1049/1498] Bump pylint from 2.13.2 to 2.13.4 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.13.2 to 2.13.4. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.13.2...v2.13.4) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1df6ddabf8..1b2d5dec46 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.13.2 +pylint==2.13.4 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From 529bfc9aa2b2b3285f820dd11aed43e1dd6e760d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Apr 2022 00:48:41 +0000 Subject: [PATCH 1050/1498] Bump black from 22.1.0 to 22.3.0 Bumps [black](https://github.com/psf/black) from 22.1.0 to 22.3.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.1.0...22.3.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1b2d5dec46..137e20ea88 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==21.4.0 # pytest backcall==0.2.0 # via ipython -black==22.1.0 ; implementation_name == "cpython" +black==22.3.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography From bc7c10375083e8f6fff5253bb56ec9b684f4beef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Apr 2022 05:15:33 +0000 Subject: [PATCH 1051/1498] Bump click from 8.0.4 to 8.1.2 Bumps [click](https://github.com/pallets/click) from 8.0.4 to 8.1.2. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.0.4...8.1.2) --- updated-dependencies: - dependency-name: click dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 752d94b772..ec5bb072ef 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.12 # via requests -click==8.0.4 +click==8.1.2 # via # click-default-group # towncrier diff --git a/test-requirements.txt b/test-requirements.txt index 137e20ea88..f4a687874c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ black==22.3.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography -click==8.0.4 +click==8.1.2 # via black coverage[toml]==6.0.2 # via pytest-cov From 39ea8ed8118665034b2c819ce2a50a2421fb8760 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Apr 2022 10:21:06 +0000 Subject: [PATCH 1052/1498] Bump prompt-toolkit from 3.0.28 to 3.0.29 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.28 to 3.0.29. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.28...3.0.29) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f4a687874c..b238b03b86 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -83,7 +83,7 @@ platformdirs==2.5.1 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.28 +prompt-toolkit==3.0.29 # via ipython ptyprocess==0.7.0 # via pexpect From d40638c40be63389541f9d748076b14cfa5193d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 10:25:32 +0000 Subject: [PATCH 1053/1498] Bump pyparsing from 3.0.7 to 3.0.8 Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 3.0.7 to 3.0.8. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_3.0.7...pyparsing_3.0.8) --- updated-dependencies: - dependency-name: pyparsing dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index ec5bb072ef..45c6dcc42f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -50,7 +50,7 @@ packaging==21.3 # via sphinx pygments==2.10.0 # via sphinx -pyparsing==3.0.7 +pyparsing==3.0.8 # via packaging pytz==2022.1 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index b238b03b86..6669b22c8e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -101,7 +101,7 @@ pylint==2.13.4 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in -pyparsing==3.0.7 +pyparsing==3.0.8 # via packaging pytest==7.1.1 # via From 90cfa808468c32517643dab7e3410d852e749107 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Wed, 13 Apr 2022 09:03:45 +0900 Subject: [PATCH 1054/1498] unawesome unmaintained libraries * purerpc - not maintained, depends on ancient anyio version * trio-jsonrpc - not maintained --- docs/source/awesome-trio-libraries.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 37bf4c08bb..50f78d528e 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -83,12 +83,6 @@ Stream Processing * `Slurry `__ - Slurry is a microframework for building reactive, data processing applications with Trio. -RPC ---- -* `purepc `__ - Asynchronous pure Python gRPC client and server implementation using anyio. -* `trio-jsonrpc `__ - JSON-RPC v2.0 for Trio. - - Testing ------- * `pytest-trio `__ - Pytest plugin for trio. From e123113640ba961cecfe222363653114fb7d05e4 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Mon, 18 Apr 2022 08:11:17 +0900 Subject: [PATCH 1055/1498] purerpc is back! (#2291) --- docs/source/awesome-trio-libraries.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/awesome-trio-libraries.rst b/docs/source/awesome-trio-libraries.rst index 50f78d528e..50b3d698a3 100644 --- a/docs/source/awesome-trio-libraries.rst +++ b/docs/source/awesome-trio-libraries.rst @@ -83,6 +83,11 @@ Stream Processing * `Slurry `__ - Slurry is a microframework for building reactive, data processing applications with Trio. +RPC +--- +* `purepc `__ - Native, async Python gRPC client and server implementation using anyio. + + Testing ------- * `pytest-trio `__ - Pytest plugin for trio. From 6a5f2c095853156b01b497d42292b7fc9cface5a Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Mon, 18 Apr 2022 08:45:59 +0900 Subject: [PATCH 1056/1498] CI: enable pip caching (#2292) --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65fa899179..7611322c32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,8 @@ jobs: # PyPy -> pypy-3.7 python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} architecture: '${{ matrix.arch }}' + cache: pip + cache-dependency-path: test-requirements.txt - name: Run tests run: ./ci.sh shell: bash @@ -94,11 +96,15 @@ jobs: if: "!endsWith(matrix.python, '-dev')" with: python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} + cache: pip + cache-dependency-path: test-requirements.txt - name: Setup python (dev) uses: deadsnakes/action@v2.0.2 if: endsWith(matrix.python, '-dev') with: python-version: '${{ matrix.python }}' + cache: pip + cache-dependency-path: test-requirements.txt - name: Run tests run: ./ci.sh env: @@ -127,6 +133,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ fromJSON(format('["{0}", "{1}"]', format('{0}.0-alpha - {0}.X', matrix.python), matrix.python))[startsWith(matrix.python, 'pypy')] }} + cache: pip + cache-dependency-path: test-requirements.txt - name: Run tests run: ./ci.sh env: From 0fdeb1116cc10b4a831ac04f0f8f87cd1b29a898 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Mon, 18 Apr 2022 09:27:22 +0900 Subject: [PATCH 1057/1498] CI: deadsnakes/action does not support caching (#2293) --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7611322c32..ece2b0d0a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,8 +103,6 @@ jobs: if: endsWith(matrix.python, '-dev') with: python-version: '${{ matrix.python }}' - cache: pip - cache-dependency-path: test-requirements.txt - name: Run tests run: ./ci.sh env: From e6696f7bb78d10631911c7f91b57ebe2160c0d5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 10:25:31 +0000 Subject: [PATCH 1058/1498] Bump typing-extensions from 4.1.1 to 4.2.0 Bumps [typing-extensions](https://github.com/python/typing) from 4.1.1 to 4.2.0. - [Release notes](https://github.com/python/typing/releases) - [Changelog](https://github.com/python/typing/blob/master/typing_extensions/CHANGELOG) - [Commits](https://github.com/python/typing/compare/4.1.1...4.2.0) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6669b22c8e..d06e8177ee 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -126,7 +126,7 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typing-extensions==4.1.1 ; implementation_name == "cpython" +typing-extensions==4.2.0 ; implementation_name == "cpython" # via # -r test-requirements.in # mypy From 1e85ee17f93f0a723a77ef22edf97b22d70191b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 10:26:25 +0000 Subject: [PATCH 1059/1498] Bump platformdirs from 2.5.1 to 2.5.2 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.5.1 to 2.5.2. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.5.1...2.5.2) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6669b22c8e..e67d30d3e3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -77,7 +77,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.1 +platformdirs==2.5.2 # via # black # pylint From 245e8a3a40fc197c79a50ccba819c1a18e57be2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 10:34:13 +0000 Subject: [PATCH 1060/1498] Bump astroid from 2.11.2 to 2.11.3 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.11.2 to 2.11.3. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.11.2...v2.11.3) --- updated-dependencies: - dependency-name: astroid dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e67d30d3e3..684a67980f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.11.2 +astroid==2.11.3 # via pylint async-generator==1.10 # via -r test-requirements.in From 3c2bf97a1597f3c3132b6dceb7581292d3c7956c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:21:30 +0000 Subject: [PATCH 1061/1498] Bump pytest from 7.1.1 to 7.1.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.1 to 7.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.1...7.1.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e67d30d3e3..0765d641f1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -103,7 +103,7 @@ pyopenssl==22.0.0 # via -r test-requirements.in pyparsing==3.0.8 # via packaging -pytest==7.1.1 +pytest==7.1.2 # via # -r test-requirements.in # pytest-cov From 6e30a3f4239d97726a29b74852ab6b494546da7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 12:46:45 +0000 Subject: [PATCH 1062/1498] Bump pylint from 2.13.4 to 2.13.7 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.13.4 to 2.13.7. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.13.4...v2.13.7) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f51c283dbe..e334af1848 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.13.4 +pylint==2.13.7 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From 806b3bc5a095c5e20d885ec27afb605c34fac4eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Apr 2022 10:18:24 +0000 Subject: [PATCH 1063/1498] Bump cryptography from 36.0.2 to 37.0.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 36.0.2 to 37.0.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/36.0.2...37.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e334af1848..0c199c454a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.2 # via black coverage[toml]==6.0.2 # via pytest-cov -cryptography==36.0.2 +cryptography==37.0.1 # via # -r test-requirements.in # pyopenssl From 654249d5c32e15b336a6ced248514be126a6fbf7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Apr 2022 10:21:11 +0000 Subject: [PATCH 1064/1498] Bump mypy from 0.942 to 0.950 Bumps [mypy](https://github.com/python/mypy) from 0.942 to 0.950. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.942...v0.950) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e334af1848..7aee396255 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -58,7 +58,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.942 ; implementation_name == "cpython" +mypy==0.950 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From dd315b329afd261f711bac331331a9ecdacfff27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 May 2022 10:29:43 +0000 Subject: [PATCH 1065/1498] Bump pylint from 2.13.7 to 2.13.8 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.13.7 to 2.13.8. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.13.7...v2.13.8) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index aeac75432f..3099154c47 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.13.7 +pylint==2.13.8 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From 62bcdcfb813d1a0ba0fc2cc7e5de2f173af3cc5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 May 2022 10:29:30 +0000 Subject: [PATCH 1066/1498] Bump astroid from 2.11.3 to 2.11.5 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.11.3 to 2.11.5. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.11.3...v2.11.5) --- updated-dependencies: - dependency-name: astroid dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3099154c47..7382736118 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.11.3 +astroid==2.11.5 # via pylint async-generator==1.10 # via -r test-requirements.in From 3e078ebfa9acce0b1e72bb478f8816ae86df1761 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 May 2022 10:24:31 +0000 Subject: [PATCH 1067/1498] Bump pyparsing from 3.0.8 to 3.0.9 Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 3.0.8 to 3.0.9. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_3.0.8...pyparsing_3.0.9) --- updated-dependencies: - dependency-name: pyparsing dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 45c6dcc42f..5c0ca035b7 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -50,7 +50,7 @@ packaging==21.3 # via sphinx pygments==2.10.0 # via sphinx -pyparsing==3.0.8 +pyparsing==3.0.9 # via packaging pytz==2022.1 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index 3099154c47..73b8191695 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -101,7 +101,7 @@ pylint==2.13.8 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in -pyparsing==3.0.8 +pyparsing==3.0.9 # via packaging pytest==7.1.2 # via From d49e8ec4af92655cf261eeae0a45305802e7f1e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Fri, 13 May 2022 20:02:11 +0300 Subject: [PATCH 1068/1498] Update trio/_core/_multierror.py --- trio/_core/_multierror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 78c828ebe2..1b32b91534 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -420,7 +420,7 @@ def traceback_exception_init( # copy the set of _seen exceptions so that duplicates # shared between sub-exceptions are not omitted _seen=None if seen_was_none else set(_seen), - **kwargs + **kwargs, ) ) self.embedded = embedded From 95af17a8098731a8b807da0ef962a44af02ed07b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Fri, 13 May 2022 20:02:16 +0300 Subject: [PATCH 1069/1498] Update trio/_core/_multierror.py --- trio/_core/_multierror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 1b32b91534..de0d56d464 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -383,7 +383,7 @@ def traceback_exception_init( capture_locals=False, compact=False, _seen=None, - **kwargs + **kwargs, ): if sys.version_info >= (3, 10): kwargs["compact"] = compact From b4791f805e9718798d257c1df0e9213306880365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 May 2022 10:54:29 +0300 Subject: [PATCH 1070/1498] Fixed error in test_coroutine_or_error() on py3.11 --- trio/tests/test_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 40743f0cfa..15ab09a80b 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -116,9 +116,9 @@ async def f(): # pragma: no cover def generator_based_coro(): # pragma: no cover yield from asyncio.sleep(1) - with pytest.raises(TypeError) as excinfo: - coroutine_or_error(generator_based_coro()) - assert "asyncio" in str(excinfo.value) + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(generator_based_coro()) + assert "asyncio" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: coroutine_or_error(create_asyncio_future_in_new_loop()) From 70bb109b901a5542b668c8dbc614afea881ceffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 May 2022 10:55:20 +0300 Subject: [PATCH 1071/1498] Updated dependencies Seems like we don't need to fetch wrapt directly from git anymore. --- docs-requirements.txt | 17 +++++++++++++++-- test-requirements.in | 1 - test-requirements.txt | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 5c0ca035b7..def71ea6e3 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # -# pip-compile --output-file docs-requirements.txt docs-requirements.in +# pip-compile --output-file=docs-requirements.txt docs-requirements.in # alabaster==0.7.12 # via sphinx @@ -28,6 +28,8 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +exceptiongroup==1.0.0rc5 + # via -r docs-requirements.in idna==3.3 # via # -r docs-requirements.in @@ -36,6 +38,8 @@ imagesize==1.3.0 # via sphinx immutables==0.17 # via -r docs-requirements.in +importlib-metadata==4.11.3 + # via click incremental==21.3.0 # via towncrier jinja2==3.0.3 @@ -87,5 +91,14 @@ tomli==2.0.1 # via towncrier towncrier==21.9.0 # via -r docs-requirements.in +typing-extensions==4.2.0 + # via + # immutables + # importlib-metadata urllib3==1.26.9 # via requests +zipp==3.8.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/test-requirements.in b/test-requirements.in index fd0d482378..024ea35b21 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -9,7 +9,6 @@ pylint # for pylint finding all symbols tests jedi # for jedi code completion tests cryptography>=36.0.0 # 35.0.0 is transitive but fails exceptiongroup # for catch() -wrapt @ git+https://github.com/GrahamDumpleton/wrapt.git@a1a06529f41cffcc9fd5132fcf1923ea538f3d3c # fixes Python 3.11 compat for pylint # Tools black; implementation_name == "cpython" diff --git a/test-requirements.txt b/test-requirements.txt index f0b4eec163..cbddabf8fc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -34,12 +34,20 @@ decorator==5.1.1 # via ipython dill==0.3.4 # via pylint +exceptiongroup==1.0.0rc5 + # via -r test-requirements.in flake8==4.0.1 # via -r test-requirements.in idna==3.3 # via # -r test-requirements.in # trustme +importlib-metadata==4.2.0 + # via + # click + # flake8 + # pluggy + # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 @@ -126,14 +134,26 @@ traitlets==5.1.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in +typed-ast==1.5.3 ; implementation_name == "cpython" and python_version < "3.8" + # via + # -r test-requirements.in + # astroid + # black + # mypy typing-extensions==4.2.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black + # importlib-metadata # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.14.0 # via astroid +zipp==3.8.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 1c403b17861f8f84440cd9c862afba26c918ec9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 May 2022 11:31:43 +0300 Subject: [PATCH 1072/1498] Move Python 3.11 to its separate test job --- .github/workflows/ci.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c35774450..2405cdc173 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.7', 'pypy-3.8', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev', '3.11-dev'] + python: ['pypy-3.7', 'pypy-3.8', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] @@ -111,6 +111,25 @@ jobs: # Should match 'name:' up above JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' + # Requires a separate job to avoid using coverage which is currently broken on 3.11 + Python311: + name: 'Ubuntu (3.11-dev)' + timeout-minutes: 10 + runs-on: 'ubuntu-latest' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.11-dev + cache: pip + cache-dependency-path: test-requirements.txt + - name: Install dependencies + run: pip install -r test-requirements.txt + - name: Run tests + run: pytest -r -a --verbose + macOS: name: 'macOS (${{ matrix.python }})' timeout-minutes: 10 From a28bfaca8963f8614f20d7b61466f3344efe9f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 May 2022 11:33:57 +0300 Subject: [PATCH 1073/1498] Fixed bad pytest switch --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2405cdc173..eee8e409cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,7 +128,7 @@ jobs: - name: Install dependencies run: pip install -r test-requirements.txt - name: Run tests - run: pytest -r -a --verbose + run: pytest -ra --verbose macOS: name: 'macOS (${{ matrix.python }})' From 769459205a62a12dc813fbc6ad7e976522649192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 May 2022 22:22:08 +0300 Subject: [PATCH 1074/1498] Implemented the strict_exception_groups setting --- trio/_core/_run.py | 59 +++++++++++++++++++++++++++++------- trio/_core/tests/test_run.py | 42 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 9ff7f890f5..3f2ac9e3a4 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -472,7 +472,7 @@ def __enter__(self): task._activate_cancel_status(self._cancel_status) return self - def _close(self, exc): + def _close(self, exc, collapse=True): if self._cancel_status is None: new_exc = RuntimeError( "Cancel scope stack corrupted: attempted to exit {!r} " @@ -537,7 +537,7 @@ def _close(self, exc): if matched: self.cancelled_caught = True - if isinstance(exc, BaseExceptionGroup): + if collapse and isinstance(exc, BaseExceptionGroup): exc = collapse_exception_group(exc) self._cancel_status.close() @@ -807,6 +807,7 @@ def started(self, value=None): self._old_nursery._check_nursery_closed() +@attr.s class NurseryManager: """Nursery context manager. @@ -817,11 +818,15 @@ class NurseryManager: """ + strict_exception_groups = attr.ib(default=False) + @enable_ki_protection async def __aenter__(self): self._scope = CancelScope() self._scope.__enter__() - self._nursery = Nursery._create(current_task(), self._scope) + self._nursery = Nursery._create( + current_task(), self._scope, self.strict_exception_groups + ) return self._nursery @enable_ki_protection @@ -829,7 +834,9 @@ async def __aexit__(self, etype, exc, tb): new_exc = await self._nursery._nested_child_finished(exc) # Tracebacks show the 'raise' line below out of context, so let's give # this variable a name that makes sense out of context. - combined_error_from_nursery = self._scope._close(new_exc) + combined_error_from_nursery = self._scope._close( + new_exc, collapse=not self.strict_exception_groups + ) if combined_error_from_nursery is None: return True elif combined_error_from_nursery is exc: @@ -857,15 +864,23 @@ def __exit__(self): # pragma: no cover assert False, """Never called, but should be defined""" -def open_nursery(): +def open_nursery(strict_exception_groups=None): """Returns an async context manager which must be used to create a new `Nursery`. It does not block on entry; on exit it blocks until all child tasks have exited. + Args: + strict_exception_groups (bool): If true, even a single raised exception will be + wrapped in an exception group. This will eventually become the default + behavior. If not specified, uses the value passed to :func:`run`. + """ - return NurseryManager() + if strict_exception_groups is None: + strict_exception_groups = GLOBAL_RUN_CONTEXT.runner.strict_exception_groups + + return NurseryManager(strict_exception_groups=strict_exception_groups) class Nursery(metaclass=NoPublicConstructor): @@ -890,8 +905,9 @@ class Nursery(metaclass=NoPublicConstructor): in response to some external event. """ - def __init__(self, parent_task, cancel_scope): + def __init__(self, parent_task, cancel_scope, strict_exception_groups): self._parent_task = parent_task + self._strict_exception_groups = strict_exception_groups parent_task._child_nurseries.append(self) # the cancel status that children inherit - we take a snapshot, so it # won't be affected by any changes in the parent. @@ -970,7 +986,9 @@ def aborted(raise_cancel): assert popped is self if self._pending_excs: try: - return MultiError(self._pending_excs) + return MultiError( + self._pending_excs, _collapse=not self._strict_exception_groups + ) finally: # avoid a garbage cycle # (see test_nursery_cancel_doesnt_create_cyclic_garbage) @@ -1309,6 +1327,7 @@ class Runner: instruments: Instruments = attr.ib() io_manager = attr.ib() ki_manager = attr.ib() + strict_exception_groups = attr.ib() # Run-local values, see _local.py _locals = attr.ib(factory=dict) @@ -1847,7 +1866,12 @@ def abort(_): # 'is_guest' to see the special cases we need to handle this. -def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints): +def setup_runner( + clock, + instruments, + restrict_keyboard_interrupt_to_checkpoints, + strict_exception_groups, +): """Create a Runner object and install it as the GLOBAL_RUN_CONTEXT.""" # It wouldn't be *hard* to support nested calls to run(), but I can't # think of a single good reason for it, so let's be conservative for @@ -1869,6 +1893,7 @@ def setup_runner(clock, instruments, restrict_keyboard_interrupt_to_checkpoints) io_manager=io_manager, system_context=system_context, ki_manager=ki_manager, + strict_exception_groups=strict_exception_groups, ) runner.asyncgens.install_hooks(runner) @@ -1886,6 +1911,7 @@ def run( clock=None, instruments=(), restrict_keyboard_interrupt_to_checkpoints=False, + strict_exception_groups=False, ): """Run a Trio-flavored async function, and return the result. @@ -1942,6 +1968,10 @@ def run( main thread (this is a Python limitation), or if you use :func:`open_signal_receiver` to catch SIGINT. + strict_exception_groups (bool): If true, nurseries will always wrap even a single + raised exception in an exception group. This can be overridden on the level of + individual nurseries. This will eventually become the default behavior. + Returns: Whatever ``async_fn`` returns. @@ -1958,7 +1988,10 @@ def run( __tracebackhide__ = True runner = setup_runner( - clock, instruments, restrict_keyboard_interrupt_to_checkpoints + clock, + instruments, + restrict_keyboard_interrupt_to_checkpoints, + strict_exception_groups, ) gen = unrolled_run(runner, async_fn, args) @@ -1987,6 +2020,7 @@ def start_guest_run( clock=None, instruments=(), restrict_keyboard_interrupt_to_checkpoints=False, + strict_exception_groups=False, ): """Start a "guest" run of Trio on top of some other "host" event loop. @@ -2038,7 +2072,10 @@ def my_done_callback(run_outcome): """ runner = setup_runner( - clock, instruments, restrict_keyboard_interrupt_to_checkpoints + clock, + instruments, + restrict_keyboard_interrupt_to_checkpoints, + strict_exception_groups, ) runner.is_guest = True runner.guest_tick_scheduled = True diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 8fcf8c8b78..9160e04813 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2346,3 +2346,45 @@ async def task(): nursery.start_soon(task) nursery.cancel_scope.cancel() assert destroyed + + +def test_run_strict_exception_groups(): + """ + Test that nurseries respect the global context setting of strict_exception_groups. + """ + + async def main(): + async with _core.open_nursery(): + raise Exception("foo") + + with pytest.raises(MultiError) as exc: + _core.run(main, strict_exception_groups=True) + + assert len(exc.value.exceptions) == 1 + assert type(exc.value.exceptions[0]) is Exception + assert exc.value.exceptions[0].args == ("foo",) + + +def test_run_strict_exception_groups_nursery_override(): + """ + Test that a nursery can override the global context setting of + strict_exception_groups. + """ + + async def main(): + async with _core.open_nursery(strict_exception_groups=False): + raise Exception("foo") + + with pytest.raises(Exception, match="foo"): + _core.run(main, strict_exception_groups=True) + + +async def test_nursery_strict_exception_groups(): + """Test that strict exception groups can be enabled on a per-nursery basis.""" + with pytest.raises(MultiError) as exc: + async with _core.open_nursery(strict_exception_groups=True): + raise Exception("foo") + + assert len(exc.value.exceptions) == 1 + assert type(exc.value.exceptions[0]) is Exception + assert exc.value.exceptions[0].args == ("foo",) From 161c0ab745d2529b2ec3f7bc0e242cd4338e9451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 14 May 2022 23:43:05 +0300 Subject: [PATCH 1075/1498] Implemented NonBaseMultiError which inherits from both MultiError and ExceptionGroup This is intended to let users use "except Exception:" with MultiError. --- trio/_core/_multierror.py | 10 +++++++++- trio/_core/tests/test_multierror.py | 26 +++++++++++++++++++++++++- trio/_core/tests/test_run.py | 4 ++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index ac6e3bd2b2..0412b4c214 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -5,7 +5,7 @@ import attr if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup + from exceptiongroup import BaseExceptionGroup, ExceptionGroup ################################################################ # MultiError @@ -203,6 +203,9 @@ def __new__(cls, exceptions, *, _collapse=True): # In an earlier version of the code, we didn't define __init__ and # simply set the `exceptions` attribute directly on the new object. # However, linters expect attributes to be initialized in __init__. + if all(isinstance(exc, Exception) for exc in exceptions): + cls = NonBaseMultiError + return super().__new__(cls, "multiple tasks failed", exceptions) def __str__(self): @@ -256,8 +259,13 @@ def catch(cls, handler): return MultiErrorCatcher(handler) +class NonBaseMultiError(MultiError, ExceptionGroup): + pass + + # Clean up exception printing: MultiError.__module__ = "trio" +NonBaseMultiError.__module__ = "trio" ################################################################ # concat_tb diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 01e6253d07..5890e21c2c 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -11,9 +11,12 @@ import sys import re -from .._multierror import MultiError, concat_tb +from .._multierror import MultiError, concat_tb, NonBaseMultiError from ..._core import open_nursery +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + class NotHashableException(Exception): code = None @@ -392,3 +395,24 @@ def test_assert_match_in_seq(): assert_match_in_seq(["b", "a"], "xx b xx a xx") with pytest.raises(AssertionError): assert_match_in_seq(["a", "b"], "xx b xx a xx") + + +def test_base_multierror(): + """ + Test that MultiError() witho at least one base exception will return a MultiError + object. + """ + + exc = MultiError([ZeroDivisionError(), KeyboardInterrupt()]) + assert type(exc) is MultiError + + +def test_non_base_multierror(): + """ + Test that MultiError() without base exceptions will return a NonBaseMultiError + object. + """ + + exc = MultiError([ZeroDivisionError(), ValueError()]) + assert type(exc) is NonBaseMultiError + assert isinstance(exc, ExceptionGroup) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 9160e04813..a3e0e3e28f 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -29,7 +29,7 @@ ) from ... import _core -from ..._core._multierror import MultiError +from ..._core._multierror import MultiError, NonBaseMultiError from .._run import DEADLINE_HEAP_MIN_PRUNE_THRESHOLD from ..._threads import to_thread_run_sync from ..._timeouts import sleep, fail_after @@ -776,7 +776,7 @@ async def task2(): with pytest.raises(RuntimeError) as exc_info: await nursery_mgr.__aexit__(*sys.exc_info()) assert "which had already been exited" in str(exc_info.value) - assert type(exc_info.value.__context__) is MultiError + assert type(exc_info.value.__context__) is NonBaseMultiError assert len(exc_info.value.__context__.exceptions) == 3 cancelled_in_context = False for exc in exc_info.value.__context__.exceptions: From 226184f2dc89b90112204d758bec2db12d455d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 15 May 2022 00:39:26 +0300 Subject: [PATCH 1076/1498] Fixed MultiError reference in version history --- docs/source/history.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/history.rst b/docs/source/history.rst index e2576fa64d..72d27955bf 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -31,8 +31,8 @@ Features Bugfixes ~~~~~~~~ -- Trio now avoids creating cyclic garbage when a `MultiError` is generated and filtered, - including invisibly within the cancellation system. This means errors raised +- Trio now avoids creating cyclic garbage when a ``MultiError`` is generated and + filtered, including invisibly within the cancellation system. This means errors raised through nurseries and cancel scopes should result in less GC latency. (`#2063 `__) - Trio now deterministically cleans up file descriptors that were opened before subprocess creation fails. Previously, they would remain open until the next run of From 748ceb8a2f220bcad50886152c508f44bdda6cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 15 May 2022 00:41:56 +0300 Subject: [PATCH 1077/1498] Removed Python 3.11 from test matrix It's not ready yet (CFFI is segfaulting). --- .github/workflows/ci.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eee8e409cf..ece2b0d0a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,25 +111,6 @@ jobs: # Should match 'name:' up above JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' - # Requires a separate job to avoid using coverage which is currently broken on 3.11 - Python311: - name: 'Ubuntu (3.11-dev)' - timeout-minutes: 10 - runs-on: 'ubuntu-latest' - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: 3.11-dev - cache: pip - cache-dependency-path: test-requirements.txt - - name: Install dependencies - run: pip install -r test-requirements.txt - - name: Run tests - run: pytest -ra --verbose - macOS: name: 'macOS (${{ matrix.python }})' timeout-minutes: 10 From dfdda5bd9e226f7a11dad3f12541ec19cf13e17f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 15 May 2022 01:34:41 +0300 Subject: [PATCH 1078/1498] Documented the MultiError compatibility issue --- docs/source/reference-core.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 9639287d71..6fd4b6631f 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -750,6 +750,20 @@ callbacks:: .. hint:: If your code, written using ``except*``, would set local variables, you can do the same with handler callbacks as long as you declare those variables ``nonlocal``. +For reasons of backwards compatibility, nurseries raise ``MultiError`` and +``NonBaseMultiError`` which inherit from `BaseExceptionGroup` and `ExceptionGroup`, +respectively. Users should refrain from attempting to raise or catch the trio specific +exceptions themselves, and treat them as if they were standard `BaseExceptionGroup` or +`ExceptionGroup` instances instead. + +The compatibility exception classes have one additional quirk: when only a single +exception is passed to them, they will spit out that single exception from the +constructor instead of the appropriate ``MultiError`` instance. This means that a +nursery can pass through any arbitrary exception raised by either a spawned task or the +host task. This behavior can be controlled by the ``strict_exception_groups=True`` +argument passed to either :func:`open_nursery` or :func:`run`. If enabled, a nursery +will always wrap even single exception raised in it in an exception group. + .. _exceptiongroup: https://pypi.org/project/exceptiongroup/ Spawning tasks without becoming a parent From 3dd0ef808bccf08edfda20b5fe74e74e79fbf10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 15 May 2022 01:53:44 +0300 Subject: [PATCH 1079/1498] Fixed test failure on Python 3.11 asyncio.coroutine has been removed in Python 3.11. --- trio/tests/test_util.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/trio/tests/test_util.py b/trio/tests/test_util.py index 2d57e0ebfc..15ab09a80b 100644 --- a/trio/tests/test_util.py +++ b/trio/tests/test_util.py @@ -1,4 +1,6 @@ import signal +import sys + import pytest import trio @@ -108,13 +110,15 @@ async def f(): # pragma: no cover import asyncio - @asyncio.coroutine - def generator_based_coro(): # pragma: no cover - yield from asyncio.sleep(1) + if sys.version_info < (3, 11): - with pytest.raises(TypeError) as excinfo: - coroutine_or_error(generator_based_coro()) - assert "asyncio" in str(excinfo.value) + @asyncio.coroutine + def generator_based_coro(): # pragma: no cover + yield from asyncio.sleep(1) + + with pytest.raises(TypeError) as excinfo: + coroutine_or_error(generator_based_coro()) + assert "asyncio" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: coroutine_or_error(create_asyncio_future_in_new_loop()) From 09fcd83e98d453d4039909902a2a9f8313629cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 16 May 2022 10:53:37 +0300 Subject: [PATCH 1080/1498] Fixed remaining Python 3.11 test failures (#2319) * Fixed remaining test failures on Python 3.11 * Updated coverage.py * Added py3.11-dev to test matrix, removed 3.8-3.10-dev * Removed unnecessary warning filters and added issue link for the new one --- .github/workflows/ci.yml | 2 +- test-requirements.txt | 2 +- trio/socket.py | 2 +- trio/tests/test_exports.py | 15 ++------------- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ece2b0d0a0..934fbbe026 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['pypy-3.7', 'pypy-3.8', '3.7', '3.8', '3.9', '3.10', '3.8-dev', '3.9-dev', '3.10-dev'] + python: ['pypy-3.7', 'pypy-3.8', '3.7', '3.8', '3.9', '3.10', '3.11-dev'] check_formatting: ['0'] pypy_nightly_branch: [''] extra_name: [''] diff --git a/test-requirements.txt b/test-requirements.txt index f0b4eec163..a8aa981d1e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,7 +23,7 @@ cffi==1.15.0 # via cryptography click==8.1.2 # via black -coverage[toml]==6.0.2 +coverage[toml]==6.3.3 # via pytest-cov cryptography==37.0.1 # via diff --git a/trio/socket.py b/trio/socket.py index 416a2ba9b9..4bbc7d14f6 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -118,7 +118,7 @@ SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, SCM_J1939_PRIO, SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, SO_J1939_SEND_PRIO, UDPLITE_RECV_CSCOV, UDPLITE_SEND_CSCOV, IP_RECVTOS, - TCP_KEEPALIVE + TCP_KEEPALIVE, SO_INCOMING_CPU ) # fmt: on except ImportError: diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index ef7edb4ccb..8d6b2d6131 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -64,22 +64,11 @@ def public_modules(module): sys.version_info.releaselevel == "alpha", reason="skip static introspection tools on Python dev/alpha releases", ) -@pytest.mark.filterwarnings( - # https://github.com/PyCQA/astroid/issues/681 - "ignore:the imp module is deprecated.*:DeprecationWarning" -) @pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) @pytest.mark.parametrize("tool", ["pylint", "jedi"]) @pytest.mark.filterwarnings( - "ignore:" - + re.escape( - "The distutils package is deprecated and slated for removal in Python 3.12. " - "Use setuptools or check PEP 632 for potential alternatives" - ) - + ":DeprecationWarning", - "ignore:" - + re.escape("The distutils.sysconfig module is deprecated, use sysconfig instead") - + ":DeprecationWarning", + # https://github.com/pypa/setuptools/issues/3274 + "ignore:module 'sre_constants' is deprecated:DeprecationWarning", ) def test_static_tool_sees_all_symbols(tool, modname): module = importlib.import_module(modname) From 465613c2945de1bc3f14b8693e3dd35a3cfe3c37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 11:12:17 +0300 Subject: [PATCH 1081/1498] Bump click from 8.1.2 to 8.1.3 (#2308) Bumps [click](https://github.com/pallets/click) from 8.1.2 to 8.1.3. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.1.2...8.1.3) --- updated-dependencies: - dependency-name: click dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 5c0ca035b7..8d3434789e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -18,7 +18,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.12 # via requests -click==8.1.2 +click==8.1.3 # via # click-default-group # towncrier diff --git a/test-requirements.txt b/test-requirements.txt index a8aa981d1e..09bcd5835d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,7 +21,7 @@ black==22.3.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.0 # via cryptography -click==8.1.2 +click==8.1.3 # via black coverage[toml]==6.3.3 # via pytest-cov From 2dd7198c8231fa13bff36ebaa4c81ce1045fec21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 11:17:10 +0300 Subject: [PATCH 1082/1498] Bump wrapt from 1.14.0 to 1.14.1 (#2310) Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.14.0 to 1.14.1. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.14.0...1.14.1) --- updated-dependencies: - dependency-name: wrapt dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 09bcd5835d..a61e5bee75 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -132,7 +132,7 @@ typing-extensions==4.2.0 ; implementation_name == "cpython" # mypy wcwidth==0.2.5 # via prompt-toolkit -wrapt==1.14.0 +wrapt==1.14.1 # via astroid # The following packages are considered to be unsafe in a requirements file: From 65f88980dfacfe2484471c889b040e5d129781c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 11:19:29 +0300 Subject: [PATCH 1083/1498] Bump traitlets from 5.1.1 to 5.2.0 (#2315) Bumps [traitlets](https://github.com/ipython/traitlets) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.1.1...5.2.0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a61e5bee75..cc74568cac 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -120,7 +120,7 @@ tomli==2.0.1 # mypy # pylint # pytest -traitlets==5.1.1 +traitlets==5.2.0 # via # ipython # matplotlib-inline From b2ea3b3348f87693a71093e25fc56e8a28068a63 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 16 May 2022 12:24:24 +0400 Subject: [PATCH 1084/1498] Skip frequently failing test on Windows + 3.10 (#2158) --- trio/tests/test_ssl.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 10e6c13d8f..7825e319c2 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1,3 +1,4 @@ +import os import re import sys @@ -760,6 +761,10 @@ async def wait_send_all_might_not_block(self): assert record == ["ok"] +@pytest.mark.skipif( + os.name == "nt" and sys.version_info >= (3, 10), + reason="frequently fails on Windows + Python 3.10", +) async def test_checkpoints(client_ctx): async with ssl_echo_server(client_ctx) as s: with assert_checkpoints(): From 1a5e3bee295b9cd70e631f18ec3f5e5f6ea6baf9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 10:23:34 +0000 Subject: [PATCH 1085/1498] Bump pylint from 2.13.8 to 2.13.9 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.13.8 to 2.13.9. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/v2.13.8...v2.13.9) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index cc74568cac..17af37f0b1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.13.8 +pylint==2.13.9 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From 6bd6045b7ae01ac3ffc5fdf2f7579866da4dc125 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 10:35:38 +0000 Subject: [PATCH 1086/1498] Bump cryptography from 37.0.1 to 37.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 37.0.1 to 37.0.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/37.0.1...37.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index cc74568cac..1186e4969a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.3.3 # via pytest-cov -cryptography==37.0.1 +cryptography==37.0.2 # via # -r test-requirements.in # pyopenssl From e8a1b604ef0dbc17c63e1f0092262ee52b2e757b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 17 May 2022 09:28:43 +0300 Subject: [PATCH 1087/1498] Documentation updates --- docs/source/reference-core.rst | 44 ++++++++++++++++++++++------- trio/_core/tests/test_multierror.py | 2 +- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 6fd4b6631f..bf9fd3202d 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -750,19 +750,43 @@ callbacks:: .. hint:: If your code, written using ``except*``, would set local variables, you can do the same with handler callbacks as long as you declare those variables ``nonlocal``. -For reasons of backwards compatibility, nurseries raise ``MultiError`` and -``NonBaseMultiError`` which inherit from `BaseExceptionGroup` and `ExceptionGroup`, -respectively. Users should refrain from attempting to raise or catch the trio specific +For reasons of backwards compatibility, nurseries raise ``trio.MultiError`` and +``trio.NonBaseMultiError`` which inherit from `BaseExceptionGroup` and `ExceptionGroup`, +respectively. Users should refrain from attempting to raise or catch the Trio specific exceptions themselves, and treat them as if they were standard `BaseExceptionGroup` or `ExceptionGroup` instances instead. -The compatibility exception classes have one additional quirk: when only a single -exception is passed to them, they will spit out that single exception from the -constructor instead of the appropriate ``MultiError`` instance. This means that a -nursery can pass through any arbitrary exception raised by either a spawned task or the -host task. This behavior can be controlled by the ``strict_exception_groups=True`` -argument passed to either :func:`open_nursery` or :func:`run`. If enabled, a nursery -will always wrap even single exception raised in it in an exception group. +"Strict" versus "loose" ExceptionGroup semantics +++++++++++++++++++++++++++++++++++++++++++++++++ + +Ideally, in some abstract sense we'd want everything that *can* raise an +`ExceptionGroup` to *always* raise an `ExceptionGroup` (rather than, say, a single +`ValueError`). Otherwise, it would be easy to accidentally write something like ``except +ValueError:`` (not ``except*``), which works if a single exception is raised but fails to +catch _anything_ in the case of multiple simultaneous exceptions (even if one of them is +a ValueError). However, this is not how Trio worked in the past: as a concession to +practicality when the ``except*`` syntax hadn't been dreamed up yet, the old +``trio.MultiError`` was raised only when at least two exceptions occurred +simultaneously. Adding a layer of `ExceptionGroup` around every nursery, while +theoretically appealing, would probably break a lot of existing code in practice. + +Therefore, we've chosen to gate the newer, "stricter" behavior behind a parameter +called ``strict_exception_groups``. This is accepted as a parameter to +:func:`open_nursery`, to set the behavior for that nursery, and to :func:`trio.run`, +to set the default behavior for any nursery in your program that doesn't override it. + +* With ``strict_exception_groups=True``, the exception(s) coming out of a nursery will + always be wrapped in an `ExceptionGroup`, so you'll know that if you're handling + single errors correctly, multiple simultaneous errors will work as well. + +* With ``strict_exception_groups=False``, a nursery in which only one task has failed + will raise that task's exception without an additional layer of `ExceptionGroup` + wrapping, so you'll get maximum compatibility with code that was written to + support older versions of Trio. + +To maintain backwards compatibility, the default is ``strict_exception_groups=False``. +The default will eventually change to ``True`` in a future version of Trio, once +Python 3.11 and later versions are in wide use. .. _exceptiongroup: https://pypi.org/project/exceptiongroup/ diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 5890e21c2c..9b8d7356cb 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -399,7 +399,7 @@ def test_assert_match_in_seq(): def test_base_multierror(): """ - Test that MultiError() witho at least one base exception will return a MultiError + Test that MultiError() with at least one base exception will return a MultiError object. """ From 4bd09bc346ee44ac3588ebcc2caaa3ab4ab46314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 17 May 2022 09:36:34 +0300 Subject: [PATCH 1088/1498] Changed open_tcp_listeners() and open_tcp_stream() to use ExceptionGroup directly --- trio/_highlevel_open_tcp_listeners.py | 14 +++++++++----- trio/_highlevel_open_tcp_stream.py | 8 ++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index cce6f1d602..b650ac973f 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -4,7 +4,9 @@ import trio from . import socket as tsocket -from ._core._multierror import MultiError + +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup # Default backlog size: @@ -135,11 +137,13 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): raise if unsupported_address_families and not listeners: - raise OSError( - errno.EAFNOSUPPORT, + msg = ( "This system doesn't support any of the kinds of " - "socket that that address could use", - ) from MultiError(unsupported_address_families) + "socket that that address could use" + ) + raise OSError(errno.EAFNOSUPPORT, msg) from ExceptionGroup( + msg, unsupported_address_families + ) return listeners diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index c57d421043..740c0b1752 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -1,10 +1,14 @@ -import warnings +import sys from contextlib import contextmanager import trio from trio._core._multierror import MultiError from trio.socket import getaddrinfo, SOCK_STREAM, socket +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + + # Implementation of RFC 6555 "Happy eyeballs" # https://tools.ietf.org/html/rfc6555 # @@ -368,7 +372,7 @@ async def attempt_connect(socket_args, sockaddr, attempt_failed): msg = "all attempts to connect to {} failed".format( format_host_port(host, port) ) - raise OSError(msg) from MultiError(oserrors) + raise OSError(msg) from ExceptionGroup(msg, oserrors) else: stream = trio.SocketStream(winning_socket) open_sockets.remove(winning_socket) From 1101ead769433b76bacd940d2925fa3a3c47a46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 17 May 2022 09:43:35 +0300 Subject: [PATCH 1089/1498] Expose NonBaseMultiError in trio (as a deprecated attribute) --- trio/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/trio/__init__.py b/trio/__init__.py index ec8b58e616..d3be8fcb4b 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -88,6 +88,7 @@ ) from ._core._multierror import MultiError as _MultiError +from ._core._multierror import NonBaseMultiError as _NonBaseMultiError from ._deprecate import TrioDeprecationWarning @@ -116,10 +117,16 @@ ), "MultiError": _deprecate.DeprecatedAttribute( value=_MultiError, - version="0.20.0", + version="0.21.0", issue=2211, instead="BaseExceptionGroup (py3.11+) or exceptiongroup.BaseExceptionGroup", ), + "NonBaseMultiError": _deprecate.DeprecatedAttribute( + value=_NonBaseMultiError, + version="0.21.0", + issue=2211, + instead="ExceptionGroup (py3.11+) or exceptiongroup.ExceptionGroup", + ), } # Having the public path in .__module__ attributes is important for: From b62e5154f45141be7d593c5d3c1234f9cc296352 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 May 2022 10:20:03 +0000 Subject: [PATCH 1090/1498] Bump traitlets from 5.2.0 to 5.2.1.post0 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.2.0 to 5.2.1.post0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.2.0...5.2.1.post0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3421e49edd..6bd2dd113b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -120,7 +120,7 @@ tomli==2.0.1 # mypy # pylint # pytest -traitlets==5.2.0 +traitlets==5.2.1.post0 # via # ipython # matplotlib-inline From 6bc10ca0deb0ffb20ba6506a6e702e24c4cc6a31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 May 2022 10:19:31 +0000 Subject: [PATCH 1091/1498] Bump certifi from 2021.10.8 to 2022.5.18 Bumps [certifi](https://github.com/certifi/python-certifi) from 2021.10.8 to 2022.5.18. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/commits) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 8d3434789e..673ca146f9 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==21.4.0 # outcome babel==2.9.1 # via sphinx -certifi==2021.10.8 +certifi==2022.5.18 # via requests charset-normalizer==2.0.12 # via requests From 3a6fd6f4fbba644fd900b59df8133384f1a9f1b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 May 2022 10:18:18 +0000 Subject: [PATCH 1092/1498] Bump certifi from 2022.5.18 to 2022.5.18.1 Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.5.18 to 2022.5.18.1. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.05.18...2022.05.18.1) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 673ca146f9..8454af954b 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==21.4.0 # outcome babel==2.9.1 # via sphinx -certifi==2022.5.18 +certifi==2022.5.18.1 # via requests charset-normalizer==2.0.12 # via requests From d2ffffeb3087db2b5161a741f6e7b9c0b92fbc3c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 May 2022 10:19:15 +0000 Subject: [PATCH 1093/1498] Bump dill from 0.3.4 to 0.3.5 Bumps [dill](https://github.com/uqfoundation/dill) from 0.3.4 to 0.3.5. - [Release notes](https://github.com/uqfoundation/dill/releases) - [Commits](https://github.com/uqfoundation/dill/compare/dill-0.3.4...dill-0.3.5) --- updated-dependencies: - dependency-name: dill dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6bd2dd113b..edd85717c8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -32,7 +32,7 @@ cryptography==37.0.2 # trustme decorator==5.1.1 # via ipython -dill==0.3.4 +dill==0.3.5 # via pylint flake8==4.0.1 # via -r test-requirements.in From cd11213084c80a22fa87c5d80b08d46c88876ddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 21 May 2022 00:38:23 +0300 Subject: [PATCH 1094/1498] Changed wording in MultiError deprecation messages --- trio/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index d3be8fcb4b..a7ddeb539d 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -119,13 +119,19 @@ value=_MultiError, version="0.21.0", issue=2211, - instead="BaseExceptionGroup (py3.11+) or exceptiongroup.BaseExceptionGroup", + instead=( + "BaseExceptionGroup (on Python 3.11 and later) or " + "exceptiongroup.BaseExceptionGroup (earlier versions)" + ), ), "NonBaseMultiError": _deprecate.DeprecatedAttribute( value=_NonBaseMultiError, version="0.21.0", issue=2211, - instead="ExceptionGroup (py3.11+) or exceptiongroup.ExceptionGroup", + instead=( + "ExceptionGroup (on Python 3.11 and later) or " + "exceptiongroup.ExceptionGroup (earlier versions)" + ), ), } From 7088be4612b2506913b2df4cf7a097c9668bbd52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 21 May 2022 00:40:13 +0300 Subject: [PATCH 1095/1498] Added missing install dependency on exceptiongroup --- setup.py | 1 + test-requirements.in | 3 ++- test-requirements.txt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 732fa7acd7..f6a9f8f4b5 100644 --- a/setup.py +++ b/setup.py @@ -89,6 +89,7 @@ # cffi 1.14 fixes memory leak inside ffi.getwinerror() # cffi is required on Windows, except on PyPy where it is built-in "cffi>=1.14; os_name == 'nt' and implementation_name != 'pypy'", + "exceptiongroup; python_version < '3.11'" ], # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: diff --git a/test-requirements.in b/test-requirements.in index 024ea35b21..80abb7ebe2 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -8,7 +8,6 @@ trustme # for the ssl tests pylint # for pylint finding all symbols tests jedi # for jedi code completion tests cryptography>=36.0.0 # 35.0.0 is transitive but fails -exceptiongroup # for catch() # Tools black; implementation_name == "cpython" @@ -30,3 +29,5 @@ async_generator >= 1.9 idna outcome sniffio +exceptiongroup; python_version < "3.11" + diff --git a/test-requirements.txt b/test-requirements.txt index 8902a6a1c9..5857d92e7e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -34,7 +34,7 @@ decorator==5.1.1 # via ipython dill==0.3.4 # via pylint -exceptiongroup==1.0.0rc5 +exceptiongroup==1.0.0rc7 # via -r test-requirements.in flake8==4.0.1 # via -r test-requirements.in From c56fa4fc5ff39f9640287b8e1f023611e29e0524 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 May 2022 10:20:06 +0000 Subject: [PATCH 1096/1498] Bump dill from 0.3.5 to 0.3.5.1 Bumps [dill](https://github.com/uqfoundation/dill) from 0.3.5 to 0.3.5.1. - [Release notes](https://github.com/uqfoundation/dill/releases) - [Commits](https://github.com/uqfoundation/dill/compare/dill-0.3.5...dill-0.3.5.1) --- updated-dependencies: - dependency-name: dill dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index edd85717c8..6897469224 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -32,7 +32,7 @@ cryptography==37.0.2 # trustme decorator==5.1.1 # via ipython -dill==0.3.5 +dill==0.3.5.1 # via pylint flake8==4.0.1 # via -r test-requirements.in From d336a6562ee0f4ee7b2b82c1cc1e2e44cfd5c476 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 May 2022 10:21:04 +0000 Subject: [PATCH 1097/1498] Bump immutables from 0.17 to 0.18 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.17 to 0.18. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.17...v0.18) --- updated-dependencies: - dependency-name: immutables dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 8454af954b..f063dc843f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -34,7 +34,7 @@ idna==3.3 # requests imagesize==1.3.0 # via sphinx -immutables==0.17 +immutables==0.18 # via -r docs-requirements.in incremental==21.3.0 # via towncrier From 98c7897830d59b74dc7e447ce233fa91dc99230c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 May 2022 10:14:22 +0000 Subject: [PATCH 1098/1498] Bump mypy from 0.950 to 0.960 Bumps [mypy](https://github.com/python/mypy) from 0.950 to 0.960. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.950...v0.960) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6897469224..4494afb7c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -58,7 +58,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.950 ; implementation_name == "cpython" +mypy==0.960 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From 18ca12639125ae90e9e363925a046b22f1d2b164 Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Wed, 1 Jun 2022 10:25:30 +0200 Subject: [PATCH 1099/1498] Python raises a `TypeError` if you try to (re-)install a C signal handler. Closes #2333. --- newsfragments/2333.bugfix.rst | 1 + trio/_util.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/2333.bugfix.rst diff --git a/newsfragments/2333.bugfix.rst b/newsfragments/2333.bugfix.rst new file mode 100644 index 0000000000..efab3a6380 --- /dev/null +++ b/newsfragments/2333.bugfix.rst @@ -0,0 +1 @@ +Python raises a `TypeError` if you try to (re-)install a C signal handler. diff --git a/trio/_util.py b/trio/_util.py index b5a2ceabdc..539fc540ff 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -78,7 +78,7 @@ def is_main_thread(): try: signal.signal(signal.SIGINT, signal.getsignal(signal.SIGINT)) return True - except ValueError: + except (TypeError, ValueError): return False From ac72b600554d7395c0c4f4fba4f090cf9adfed39 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 7 Jun 2022 09:43:03 +0400 Subject: [PATCH 1100/1498] Make setuptools license field SPDX-compliant https://spdx.org/licenses/ https://spdx.github.io/spdx-spec/SPDX-license-expressions/#d42-disjunctive-or-operator --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 732fa7acd7..7021664c8d 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ author="Nathaniel J. Smith", author_email="njs@pobox.com", url="https://github.com/python-trio/trio", - license="MIT -or- Apache License 2.0", + license="MIT OR Apache-2.0", packages=find_packages(), install_requires=[ "attrs >= 19.2.0", # for eq From 6676d74e20e0e0f112d54b359835ac2f25de5cc0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jun 2022 10:20:59 +0000 Subject: [PATCH 1101/1498] Bump mypy from 0.960 to 0.961 Bumps [mypy](https://github.com/python/mypy) from 0.960 to 0.961. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.960...v0.961) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4494afb7c7..c6bbdd4ebc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -58,7 +58,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.960 ; implementation_name == "cpython" +mypy==0.961 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From 7cb3ef9e372df9b6be2690cff27a887769aec154 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jun 2022 10:22:17 +0000 Subject: [PATCH 1102/1498] Bump pylint from 2.13.9 to 2.14.1 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.13.9 to 2.14.1. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.13.9...v2.14.1) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 4494afb7c7..595105d0fa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.13.9 +pylint==2.14.1 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in @@ -120,6 +120,8 @@ tomli==2.0.1 # mypy # pylint # pytest +tomlkit==0.11.0 + # via pylint traitlets==5.2.1.post0 # via # ipython From de7a097733981d1f09f370c1ae6b2511e155822e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jun 2022 17:29:18 +0000 Subject: [PATCH 1103/1498] Bump traitlets from 5.2.1.post0 to 5.2.2.post1 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.2.1.post0 to 5.2.2.post1. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/commits) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 53a193e0a5..e5e4679d40 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -122,7 +122,7 @@ tomli==2.0.1 # pytest tomlkit==0.11.0 # via pylint -traitlets==5.2.1.post0 +traitlets==5.2.2.post1 # via # ipython # matplotlib-inline From 72f1c4d176be6b8c93dce00185e618b716a15668 Mon Sep 17 00:00:00 2001 From: lojack5 <1458329+lojack5@users.noreply.github.com> Date: Tue, 7 Jun 2022 11:48:21 -0600 Subject: [PATCH 1104/1498] replace `async_cm` decorator: Explicitly subclass from a mixin providing the same functionality, rather than setting class attributes with the decorator. This plays nicer with type- checkers and IDEs. --- trio/_sync.py | 21 +++++---------------- trio/tests/test_sync.py | 11 ++++------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/trio/_sync.py b/trio/_sync.py index bf5f5d5e0b..c8c6d0dfe5 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -87,22 +87,15 @@ def statistics(self): return _EventStatistics(tasks_waiting=len(self._tasks)) -def async_cm(cls): +class AsyncContextManagerMixin: @enable_ki_protection async def __aenter__(self): await self.acquire() - __aenter__.__qualname__ = cls.__qualname__ + ".__aenter__" - cls.__aenter__ = __aenter__ - @enable_ki_protection async def __aexit__(self, *args): self.release() - __aexit__.__qualname__ = cls.__qualname__ + ".__aexit__" - cls.__aexit__ = __aexit__ - return cls - @attr.s(frozen=True) class _CapacityLimiterStatistics: @@ -112,8 +105,7 @@ class _CapacityLimiterStatistics: tasks_waiting = attr.ib() -@async_cm -class CapacityLimiter(metaclass=Final): +class CapacityLimiter(AsyncContextManagerMixin, metaclass=Final): """An object for controlling access to a resource with limited capacity. Sometimes you need to put a limit on how many tasks can do something at @@ -355,8 +347,7 @@ def statistics(self): ) -@async_cm -class Semaphore(metaclass=Final): +class Semaphore(AsyncContextManagerMixin, metaclass=Final): """A `semaphore `__. A semaphore holds an integer value, which can be incremented by @@ -485,9 +476,8 @@ class _LockStatistics: tasks_waiting = attr.ib() -@async_cm @attr.s(eq=False, hash=False, repr=False) -class _LockImpl: +class _LockImpl(AsyncContextManagerMixin): _lot = attr.ib(factory=ParkingLot, init=False) _owner = attr.ib(default=None, init=False) @@ -659,8 +649,7 @@ class _ConditionStatistics: lock_statistics = attr.ib() -@async_cm -class Condition(metaclass=Final): +class Condition(AsyncContextManagerMixin, metaclass=Final): """A classic `condition variable `__, similar to :class:`threading.Condition`. diff --git a/trio/tests/test_sync.py b/trio/tests/test_sync.py index 229dea301c..33f79c4df2 100644 --- a/trio/tests/test_sync.py +++ b/trio/tests/test_sync.py @@ -401,15 +401,14 @@ async def waiter(i): assert c.locked() -from .._sync import async_cm +from .._sync import AsyncContextManagerMixin from .._channel import open_memory_channel # Three ways of implementing a Lock in terms of a channel. Used to let us put # the channel through the generic lock tests. -@async_cm -class ChannelLock1: +class ChannelLock1(AsyncContextManagerMixin): def __init__(self, capacity): self.s, self.r = open_memory_channel(capacity) for _ in range(capacity - 1): @@ -425,8 +424,7 @@ def release(self): self.r.receive_nowait() -@async_cm -class ChannelLock2: +class ChannelLock2(AsyncContextManagerMixin): def __init__(self): self.s, self.r = open_memory_channel(10) self.s.send_nowait(None) @@ -441,8 +439,7 @@ def release(self): self.s.send_nowait(None) -@async_cm -class ChannelLock3: +class ChannelLock3(AsyncContextManagerMixin): def __init__(self): self.s, self.r = open_memory_channel(0) # self.acquired is true when one task acquires the lock and From 077e8fc3f1634b42ef5024814898439d8a8430d4 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 1 Jun 2022 09:56:26 +0400 Subject: [PATCH 1105/1498] Bump version to 0.21.0 --- docs/source/history.rst | 17 +++++++++++++++++ newsfragments/2210.removal.rst | 1 - trio/_version.py | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) delete mode 100644 newsfragments/2210.removal.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 3fb51f346a..414a5c518c 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,23 @@ Release history .. towncrier release notes start +Trio 0.21.0 (2022-06-07) +---------------------------- + +Features +~~~~~~~~ + +- Trio now supports Python 3.11. (`#2270 + `__, `#2318 + `__, `#2319 + `__) + +Deprecations and Removals +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Remove support for Python 3.6. (`#2210 `__) + + Trio 0.20.0 (2022-02-21) ------------------------ diff --git a/newsfragments/2210.removal.rst b/newsfragments/2210.removal.rst deleted file mode 100644 index aa7c74034c..0000000000 --- a/newsfragments/2210.removal.rst +++ /dev/null @@ -1 +0,0 @@ -Remove support for Python 3.6. \ No newline at end of file diff --git a/trio/_version.py b/trio/_version.py index 5a02c2e464..d05f3b85fa 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.20.0+dev" +__version__ = "0.21.0" From 3e4b5007aa7f8962590f97633ffdc2aec9fe1693 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 7 Jun 2022 22:29:43 +0400 Subject: [PATCH 1106/1498] Bump version to 0.21.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index d05f3b85fa..d722bdc7f1 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.21.0" +__version__ = "0.21.0+dev" From e87de6c1900495d3872a7cb5a20d646fcc8f812b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 7 Jun 2022 22:11:55 +0300 Subject: [PATCH 1107/1498] Updated trio version in deprecation notes --- trio/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index a7ddeb539d..7a1e16dfb1 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -117,7 +117,7 @@ ), "MultiError": _deprecate.DeprecatedAttribute( value=_MultiError, - version="0.21.0", + version="0.22.0", issue=2211, instead=( "BaseExceptionGroup (on Python 3.11 and later) or " @@ -126,7 +126,7 @@ ), "NonBaseMultiError": _deprecate.DeprecatedAttribute( value=_NonBaseMultiError, - version="0.21.0", + version="0.22.0", issue=2211, instead=( "ExceptionGroup (on Python 3.11 and later) or " From 6b5e4f6f06cccd147cb7ad76831b8f7e978fd3ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 7 Jun 2022 22:13:20 +0300 Subject: [PATCH 1108/1498] Fixed formatting error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2375371a08..d2f09d5055 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ # cffi 1.14 fixes memory leak inside ffi.getwinerror() # cffi is required on Windows, except on PyPy where it is built-in "cffi>=1.14; os_name == 'nt' and implementation_name != 'pypy'", - "exceptiongroup; python_version < '3.11'" + "exceptiongroup; python_version < '3.11'", ], # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: From cfa1b2dfb3f7cf70a2b24ba41c9d28a5fa80603f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jun 2022 10:37:13 +0000 Subject: [PATCH 1109/1498] Bump babel from 2.9.1 to 2.10.2 Bumps [babel](https://github.com/python-babel/babel) from 2.9.1 to 2.10.2. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES.rst) - [Commits](https://github.com/python-babel/babel/compare/v2.9.1...v2.10.2) --- updated-dependencies: - dependency-name: babel dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f063dc843f..b56814e5ec 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -12,7 +12,7 @@ attrs==21.4.0 # via # -r docs-requirements.in # outcome -babel==2.9.1 +babel==2.10.2 # via sphinx certifi==2022.5.18.1 # via requests From 682200bddd81dba8e99fbc86b30b34ecbbc2ca26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jun 2022 10:38:05 +0000 Subject: [PATCH 1110/1498] Bump astroid from 2.11.5 to 2.11.6 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.11.5 to 2.11.6. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.11.5...v2.11.6) --- updated-dependencies: - dependency-name: astroid dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e5e4679d40..ce9c09026a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.11.5 +astroid==2.11.6 # via pylint async-generator==1.10 # via -r test-requirements.in From a126479365e7ad0d3af9339347210d6a0e96667b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jun 2022 10:58:24 +0000 Subject: [PATCH 1111/1498] Bump outcome from 1.1.0 to 1.2.0 Bumps [outcome](https://github.com/python-trio/outcome) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/python-trio/outcome/releases) - [Commits](https://github.com/python-trio/outcome/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: outcome dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index f063dc843f..2cac38697e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -44,7 +44,7 @@ jinja2==3.0.3 # towncrier markupsafe==2.1.1 # via jinja2 -outcome==1.1.0 +outcome==1.2.0 # via -r docs-requirements.in packaging==21.3 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index e5e4679d40..f875b32fe1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -65,7 +65,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # -r test-requirements.in # black # mypy -outcome==1.1.0 +outcome==1.2.0 # via -r test-requirements.in packaging==21.3 # via pytest From 6c0cd9ca4db7f83b6c3277945b56a5241e98665d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jun 2022 10:23:43 +0000 Subject: [PATCH 1112/1498] Bump pylint from 2.14.1 to 2.14.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.14.1 to 2.14.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.14.1...v2.14.2) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7ac39203ee..70d2f268d7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.14.1 +pylint==2.14.2 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From 79cbd613a00f9a5f4ad2d0fd34a50e6411e33ff4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jun 2022 10:22:17 +0000 Subject: [PATCH 1113/1498] Bump certifi from 2022.5.18.1 to 2022.6.15 Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.5.18.1 to 2022.6.15. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.05.18.1...2022.06.15) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 90fefb3396..864f1933b0 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==21.4.0 # outcome babel==2.10.2 # via sphinx -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests charset-normalizer==2.0.12 # via requests From 69bca2e5df36c121978c79170bfcaecd4162b047 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jun 2022 10:22:45 +0000 Subject: [PATCH 1114/1498] Bump babel from 2.10.2 to 2.10.3 Bumps [babel](https://github.com/python-babel/babel) from 2.10.2 to 2.10.3. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES.rst) - [Commits](https://github.com/python-babel/babel/compare/v2.10.2...v2.10.3) --- updated-dependencies: - dependency-name: babel dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 90fefb3396..646514d5cd 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -12,7 +12,7 @@ attrs==21.4.0 # via # -r docs-requirements.in # outcome -babel==2.10.2 +babel==2.10.3 # via sphinx certifi==2022.5.18.1 # via requests From 4e1c8d664a9d0c55f970daed29eced277db5e78e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:39:26 +0000 Subject: [PATCH 1115/1498] Bump requests from 2.27.1 to 2.28.0 Bumps [requests](https://github.com/psf/requests) from 2.27.1 to 2.28.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.27.1...v2.28.0) --- updated-dependencies: - dependency-name: requests dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 483466a697..fa2ba66793 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -54,7 +54,7 @@ pyparsing==3.0.9 # via packaging pytz==2022.1 # via babel -requests==2.27.1 +requests==2.28.0 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in From 5d9ba653a6c2e072bec2be87c3ba55d8dba57c6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jun 2022 10:26:35 +0000 Subject: [PATCH 1116/1498] Bump traitlets from 5.2.2.post1 to 5.3.0 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.2.2.post1 to 5.3.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.2.2.post1...5.3.0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 70d2f268d7..7f7ce0a071 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -122,7 +122,7 @@ tomli==2.0.1 # pytest tomlkit==0.11.0 # via pylint -traitlets==5.2.2.post1 +traitlets==5.3.0 # via # ipython # matplotlib-inline From c84e0d39623620bbd84a44fc68856d6ac9c5cd91 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sun, 19 Jun 2022 12:39:27 -0400 Subject: [PATCH 1117/1498] replace Queue with SimpleQueue in thread dispatch Considering this queue exists just to block and receive one object and be destroyed, the more lightweight object should be preferred. It could have been used in the first place except that SimpleQueue was not implemented until python 3.7, and we have only recently dropped 3.6 support. --- trio/_threads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_threads.py b/trio/_threads.py index adad6343d6..0cec0cc237 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -240,7 +240,7 @@ def _run_fn_as_system_task(cb, fn, *args, context, trio_token=None): else: raise RuntimeError("this is a blocking function; call it from a thread") - q = stdlib_queue.Queue() + q = stdlib_queue.SimpleQueue() trio_token.run_sync_soon(context.run, cb, q, fn, args) return q.get().unwrap() From 2953850e10a26bab0d0b7f6816a5896c91955388 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sun, 19 Jun 2022 12:29:33 -0400 Subject: [PATCH 1118/1498] Guard rail cancellable argument of thread dispatch The cancellable argument is typed bool but at runtime will work with any object with truthiness. However, if someone passes something like an numpy ndarray, with no notion of truthiness, the error is raised in an abort function and causes a TrioInternalError. There are other unguarded uses of user arguments in abort functions, but they are in the lowlevel namespace, and aren't expected to have robust guardrails. --- trio/_threads.py | 1 + trio/tests/test_threads.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/trio/_threads.py b/trio/_threads.py index 0cec0cc237..c8cab2ee76 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -138,6 +138,7 @@ async def to_thread_run_sync(sync_fn, *args, cancellable=False, limiter=None): """ await trio.lowlevel.checkpoint_if_cancelled() + cancellable = bool(cancellable) # raise early if cancellable.__bool__ raises if limiter is None: limiter = current_default_thread_limiter() diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 02f1b7f0c2..5bbad1b940 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -738,3 +738,16 @@ async def test_trio_token_weak_referenceable(): assert isinstance(token, TrioToken) weak_reference = weakref.ref(token) assert token is weak_reference() + + +async def test_unsafe_cancellable_kwarg(): + + # This is a stand in for a numpy ndarray or other objects + # that (maybe surprisingly) lack a notion of truthiness + class BadBool: + def __bool__(self): + raise NotImplementedError + + with pytest.raises(NotImplementedError): + with _core.CancelScope(deadline=_core.current_time() + 0.01): + await to_thread_run_sync(time.sleep(1), cancellable=BadBool()) From 0cadb79986e2dd242dbabab4b10e468c2eaf42ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 10:22:22 +0000 Subject: [PATCH 1119/1498] Bump pylint from 2.14.2 to 2.14.3 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.14.2 to 2.14.3. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.14.2...v2.14.3) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7f7ce0a071..da7a2e84c9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.10.0 # via ipython -pylint==2.14.2 +pylint==2.14.3 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From cf5e99e965b506842b5a2279f2b5ffb3908b1b60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 10:28:02 +0000 Subject: [PATCH 1120/1498] Bump cffi from 1.15.0 to 1.15.1 Bumps [cffi](http://cffi.readthedocs.org) from 1.15.0 to 1.15.1. --- updated-dependencies: - dependency-name: cffi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index da7a2e84c9..15e93c9635 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -19,7 +19,7 @@ backcall==0.2.0 # via ipython black==22.3.0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.15.0 +cffi==1.15.1 # via cryptography click==8.1.3 # via black From 28cbbc7ee78b71641887cb873576533f810b1e14 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 3 Jul 2022 14:36:15 +1000 Subject: [PATCH 1121/1498] docs: fix simple typo, subproces -> subprocess There is a small typo in trio/_subprocess.py. Should read `subprocess` rather than `subproces`. --- trio/_subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_subprocess.py b/trio/_subprocess.py index aef7bcbedb..2bb0adcbdc 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -512,7 +512,7 @@ async def run_process( the subprocess's output for further processing, then use ``stdout=subprocess.PIPE`` and then make sure to read the data out of the `Process.stdout` stream. If you want to capture stderr separately, use ``stderr=subprocess.PIPE``. If you want to capture - both, but mixed together in the correct order, use ``stdout=subproces.PIPE, + both, but mixed together in the correct order, use ``stdout=subprocess.PIPE, stderr=subprocess.STDOUT``. **Error checking:** If the subprocess exits with a nonzero status From 6f14debd9bae64c8986c23a6bc54b0b940599759 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 Jul 2022 08:52:49 +0000 Subject: [PATCH 1122/1498] Bump black from 22.3.0 to 22.6.0 Bumps [black](https://github.com/psf/black) from 22.3.0 to 22.6.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.3.0...22.6.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 15e93c9635..38e62ad22f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==21.4.0 # pytest backcall==0.2.0 # via ipython -black==22.3.0 ; implementation_name == "cpython" +black==22.6.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.1 # via cryptography From 11e19ca8c6b6e700277a9116571abe7b122c5325 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jul 2022 10:25:49 +0000 Subject: [PATCH 1123/1498] Bump typing-extensions from 4.2.0 to 4.3.0 Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/python/typing_extensions/releases) - [Changelog](https://github.com/python/typing_extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/python/typing_extensions/compare/4.2.0...4.3.0) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 38e62ad22f..e57fa3973b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -128,7 +128,7 @@ traitlets==5.3.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typing-extensions==4.2.0 ; implementation_name == "cpython" +typing-extensions==4.3.0 ; implementation_name == "cpython" # via # -r test-requirements.in # mypy From e38fc3b2bdd840c684c4433e8929fafb5524ffc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jul 2022 10:27:56 +0000 Subject: [PATCH 1124/1498] Bump imagesize from 1.3.0 to 1.4.1 Bumps [imagesize](https://github.com/shibukawa/imagesize_py) from 1.3.0 to 1.4.1. - [Release notes](https://github.com/shibukawa/imagesize_py/releases) - [Commits](https://github.com/shibukawa/imagesize_py/compare/1.3.0...1.4.1) --- updated-dependencies: - dependency-name: imagesize dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index fa2ba66793..3d0904f387 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -32,7 +32,7 @@ idna==3.3 # via # -r docs-requirements.in # requests -imagesize==1.3.0 +imagesize==1.4.1 # via sphinx immutables==0.18 # via -r docs-requirements.in From 955139d3a49e4f6730554935a7e4ba9b0db7c76e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jul 2022 10:49:19 +0000 Subject: [PATCH 1125/1498] Bump requests from 2.28.0 to 2.28.1 Bumps [requests](https://github.com/psf/requests) from 2.28.0 to 2.28.1. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.0...v2.28.1) --- updated-dependencies: - dependency-name: requests dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 3d0904f387..51e57523e8 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -54,7 +54,7 @@ pyparsing==3.0.9 # via packaging pytz==2022.1 # via babel -requests==2.28.0 +requests==2.28.1 # via sphinx sniffio==1.2.0 # via -r docs-requirements.in From 9ff7a4550a0eddbdcf27a86f7133a3c15d2e925c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 5 Jul 2022 01:40:51 +0300 Subject: [PATCH 1126/1498] Use trio's own deprecation mechanism --- trio/_core/_multierror.py | 9 ++++----- trio/_core/tests/test_multierror.py | 22 ++++++++++++++++------ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 0412b4c214..7305dff422 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -4,6 +4,8 @@ import attr +from trio._deprecate import warn_deprecated + if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup, ExceptionGroup @@ -233,12 +235,9 @@ def filter(cls, handler, root_exc): ``handler`` returned None for all the inputs, returns None. """ - warnings.warn( - "MultiError.filter() has been deprecated. " - "Use the .split() method instead.", - DeprecationWarning, + warn_deprecated( + "MultiError.filter()", "0.22.0", instead="MultiError.split()", issue=2211 ) - return _filter_impl(handler, root_exc) @classmethod diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 9b8d7356cb..1eaab6d4b3 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -12,6 +12,7 @@ import re from .._multierror import MultiError, concat_tb, NonBaseMultiError +from ... import TrioDeprecationWarning from ..._core import open_nursery if sys.version_info < (3, 11): @@ -144,7 +145,9 @@ def handle_ValueError(exc): else: return exc - filtered_excs = pytest.deprecated_call(MultiError.filter, handle_ValueError, excs) + with pytest.warns(TrioDeprecationWarning): + filtered_excs = MultiError.filter(handle_ValueError, excs) + assert isinstance(filtered_excs, NotHashableException) @@ -189,7 +192,9 @@ def null_handler(exc): m = make_tree() assert_tree_eq(m, m) - assert pytest.deprecated_call(MultiError.filter, null_handler, m) is m + with pytest.warns(TrioDeprecationWarning): + assert MultiError.filter(null_handler, m) is m + assert_tree_eq(m, make_tree()) # Make sure we don't pick up any detritus if run in a context where @@ -198,7 +203,8 @@ def null_handler(exc): try: raise ValueError except ValueError: - assert pytest.deprecated_call(MultiError.filter, null_handler, m) is m + with pytest.warns(TrioDeprecationWarning): + assert MultiError.filter(null_handler, m) is m assert_tree_eq(m, make_tree()) def simple_filter(exc): @@ -208,7 +214,9 @@ def simple_filter(exc): return RuntimeError() return exc - new_m = pytest.deprecated_call(MultiError.filter, simple_filter, make_tree()) + with pytest.warns(TrioDeprecationWarning): + new_m = MultiError.filter(simple_filter, make_tree()) + assert isinstance(new_m, MultiError) assert len(new_m.exceptions) == 2 # was: [[ValueError, KeyError], NameError] @@ -250,7 +258,8 @@ def filter_NameError(exc): return exc m = make_tree() - new_m = pytest.deprecated_call(MultiError.filter, filter_NameError, m) + with pytest.warns(TrioDeprecationWarning): + new_m = MultiError.filter(filter_NameError, m) # with the NameError gone, the other branch gets promoted assert new_m is m.exceptions[0] @@ -258,7 +267,8 @@ def filter_NameError(exc): def filter_all(exc): return None - assert pytest.deprecated_call(MultiError.filter, filter_all, make_tree()) is None + with pytest.warns(TrioDeprecationWarning): + assert MultiError.filter(filter_all, make_tree()) is None def test_MultiError_catch(): From 2140b70debdcfd453235708c654b6592f11673a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 5 Jul 2022 01:41:14 +0300 Subject: [PATCH 1127/1498] Added comment on _collapse=False in derive() --- trio/_core/_multierror.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 7305dff422..31a7150d6e 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -217,6 +217,8 @@ def __repr__(self): return "".format(self) def derive(self, __excs): + # We use _collapse=False here to get ExceptionGroup semantics, since derive() + # is part of the PEP 654 API return MultiError(__excs, _collapse=False) @classmethod From 564c0668aa4dc1459fed9a1f739f0c8e5fac26f1 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 6 Jul 2022 18:07:22 +0100 Subject: [PATCH 1128/1498] clarify the conduct rule about quoting interactions --- docs/source/code-of-conduct.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/code-of-conduct.rst b/docs/source/code-of-conduct.rst index 8bd6b24f42..fb0d003065 100644 --- a/docs/source/code-of-conduct.rst +++ b/docs/source/code-of-conduct.rst @@ -78,7 +78,7 @@ Examples of unacceptable behavior by participants include: electronic address, without explicit permission. This includes any sort of "outing" of any aspect of someone's identity without their consent. -- Publishing private screenshots or quotes of interactions in the +- Publishing screenshots or quotes of private interactions in the context of this project without all quoted users' *explicit* consent. - Publishing of private communication that doesn't have to do with reporting harassment. From 89086c478bef3090ddfc5252bc828773d42764b8 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 4 Jun 2022 19:23:16 -0400 Subject: [PATCH 1129/1498] refresh pinned requirements with pip-compile It looks like several transitive requirements have changed and that dependabot has been dropping the ball on some updates. --- docs-requirements.in | 5 ++++- docs-requirements.txt | 20 ++++++++++++++++---- test-requirements.in | 2 +- test-requirements.txt | 34 ++++++++++++++++++++-------------- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/docs-requirements.in b/docs-requirements.in index 23a1b0f652..c587c9b3f9 100644 --- a/docs-requirements.in +++ b/docs-requirements.in @@ -1,5 +1,8 @@ # RTD is currently installing 1.5.3, which has a bug in :lineno-match: -sphinx >= 1.7.0 +# sphinx-3.4 causes warnings about some trio._abc classes: GH#2338 +sphinx >= 1.7.0, < 3.4 +# jinja2-3.1 causes importerror with sphinx<4.0 +jinja2 < 3.1 sphinx_rtd_theme sphinxcontrib-trio towncrier diff --git a/docs-requirements.txt b/docs-requirements.txt index 51e57523e8..c1535622a8 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --output-file docs-requirements.txt docs-requirements.in +# pip-compile docs-requirements.in # alabaster==0.7.12 # via sphinx @@ -16,7 +16,9 @@ babel==2.10.3 # via sphinx certifi==2022.6.15 # via requests -charset-normalizer==2.0.12 +cffi==1.15.1 ; os_name == "nt" + # via -r docs-requirements.in +charset-normalizer==2.1.0 # via requests click==8.1.3 # via @@ -24,6 +26,10 @@ click==8.1.3 # towncrier click-default-group==1.2.2 # via towncrier +colorama==0.4.5 + # via + # click + # sphinx docutils==0.17.1 # via # sphinx @@ -40,6 +46,7 @@ incremental==21.3.0 # via towncrier jinja2==3.0.3 # via + # -r docs-requirements.in # sphinx # towncrier markupsafe==2.1.1 @@ -48,7 +55,9 @@ outcome==1.2.0 # via -r docs-requirements.in packaging==21.3 # via sphinx -pygments==2.10.0 +pycparser==2.21 + # via cffi +pygments==2.12.0 # via sphinx pyparsing==3.0.9 # via packaging @@ -89,3 +98,6 @@ towncrier==21.9.0 # via -r docs-requirements.in urllib3==1.26.9 # via requests + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/test-requirements.in b/test-requirements.in index 4d15de8e4f..3478604218 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -2,7 +2,7 @@ pytest >= 5.0 # for faulthandler in core pytest-cov >= 2.6.0 # ipython 7.x is the last major version supporting Python 3.7 -ipython ~= 7.31 # for the IPython traceback integration tests +ipython < 7.32 # for the IPython traceback integration tests pyOpenSSL # for the ssl tests trustme # for the ssl tests pylint # for pylint finding all symbols tests diff --git a/test-requirements.txt b/test-requirements.txt index e57fa3973b..17e6697e83 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --output-file=test-requirements.txt test-requirements.in +# pip-compile test-requirements.in # astor==0.8.1 # via -r test-requirements.in @@ -10,6 +10,8 @@ astroid==2.11.6 # via pylint async-generator==1.10 # via -r test-requirements.in +atomicwrites==1.4.0 + # via pytest attrs==21.4.0 # via # -r test-requirements.in @@ -19,13 +21,21 @@ backcall==0.2.0 # via ipython black==22.6.0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.15.1 - # via cryptography +cffi==1.15.1 ; os_name == "nt" + # via + # -r test-requirements.in + # cryptography click==8.1.3 # via black -coverage[toml]==6.3.3 +colorama==0.4.5 + # via + # click + # ipython + # pylint + # pytest +coverage[toml]==6.4.1 # via pytest-cov -cryptography==37.0.2 +cryptography==37.0.4 # via # -r test-requirements.in # pyopenssl @@ -73,8 +83,6 @@ parso==0.8.3 # via jedi pathspec==0.9.0 # via black -pexpect==4.8.0 - # via ipython pickleshare==0.7.5 # via ipython platformdirs==2.5.2 @@ -83,10 +91,8 @@ platformdirs==2.5.2 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.29 +prompt-toolkit==3.0.30 # via ipython -ptyprocess==0.7.0 - # via pexpect py==1.11.0 # via pytest pycodestyle==2.8.0 @@ -95,9 +101,9 @@ pycparser==2.21 # via cffi pyflakes==2.4.0 # via flake8 -pygments==2.10.0 +pygments==2.12.0 # via ipython -pylint==2.14.3 +pylint==2.14.4 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in @@ -120,7 +126,7 @@ tomli==2.0.1 # mypy # pylint # pytest -tomlkit==0.11.0 +tomlkit==0.11.1 # via pylint traitlets==5.3.0 # via From 279104e7a663c63fb96d06677c85f2c3949e6236 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jul 2022 10:51:14 +0000 Subject: [PATCH 1130/1498] Bump urllib3 from 1.26.9 to 1.26.10 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.9 to 1.26.10. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.10/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.9...1.26.10) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c1535622a8..73b2e8eaa3 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,8 +16,6 @@ babel==2.10.3 # via sphinx certifi==2022.6.15 # via requests -cffi==1.15.1 ; os_name == "nt" - # via -r docs-requirements.in charset-normalizer==2.1.0 # via requests click==8.1.3 @@ -26,10 +24,6 @@ click==8.1.3 # towncrier click-default-group==1.2.2 # via towncrier -colorama==0.4.5 - # via - # click - # sphinx docutils==0.17.1 # via # sphinx @@ -55,8 +49,6 @@ outcome==1.2.0 # via -r docs-requirements.in packaging==21.3 # via sphinx -pycparser==2.21 - # via cffi pygments==2.12.0 # via sphinx pyparsing==3.0.9 @@ -96,7 +88,7 @@ tomli==2.0.1 # via towncrier towncrier==21.9.0 # via -r docs-requirements.in -urllib3==1.26.9 +urllib3==1.26.10 # via requests # The following packages are considered to be unsafe in a requirements file: From 1da1af457b44f6d28e52535e7df6237cc2fb9eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 5 Jul 2022 01:59:10 +0300 Subject: [PATCH 1131/1498] Only modify MultiErrors in collapse_exception_group() --- trio/_core/_run.py | 4 ++-- trio/_core/tests/test_run.py | 14 -------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 3f2ac9e3a4..c200455b16 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -127,7 +127,7 @@ def collapse_exception_group(excgroup): exceptions = list(excgroup.exceptions) modified = False for i, exc in enumerate(exceptions): - if isinstance(exc, BaseExceptionGroup): + if isinstance(exc, MultiError): new_exc = collapse_exception_group(exc) if new_exc is not exc: modified = True @@ -537,7 +537,7 @@ def _close(self, exc, collapse=True): if matched: self.cancelled_caught = True - if collapse and isinstance(exc, BaseExceptionGroup): + if collapse and isinstance(exc, MultiError): exc = collapse_exception_group(exc) self._cancel_status.close() diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index a3e0e3e28f..6eeaee9aac 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1877,20 +1877,6 @@ def handle(exc): assert result == [[0, 0], [1, 1]] -async def test_nursery_collapse_exceptions(): - # Test that exception groups containing only a single exception are - # recursively collapsed - async def fail(): - raise ExceptionGroup("fail", [ValueError()]) - - try: - async with _core.open_nursery() as nursery: - nursery.start_soon(fail) - raise StopIteration - except MultiError as e: - assert tuple(map(type, e.exceptions)) == (StopIteration, ValueError) - - async def test_traceback_frame_removal(): async def my_child_task(): raise KeyError() From f9e11fce6c688f6e41d6ccb5e17773bb2cdc4024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 5 Jul 2022 02:11:10 +0300 Subject: [PATCH 1132/1498] Retain traceback frames from the MultiError itself --- trio/_core/_run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index c200455b16..10a77d56c4 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -28,7 +28,7 @@ KIManager, enable_ki_protection, ) -from ._multierror import MultiError +from ._multierror import MultiError, concat_tb from ._traps import ( Abort, wait_task_rescheduled, @@ -134,6 +134,9 @@ def collapse_exception_group(excgroup): exceptions[i] = new_exc if len(exceptions) == 1: + exceptions[0].__traceback__ = concat_tb( + exceptions[0].__traceback__, excgroup.__traceback__ + ) return exceptions[0] elif modified: return excgroup.derive(exceptions) From 1d99b232d20ec2ddc65a3f8d7577bc8cdea5b898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 9 Jul 2022 15:37:03 +0300 Subject: [PATCH 1133/1498] Improved prevention of MultiError double initialization If the instance has not been initialized yet, then accessing self.exceptions would cause an AttributeError. We detect that and only call the superclass initializer if that exception is not raised. --- trio/_core/_multierror.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 31a7150d6e..b144d5f43f 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -179,11 +179,10 @@ class MultiError(BaseExceptionGroup): """ def __init__(self, exceptions, *, _collapse=True): - # Avoid recursion when exceptions[0] returned by __new__() happens - # to be a MultiError and subsequently __init__() is called. - if _collapse and hasattr(self, "_exceptions"): - # __init__ was already called on this object - assert len(exceptions) == 1 and exceptions[0] is self + # Avoid double initialization when _collapse is True and exceptions[0] returned + # by __new__() happens to be a MultiError and subsequently __init__() is called. + if _collapse and getattr(self, "exceptions", None) is not None: + # This exception was already initialized. return super().__init__("multiple tasks failed", exceptions) From 2591808753ccd53b814ba51eb12ccd55379445a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sat, 9 Jul 2022 17:33:09 +0300 Subject: [PATCH 1134/1498] Reworded the comments in the exception group handling examples --- docs/source/reference-core.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index bf9fd3202d..399bd7a13c 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -723,9 +723,9 @@ clause was introduced in Python 3.11 (:pep:`654`). Here's how it works:: nursery.start_soon(broken1) nursery.start_soon(broken2) except* KeyError: - ... # handle each KeyError + ... # handle the KeyErrors except* IndexError: - ... # handle each IndexError + ... # handle the IndexErrors But what if you can't use ``except*`` just yet? Well, for that there is the handy exceptiongroup_ library which lets you approximate this behavior with exception handler @@ -733,11 +733,11 @@ callbacks:: from exceptiongroup import catch - def handle_keyerror(exc): - ... # handle each KeyError + def handle_keyerrors(excgroup): + ... # handle the KeyErrors - def handle_indexerror(exc): - ... # handle each IndexError + def handle_indexerrors(excgroup): + ... # handle the IndexErrors with catch({ KeyError: handle_keyerror, From 8c77459c6f848e89cc8c859d8cfd2f443988dcda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Fri, 15 Jul 2022 12:13:04 +0300 Subject: [PATCH 1135/1498] Made all references to (Base)ExceptionGroup into links --- docs/source/reference-core.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 399bd7a13c..68a042ad79 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -711,9 +711,9 @@ limitation. Consider code like:: ``broken1`` raises ``KeyError``. ``broken2`` raises ``IndexError``. Obviously ``parent`` should raise some error, but -what? The answer is that both exceptions are grouped in an `ExceptionGroup`. -The `ExceptionGroup` and its parent class `BaseExceptionGroup` are used to encapsulate -multiple exceptions being raised at once. +what? The answer is that both exceptions are grouped in an :exc:`ExceptionGroup`. +:exc:`ExceptionGroup` and its parent class :exc:`BaseExceptionGroup` are used to +encapsulate multiple exceptions being raised at once. To catch individual exceptions encapsulated in an exception group, the ``except*`` clause was introduced in Python 3.11 (:pep:`654`). Here's how it works:: @@ -751,10 +751,10 @@ callbacks:: the same with handler callbacks as long as you declare those variables ``nonlocal``. For reasons of backwards compatibility, nurseries raise ``trio.MultiError`` and -``trio.NonBaseMultiError`` which inherit from `BaseExceptionGroup` and `ExceptionGroup`, -respectively. Users should refrain from attempting to raise or catch the Trio specific -exceptions themselves, and treat them as if they were standard `BaseExceptionGroup` or -`ExceptionGroup` instances instead. +``trio.NonBaseMultiError`` which inherit from :exc:`BaseExceptionGroup` and +:exc:`ExceptionGroup`, respectively. Users should refrain from attempting to raise or +catch the Trio specific exceptions themselves, and treat them as if they were standard +:exc:`BaseExceptionGroup` or :exc:`ExceptionGroup` instances instead. "Strict" versus "loose" ExceptionGroup semantics ++++++++++++++++++++++++++++++++++++++++++++++++ From b621e70e9b19e303034472e5c1bb3c1f9fbda058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 17 Jul 2022 11:23:39 +0300 Subject: [PATCH 1136/1498] Improved the documentation for exception group handling --- docs/source/reference-core.rst | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 68a042ad79..853d41a390 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -687,8 +687,6 @@ You might wonder why Trio can't just remember "this task should be cancelled in If you want a timeout to apply to one task but not another, then you need to put the cancel scope in that individual task's function -- ``child()``, in this example. -.. _exceptiongroups: - Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -722,10 +720,16 @@ clause was introduced in Python 3.11 (:pep:`654`). Here's how it works:: async with trio.open_nursery() as nursery: nursery.start_soon(broken1) nursery.start_soon(broken2) - except* KeyError: - ... # handle the KeyErrors - except* IndexError: - ... # handle the IndexErrors + except* KeyError as excgroup: + for exc in excgroup.exceptions: + ... # handle each KeyError + except* IndexError as excgroup: + for exc in excgroup.exceptions: + ... # handle each IndexError + +If you want to reraise exceptions, or raise new ones, you can do so, but be aware that +exceptions raised in ``except*`` sections will be raised together in a new exception +group. But what if you can't use ``except*`` just yet? Well, for that there is the handy exceptiongroup_ library which lets you approximate this behavior with exception handler @@ -734,10 +738,12 @@ callbacks:: from exceptiongroup import catch def handle_keyerrors(excgroup): - ... # handle the KeyErrors + for exc in excgroup.exceptions: + ... # handle each KeyError def handle_indexerrors(excgroup): - ... # handle the IndexErrors + for exc in excgroup.exceptions: + ... # handle each IndexError with catch({ KeyError: handle_keyerror, @@ -747,8 +753,18 @@ callbacks:: nursery.start_soon(broken1) nursery.start_soon(broken2) -.. hint:: If your code, written using ``except*``, would set local variables, you can do - the same with handler callbacks as long as you declare those variables ``nonlocal``. +The semantics for the handler functions are equal to ``except*`` blocks, except for +setting local variables. If you need to set local variables, you need to declare them +inside the handler function(s) with the ``nonlocal`` keyword:: + + def handle_keyerrors(excgroup): + nonlocal myflag + myflag = True + + myflag = False + with catch({KeyError: handle_keyerror}): + async with trio.open_nursery() as nursery: + nursery.start_soon(broken1) For reasons of backwards compatibility, nurseries raise ``trio.MultiError`` and ``trio.NonBaseMultiError`` which inherit from :exc:`BaseExceptionGroup` and From 656fad4ee492940b22120067288735750a37db29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 20 Jul 2022 10:02:49 +0300 Subject: [PATCH 1137/1498] Fixed test collection error on Python 3.11 --- trio/_core/tests/test_run.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 6eeaee9aac..79f257918f 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -16,7 +16,6 @@ import outcome import sniffio import pytest -from exceptiongroup import catch from .tutil import ( slow, @@ -40,7 +39,7 @@ ) if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup, ExceptionGroup + from exceptiongroup import ExceptionGroup # slightly different from _timeouts.sleep_forever because it returns the value @@ -1858,14 +1857,12 @@ async def __anext__(self): items = [None] * len(nexts) got_stop = False - def handle(exc): - nonlocal got_stop - got_stop = True - - with catch({StopAsyncIteration: handle}): + try: async with _core.open_nursery() as nursery: for i, f in enumerate(nexts): nursery.start_soon(self._accumulate, f, items, i) + except ExceptionGroup as excgroup: + got_stop = bool(excgroup.split(StopAsyncIteration)) if got_stop: raise StopAsyncIteration From 6e86da32a561b14fb4d48d40273b308c9d4ad29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 20 Jul 2022 10:23:48 +0300 Subject: [PATCH 1138/1498] Readded the exceptiongroups label --- docs/source/reference-core.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 853d41a390..6507b6180a 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -687,6 +687,8 @@ You might wonder why Trio can't just remember "this task should be cancelled in If you want a timeout to apply to one task but not another, then you need to put the cancel scope in that individual task's function -- ``child()``, in this example. +.. _exceptiongroups: + Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From c34da2c02ba0c41237405dd7f1aaa7422df83ec5 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Wed, 20 Jul 2022 16:41:35 -0400 Subject: [PATCH 1139/1498] fix typo and extra sleep in test_unsafe_cancellable_kwarg removes a subtle early execution of a time.sleep call in the main thread. All the time and deadline stuff is not actually necessary in the happy path. --- trio/tests/test_threads.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 5bbad1b940..baff18278d 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -749,5 +749,4 @@ def __bool__(self): raise NotImplementedError with pytest.raises(NotImplementedError): - with _core.CancelScope(deadline=_core.current_time() + 0.01): - await to_thread_run_sync(time.sleep(1), cancellable=BadBool()) + await to_thread_run_sync(int, cancellable=BadBool()) From c49284372722f60b3743ca0b69dccd245293f996 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 10:29:54 +0000 Subject: [PATCH 1140/1498] Bump urllib3 from 1.26.10 to 1.26.11 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.10 to 1.26.11. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.11/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.10...1.26.11) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 73b2e8eaa3..0c9696e054 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -88,7 +88,7 @@ tomli==2.0.1 # via towncrier towncrier==21.9.0 # via -r docs-requirements.in -urllib3==1.26.10 +urllib3==1.26.11 # via requests # The following packages are considered to be unsafe in a requirements file: From 93851cfe5378e44244c345a6ab41a8962a8e693a Mon Sep 17 00:00:00 2001 From: Peter Gessler Date: Wed, 27 Jul 2022 09:01:18 -0500 Subject: [PATCH 1141/1498] Update _dtls.py --- trio/_dtls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_dtls.py b/trio/_dtls.py index 4cd4392fbc..910637455a 100644 --- a/trio/_dtls.py +++ b/trio/_dtls.py @@ -611,7 +611,7 @@ def _read_loop(read_fn): chunks = [] while True: try: - chunk = read_fn(2 ** 14) # max TLS record size + chunk = read_fn(2**14) # max TLS record size except SSL.WantReadError: break chunks.append(chunk) From 375b941db34b71ed37a239f9d40578915e674c5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 14:43:11 +0000 Subject: [PATCH 1142/1498] Bump mypy from 0.961 to 0.971 Bumps [mypy](https://github.com/python/mypy) from 0.961 to 0.971. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.961...v0.971) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5aabe871de..ccbc862895 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,8 +10,6 @@ astroid==2.11.6 # via pylint async-generator==1.10 # via -r test-requirements.in -atomicwrites==1.4.0 - # via pytest attrs==21.4.0 # via # -r test-requirements.in @@ -21,18 +19,10 @@ backcall==0.2.0 # via ipython black==22.6.0 ; implementation_name == "cpython" # via -r test-requirements.in -cffi==1.15.1 ; os_name == "nt" - # via - # -r test-requirements.in - # cryptography +cffi==1.15.1 + # via cryptography click==8.1.3 # via black -colorama==0.4.5 - # via - # click - # ipython - # pylint - # pytest coverage[toml]==6.4.1 # via pytest-cov cryptography==37.0.4 @@ -68,7 +58,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.961 ; implementation_name == "cpython" +mypy==0.971 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via @@ -83,6 +73,8 @@ parso==0.8.3 # via jedi pathspec==0.9.0 # via black +pexpect==4.8.0 + # via ipython pickleshare==0.7.5 # via ipython platformdirs==2.5.2 @@ -93,6 +85,8 @@ pluggy==1.0.0 # via pytest prompt-toolkit==3.0.30 # via ipython +ptyprocess==0.7.0 + # via pexpect py==1.11.0 # via pytest pycodestyle==2.8.0 @@ -145,10 +139,7 @@ types-pyopenssl==21.0.3 ; implementation_name == "cpython" typing-extensions==4.3.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.14.1 From e598ed94939faa31456d75da544083c1f2718b04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:35:53 +0000 Subject: [PATCH 1143/1498] Bump pylint from 2.14.4 to 2.14.5 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.14.4 to 2.14.5. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.14.4...v2.14.5) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ccbc862895..58983e34ae 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.12.0 # via ipython -pylint==2.14.4 +pylint==2.14.5 # via -r test-requirements.in pyopenssl==22.0.0 # via -r test-requirements.in From aa76c93f182cc5918b9a1c8d38c2564fbabe7580 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 15:35:54 +0000 Subject: [PATCH 1144/1498] Bump astroid from 2.11.6 to 2.11.7 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.11.6 to 2.11.7. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.11.6...v2.11.7) --- updated-dependencies: - dependency-name: astroid dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ccbc862895..947e115f99 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.11.6 +astroid==2.11.7 # via pylint async-generator==1.10 # via -r test-requirements.in From 2b0c000e2799ddacdcf6154d93995870ca57e254 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Jul 2022 10:17:56 +0000 Subject: [PATCH 1145/1498] Bump types-cryptography from 3.3.14 to 3.3.21 Bumps [types-cryptography](https://github.com/python/typeshed) from 3.3.14 to 3.3.21. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cryptography dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index ccbc862895..4469241547 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -128,12 +128,8 @@ traitlets==5.3.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-cryptography==3.3.14 +types-cryptography==3.3.21 # via types-pyopenssl -types-enum34==1.1.8 - # via types-cryptography -types-ipaddress==1.0.7 - # via types-cryptography types-pyopenssl==21.0.3 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.3.0 ; implementation_name == "cpython" From ac745174997a005f27d3717c9d133edf45a0bf59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Jul 2022 12:49:41 +0000 Subject: [PATCH 1146/1498] Bump types-pyopenssl from 21.0.3 to 22.0.9 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 21.0.3 to 22.0.9. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ca76d2dae2..d81a1fa2a1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -130,7 +130,7 @@ trustme==0.9.0 # via -r test-requirements.in types-cryptography==3.3.21 # via types-pyopenssl -types-pyopenssl==21.0.3 ; implementation_name == "cpython" +types-pyopenssl==22.0.9 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.3.0 ; implementation_name == "cpython" # via From 14eb6fe8c2a30a9323ae4dc1901b8b0408c7c1e8 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 6 Aug 2022 08:16:51 -0400 Subject: [PATCH 1147/1498] fix deadlock when deliver raises --- trio/_core/_thread_cache.py | 8 ++++- trio/_core/tests/test_thread_cache.py | 43 ++++++++++++++++++++------- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index fc3b8a9508..d11a23d27a 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -1,3 +1,5 @@ +import sys +import traceback from threading import Thread, Lock import outcome from itertools import count @@ -67,7 +69,11 @@ def _handle_job(self): # 'deliver' triggers a new job, it can be assigned to us # instead of spawning a new thread. self._thread_cache._idle_workers[self] = None - deliver(result) + try: + deliver(result) + except BaseException as e: + print("Exception while delivering result of thread", file=sys.stderr) + traceback.print_exception(type(e), e, e.__traceback__) def _work(self): while True: diff --git a/trio/_core/tests/test_thread_cache.py b/trio/_core/tests/test_thread_cache.py index d60288b3c1..5f19a5ac64 100644 --- a/trio/_core/tests/test_thread_cache.py +++ b/trio/_core/tests/test_thread_cache.py @@ -108,7 +108,8 @@ def _join_started_threads(): finally: for thread in threading.enumerate(): if thread not in before: - thread.join() + thread.join(timeout=1.0) + assert not thread.is_alive() def test_race_between_idle_exit_and_job_assignment(monkeypatch): @@ -134,31 +135,53 @@ def __init__(self): self._lock = threading.Lock() self._counter = 3 - def acquire(self, timeout=None): - self._lock.acquire() - if timeout is None: + def acquire(self, timeout=-1): + got_it = self._lock.acquire(timeout=timeout) + if timeout == -1: return True - else: + elif got_it: if self._counter > 0: self._counter -= 1 self._lock.release() return False return True + else: + return False def release(self): self._lock.release() monkeypatch.setattr(_thread_cache, "Lock", JankyLock) - with disable_threading_excepthook(), _join_started_threads(): + with _join_started_threads(): tc = ThreadCache() done = threading.Event() tc.start_thread_soon(lambda: None, lambda _: done.set()) done.wait() # Let's kill the thread we started, so it doesn't hang around until the # test suite finishes. Doesn't really do any harm, but it can be confusing - # to see it in debug output. This is hacky, and leaves our ThreadCache - # object in an inconsistent state... but it doesn't matter, because we're - # not going to use it again anyway. + # to see it in debug output. + monkeypatch.setattr(_thread_cache, "IDLE_TIMEOUT", 0.0001) + tc.start_thread_soon(lambda: None, lambda _: None) + + +def test_raise_in_deliver(capfd): + seen_threads = set() + + def track_threads(): + seen_threads.add(threading.current_thread()) - tc.start_thread_soon(lambda: None, lambda _: sys.exit()) + def deliver(_): + done.set() + raise RuntimeError("don't do this") + + done = threading.Event() + start_thread_soon(track_threads, deliver) + done.wait() + done = threading.Event() + start_thread_soon(track_threads, lambda _: done.set()) + done.wait() + assert len(seen_threads) == 1 + err = capfd.readouterr().err + assert "don't do this" in err + assert "delivering result" in err From 9e8544d717379c6bee9325cc473ccd40c994bf73 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 6 Aug 2022 22:48:05 -0700 Subject: [PATCH 1148/1498] Clarify docs on checkpoints in async iterables Closes #2388 --- docs/source/reference-core.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 58d583a39d..e8d4bd6762 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -90,9 +90,9 @@ them. Here are the rules: exception, it might act as a checkpoint or might not.) * This includes async iterators: If you write ``async for ... in ``, then there will be at least one checkpoint before - each iteration of the loop and one checkpoint after the last - iteration. + trio object>``, then there will be at least one checkpoint in + each iteration of the loop, and it will still checkpoint if the + iterable is empty. * Partial exception for async context managers: Both the entry and exit of an ``async with`` block are From 6a6b89135b683a14814a7aacb3fd7d1c46d11806 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Aug 2022 10:30:22 +0000 Subject: [PATCH 1149/1498] Bump tomlkit from 0.11.1 to 0.11.3 Bumps [tomlkit](https://github.com/sdispater/tomlkit) from 0.11.1 to 0.11.3. - [Release notes](https://github.com/sdispater/tomlkit/releases) - [Changelog](https://github.com/sdispater/tomlkit/blob/master/CHANGELOG.md) - [Commits](https://github.com/sdispater/tomlkit/compare/0.11.1...0.11.3) --- updated-dependencies: - dependency-name: tomlkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d81a1fa2a1..a711af2b77 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -120,7 +120,7 @@ tomli==2.0.1 # mypy # pylint # pytest -tomlkit==0.11.1 +tomlkit==0.11.3 # via pylint traitlets==5.3.0 # via From d1e92a4b0c3c8a5cfcbba1943d6b5939164c2798 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Aug 2022 10:19:43 +0000 Subject: [PATCH 1150/1498] Bump tomlkit from 0.11.3 to 0.11.4 Bumps [tomlkit](https://github.com/sdispater/tomlkit) from 0.11.3 to 0.11.4. - [Release notes](https://github.com/sdispater/tomlkit/releases) - [Changelog](https://github.com/sdispater/tomlkit/blob/master/CHANGELOG.md) - [Commits](https://github.com/sdispater/tomlkit/compare/0.11.3...0.11.4) --- updated-dependencies: - dependency-name: tomlkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a711af2b77..edd2e39957 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -120,7 +120,7 @@ tomli==2.0.1 # mypy # pylint # pytest -tomlkit==0.11.3 +tomlkit==0.11.4 # via pylint traitlets==5.3.0 # via From fb8c16c8ba7df2b3dd1aaf32b3c50f1527caae92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Aug 2022 10:20:30 +0000 Subject: [PATCH 1151/1498] Bump pytz from 2022.1 to 2022.2.1 Bumps [pytz](https://github.com/stub42/pytz) from 2022.1 to 2022.2.1. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2022.1...release_2022.2.1) --- updated-dependencies: - dependency-name: pytz dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 0c9696e054..5c542ba1c6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -53,7 +53,7 @@ pygments==2.12.0 # via sphinx pyparsing==3.0.9 # via packaging -pytz==2022.1 +pytz==2022.2.1 # via babel requests==2.28.1 # via sphinx From 100cc1ac73ae7e887e57bbe89f96aca130d433d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Aug 2022 10:29:12 +0000 Subject: [PATCH 1152/1498] Bump types-cryptography from 3.3.21 to 3.3.22 Bumps [types-cryptography](https://github.com/python/typeshed) from 3.3.21 to 3.3.22. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cryptography dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a711af2b77..5743de1c01 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -128,7 +128,7 @@ traitlets==5.3.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-cryptography==3.3.21 +types-cryptography==3.3.22 # via types-pyopenssl types-pyopenssl==22.0.9 ; implementation_name == "cpython" # via -r test-requirements.in From 6ac12a7577a933a76536dd5484a2aed85cf61d45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Aug 2022 10:18:45 +0000 Subject: [PATCH 1153/1498] Bump matplotlib-inline from 0.1.3 to 0.1.6 Bumps [matplotlib-inline](https://github.com/ipython/matplotlib-inline) from 0.1.3 to 0.1.6. - [Release notes](https://github.com/ipython/matplotlib-inline/releases) - [Commits](https://github.com/ipython/matplotlib-inline/compare/0.1.3...0.1.6) --- updated-dependencies: - dependency-name: matplotlib-inline dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a711af2b77..29106c3d36 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -52,7 +52,7 @@ jedi==0.18.1 # ipython lazy-object-proxy==1.7.1 # via astroid -matplotlib-inline==0.1.3 +matplotlib-inline==0.1.6 # via ipython mccabe==0.6.1 # via From 7e07e9b244d2255203769bf61a5cd87aaa1d041a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 10:22:10 +0000 Subject: [PATCH 1154/1498] Bump charset-normalizer from 2.1.0 to 2.1.1 Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.1.0 to 2.1.1. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.1.0...2.1.1) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 0c9696e054..a197db513d 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.10.3 # via sphinx certifi==2022.6.15 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via From 66b7537221b9dbb445216d4c163edbeb97390628 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 10:45:25 +0000 Subject: [PATCH 1155/1498] Bump attrs from 21.4.0 to 22.1.0 Bumps [attrs](https://github.com/python-attrs/attrs) from 21.4.0 to 22.1.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-attrs/attrs/compare/21.4.0...22.1.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 690952c071..abf3e67585 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in -attrs==21.4.0 +attrs==22.1.0 # via # -r docs-requirements.in # outcome diff --git a/test-requirements.txt b/test-requirements.txt index ddd7c3e66a..def51919f6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,7 @@ astroid==2.11.7 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==21.4.0 +attrs==22.1.0 # via # -r test-requirements.in # outcome From ab5467ba41538cbcfd2c5cb65475051a924a3bdd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Aug 2022 10:19:51 +0000 Subject: [PATCH 1156/1498] Bump towncrier from 21.9.0 to 22.8.0 Bumps [towncrier](https://github.com/hawkowl/towncrier) from 21.9.0 to 22.8.0. - [Release notes](https://github.com/hawkowl/towncrier/releases) - [Changelog](https://github.com/twisted/towncrier/blob/trunk/NEWS.rst) - [Commits](https://github.com/hawkowl/towncrier/compare/21.9.0...22.8.0) --- updated-dependencies: - dependency-name: towncrier dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 690952c071..019f2fa33c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -86,7 +86,7 @@ sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in tomli==2.0.1 # via towncrier -towncrier==21.9.0 +towncrier==22.8.0 # via -r docs-requirements.in urllib3==1.26.11 # via requests From a2fa665c0f773b7849f427b8f14bda7396bfa6b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Aug 2022 17:00:35 +0000 Subject: [PATCH 1157/1498] Bump urllib3 from 1.26.11 to 1.26.12 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.11 to 1.26.12. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.11...1.26.12) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 019f2fa33c..335afce30f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -88,7 +88,7 @@ tomli==2.0.1 # via towncrier towncrier==22.8.0 # via -r docs-requirements.in -urllib3==1.26.11 +urllib3==1.26.12 # via requests # The following packages are considered to be unsafe in a requirements file: From 0670252eb3ceac55f095952cb976ee029109c794 Mon Sep 17 00:00:00 2001 From: Vincent Vanlaer Date: Wed, 31 Aug 2022 21:55:52 +0200 Subject: [PATCH 1158/1498] Add a method to get the traceback of a task --- docs/source/reference-lowlevel.rst | 22 ++--------- trio/_core/_run.py | 48 ++++++++++++++++++++++++ trio/tests/test_tracing.py | 59 ++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 19 deletions(-) create mode 100644 trio/tests/test_tracing.py diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 4fb9abf299..86298d9cc8 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -501,25 +501,9 @@ Task API .. attribute:: coro - This task's coroutine object. Example usage: extracting a stack - trace:: - - import traceback - - def walk_coro_stack(coro): - while coro is not None: - if hasattr(coro, "cr_frame"): - # A real coroutine - yield coro.cr_frame, coro.cr_frame.f_lineno - coro = coro.cr_await - else: - # A generator decorated with @types.coroutine - yield coro.gi_frame, coro.gi_frame.f_lineno - coro = coro.gi_yieldfrom - - def print_stack_for_task(task): - ss = traceback.StackSummary.extract(walk_coro_stack(task.coro)) - print("".join(ss.format())) + This task's coroutine object. + + .. automethod:: iter_await_frames .. attribute:: context diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 1eb957b8d8..6439171d49 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -4,6 +4,7 @@ import select import sys import threading +import gc from collections import deque from contextlib import contextmanager import warnings @@ -1132,6 +1133,53 @@ def child_nurseries(self): """ return list(self._child_nurseries) + def iter_await_frames(self): + """Iterates recursively over the coroutine-like objects this + task is waiting on, yielding the frame and line number at each + frame. + + This is similar to `traceback.walk_stack` in a synchronous + context. Note that `traceback.walk_stack` returns frames from + the bottom of the call stack to the top, while this function + starts from `Task.coro ` and works it + way down. + + Example usage: extracting a stack trace:: + + import traceback + + def print_stack_for_task(task): + ss = traceback.StackSummary.extract(task.iter_await_frames()) + print("".join(ss.format())) + + """ + coro = self.coro + while coro is not None: + if hasattr(coro, "cr_frame"): + # A real coroutine + yield coro.cr_frame, coro.cr_frame.f_lineno + coro = coro.cr_await + elif hasattr(coro, "gi_frame"): + # A generator decorated with @types.coroutine + yield coro.gi_frame, coro.gi_frame.f_lineno + coro = coro.gi_yieldfrom + elif coro.__class__.__name__ in [ + "async_generator_athrow", + "async_generator_asend", + ]: + # cannot extract the generator directly, see https://github.com/python/cpython/issues/76991 + # we can however use the gc to look through the object + for referent in gc.get_referents(coro): + if hasattr(referent, "ag_frame"): + yield referent.ag_frame, referent.ag_frame.f_lineno + coro = referent.ag_await + break + else: + # either cpython changed or we are running on an alternative python implementation + return + else: + return + ################ # Cancellation ################ diff --git a/trio/tests/test_tracing.py b/trio/tests/test_tracing.py new file mode 100644 index 0000000000..07d1ff7609 --- /dev/null +++ b/trio/tests/test_tracing.py @@ -0,0 +1,59 @@ +import trio + + +async def coro1(event: trio.Event): + event.set() + await trio.sleep_forever() + + +async def coro2(event: trio.Event): + await coro1(event) + + +async def coro3(event: trio.Event): + await coro2(event) + + +async def coro2_async_gen(event: trio.Event): + yield await trio.lowlevel.checkpoint() + yield await coro1(event) + yield await trio.lowlevel.checkpoint() + + +async def coro3_async_gen(event: trio.Event): + async for x in coro2_async_gen(event): + pass + + +async def test_task_iter_await_frames(): + async with trio.open_nursery() as nursery: + event = trio.Event() + nursery.start_soon(coro3, event) + await event.wait() + + (task,) = nursery.child_tasks + + assert [frame.f_code.co_name for frame, _ in task.iter_await_frames()][:3] == [ + "coro3", + "coro2", + "coro1", + ] + + nursery.cancel_scope.cancel() + + +async def test_task_iter_await_frames_async_gen(): + async with trio.open_nursery() as nursery: + event = trio.Event() + nursery.start_soon(coro3_async_gen, event) + await event.wait() + + (task,) = nursery.child_tasks + + assert [frame.f_code.co_name for frame, _ in task.iter_await_frames()][:3] == [ + "coro3_async_gen", + "coro2_async_gen", + "coro1", + ] + + nursery.cancel_scope.cancel() From 7e77cd9548caf1e42e0a559a926701406c4d41bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 10:30:43 +0000 Subject: [PATCH 1159/1498] Bump sniffio from 1.2.0 to 1.3.0 Bumps [sniffio](https://github.com/python-trio/sniffio) from 1.2.0 to 1.3.0. - [Release notes](https://github.com/python-trio/sniffio/releases) - [Commits](https://github.com/python-trio/sniffio/compare/v1.2.0...v1.3.0) --- updated-dependencies: - dependency-name: sniffio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 81f46b71ac..1abf85c3d4 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -57,7 +57,7 @@ pytz==2022.2.1 # via babel requests==2.28.1 # via sphinx -sniffio==1.2.0 +sniffio==1.3.0 # via -r docs-requirements.in snowballstemmer==2.2.0 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index def51919f6..64abfdfcf2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -109,7 +109,7 @@ pytest==7.1.2 # pytest-cov pytest-cov==3.0.0 # via -r test-requirements.in -sniffio==1.2.0 +sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in From 9c590a11761ed70398ea1bb5f5795fe00ecf5606 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Sun, 4 Sep 2022 20:58:35 +0900 Subject: [PATCH 1160/1498] docs: fix bug in NotVeryGoodLock example --- docs/source/reference-lowlevel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 4fb9abf299..2c5086b80f 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -460,7 +460,7 @@ this does serve to illustrate the basic structure of the self._held = False async def acquire(self): - while self._held: + if self._held: task = trio.lowlevel.current_task() self._blocked_tasks.append(task) def abort_fn(_): From df7c5d0d83b39f86b5ce7db02ec05ecab1046b67 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 5 Sep 2022 07:27:09 +0200 Subject: [PATCH 1161/1498] Revert "docs: fix bug in NotVeryGoodLock example" --- docs/source/reference-lowlevel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 2c5086b80f..4fb9abf299 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -460,7 +460,7 @@ this does serve to illustrate the basic structure of the self._held = False async def acquire(self): - if self._held: + while self._held: task = trio.lowlevel.current_task() self._blocked_tasks.append(task) def abort_fn(_): From d8be0d01ae4f8960c829664fcedac81abfbf12a9 Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Tue, 6 Sep 2022 08:33:35 +0900 Subject: [PATCH 1162/1498] refactor and comment --- docs/source/reference-lowlevel.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 4fb9abf299..a3a2efc9be 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -460,13 +460,15 @@ this does serve to illustrate the basic structure of the self._held = False async def acquire(self): - while self._held: + if self._held: task = trio.lowlevel.current_task() - self._blocked_tasks.append(task) def abort_fn(_): self._blocked_tasks.remove(task) return trio.lowlevel.Abort.SUCCEEDED - await trio.lowlevel.wait_task_rescheduled(abort_fn) + # may need multiple attempts since reschedule() is not instant + while self._held: + self._blocked_tasks.append(task) + await trio.lowlevel.wait_task_rescheduled(abort_fn) self._held = True def release(self): From 76beeeed75539b6ced3065fce362944957626811 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Sep 2022 10:31:58 +0000 Subject: [PATCH 1163/1498] Bump cryptography from 37.0.4 to 38.0.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 37.0.4 to 38.0.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/37.0.4...38.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 64abfdfcf2..543c9be438 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.4.1 # via pytest-cov -cryptography==37.0.4 +cryptography==38.0.1 # via # -r test-requirements.in # pyopenssl From 3576a236e1942b75bbdad0e55c3e139daf471d01 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 10 Sep 2022 15:58:22 -0700 Subject: [PATCH 1164/1498] Update reference-lowlevel.rst --- docs/source/reference-lowlevel.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index a3a2efc9be..fa6f198a07 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -460,15 +460,22 @@ this does serve to illustrate the basic structure of the self._held = False async def acquire(self): - if self._held: + # We might have to try several times to acquire the lock. + while self._held: + # Someone else has the lock, so we have to wait. task = trio.lowlevel.current_task() def abort_fn(_): self._blocked_tasks.remove(task) return trio.lowlevel.Abort.SUCCEEDED - # may need multiple attempts since reschedule() is not instant - while self._held: - self._blocked_tasks.append(task) - await trio.lowlevel.wait_task_rescheduled(abort_fn) + self._blocked_tasks.append(task) + await trio.lowlevel.wait_task_rescheduled(abort_fn) + # At this point the lock was released -- but someone else + # might have swooped in and taken it again before we + # woke up. So we loop around to check the 'while' condition + # again. + # if we reach this point, it means that the 'while' condition + # has just failed, so we know no-one is holding the lock, and + # we can take it. self._held = True def release(self): From bd28c2968396c007bd7252dfcba7cf91e26c7f6e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Sat, 10 Sep 2022 15:59:07 -0700 Subject: [PATCH 1165/1498] Update reference-lowlevel.rst --- docs/source/reference-lowlevel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index fa6f198a07..065322788d 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -464,10 +464,10 @@ this does serve to illustrate the basic structure of the while self._held: # Someone else has the lock, so we have to wait. task = trio.lowlevel.current_task() + self._blocked_tasks.append(task) def abort_fn(_): self._blocked_tasks.remove(task) return trio.lowlevel.Abort.SUCCEEDED - self._blocked_tasks.append(task) await trio.lowlevel.wait_task_rescheduled(abort_fn) # At this point the lock was released -- but someone else # might have swooped in and taken it again before we From e367aef01b085d4e02477ba4f05c27fe7e98ecb6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Sep 2022 10:30:18 +0000 Subject: [PATCH 1166/1498] Bump certifi from 2022.6.15 to 2022.6.15.2 Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.6.15 to 2022.6.15.2. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.06.15...2022.06.15.2) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 1abf85c3d4..e4c1472fb2 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==22.1.0 # outcome babel==2.10.3 # via sphinx -certifi==2022.6.15 +certifi==2022.6.15.2 # via requests charset-normalizer==2.1.1 # via requests From 7729c6691fc2c7d2de3478b5157d42af1371d129 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Sep 2022 23:05:49 +0000 Subject: [PATCH 1167/1498] Bump black from 22.6.0 to 22.8.0 Bumps [black](https://github.com/psf/black) from 22.6.0 to 22.8.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.6.0...22.8.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 543c9be438..b1fd5cd322 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==22.1.0 # pytest backcall==0.2.0 # via ipython -black==22.6.0 ; implementation_name == "cpython" +black==22.8.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.1 # via cryptography From bfbcad48fa07510dd75dc1575f8f1920097c4fb4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Sep 2022 23:05:57 +0000 Subject: [PATCH 1168/1498] Bump idna from 3.3 to 3.4 Bumps [idna](https://github.com/kjd/idna) from 3.3 to 3.4. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.3...v3.4) --- updated-dependencies: - dependency-name: idna dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- test-requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 1abf85c3d4..7e30a9761f 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==22.1.0 # outcome babel==2.10.3 # via sphinx -certifi==2022.6.15 +certifi==2022.6.15.2 # via requests charset-normalizer==2.1.1 # via requests @@ -28,7 +28,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -idna==3.3 +idna==3.4 # via # -r docs-requirements.in # requests diff --git a/test-requirements.txt b/test-requirements.txt index 64abfdfcf2..ca4d7aa3d8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.4.1 # via pytest-cov -cryptography==37.0.4 +cryptography==38.0.1 # via # -r test-requirements.in # pyopenssl @@ -36,7 +36,7 @@ dill==0.3.5.1 # via pylint flake8==4.0.1 # via -r test-requirements.in -idna==3.3 +idna==3.4 # via # -r test-requirements.in # trustme From 2bcbdb05257e72ae91474b061f818bf040a99259 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Sep 2022 23:07:47 +0000 Subject: [PATCH 1169/1498] Bump traitlets from 5.3.0 to 5.4.0 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.3.0 to 5.4.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Commits](https://github.com/ipython/traitlets/compare/5.3.0...5.4.0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 64abfdfcf2..f558cdcbf2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.4.1 # via pytest-cov -cryptography==37.0.4 +cryptography==38.0.1 # via # -r test-requirements.in # pyopenssl @@ -122,7 +122,7 @@ tomli==2.0.1 # pytest tomlkit==0.11.4 # via pylint -traitlets==5.3.0 +traitlets==5.4.0 # via # ipython # matplotlib-inline From 43ad3dd9dc4cc66ba11086f1472dff6ae523f12a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Sep 2022 08:52:59 +0000 Subject: [PATCH 1170/1498] Bump certifi from 2022.6.15.2 to 2022.9.14 Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.6.15.2 to 2022.9.14. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.06.15.2...2022.09.14) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index e4c1472fb2..8c4c164bbb 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==22.1.0 # outcome babel==2.10.3 # via sphinx -certifi==2022.6.15.2 +certifi==2022.9.14 # via requests charset-normalizer==2.1.1 # via requests @@ -28,7 +28,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -idna==3.3 +idna==3.4 # via # -r docs-requirements.in # requests From 16c19b62eb12ed8015e1d46aec48f96de3e2cbc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 12:49:36 +0300 Subject: [PATCH 1171/1498] Spelled out MultiError.filter() and MultiError.catch() --- newsfragments/2211.headline.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/newsfragments/2211.headline.rst b/newsfragments/2211.headline.rst index ed7e9e1f29..c4320ba37f 100644 --- a/newsfragments/2211.headline.rst +++ b/newsfragments/2211.headline.rst @@ -1,11 +1,12 @@ ``MultiError`` has been deprecated in favor of the standard :exc:`BaseExceptionGroup` -(introduced in :pep:`654`). On Python versions below 3.11, this exception and its derivative -:exc:`ExceptionGroup` are provided by the backport_. Trio still raises ``MultiError``, -but it has been refactored into a subclass of :exc:`BaseExceptionGroup` which users -should catch instead of ``MultiError``. Uses of the ``filter()`` class method should be -replaced with :meth:`BaseExceptionGroup.split`. Uses of the ``catch()`` class method -should be replaced with either ``except*`` clauses (on Python 3.11+) or the -``exceptiongroup.catch()`` context manager provided by the backport_. +(introduced in :pep:`654`). On Python versions below 3.11, this exception and its +derivative :exc:`ExceptionGroup` are provided by the backport_. Trio still raises +``MultiError``, but it has been refactored into a subclass of :exc:`BaseExceptionGroup` +which users should catch instead of ``MultiError``. Uses of the ``MultiError.filter()`` +class method should be replaced with :meth:`BaseExceptionGroup.split`. Uses of the +``MultiError.catch()`` class method should be replaced with either ``except*`` clauses +(on Python 3.11+) or the ``exceptiongroup.catch()`` context manager provided by the +backport_. See the :ref:`updated documentation ` for details. From c35fccc6ed702163df40c93a21ae222c9b0e0b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 12:55:03 +0300 Subject: [PATCH 1172/1498] Fixed the instead= part in the deprecation warning --- trio/_core/_multierror.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index b144d5f43f..28a4e1b50b 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -237,7 +237,10 @@ def filter(cls, handler, root_exc): """ warn_deprecated( - "MultiError.filter()", "0.22.0", instead="MultiError.split()", issue=2211 + "MultiError.filter()", + "0.22.0", + instead="BaseExceptionGroup.split()", + issue=2211, ) return _filter_impl(handler, root_exc) From 7b7acb944f1f4ea617b993e3467d6e484817b1da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 12:58:28 +0300 Subject: [PATCH 1173/1498] Replaced warn() with warn_deprecated() --- trio/_core/_multierror.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 28a4e1b50b..4a4a39ab90 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -253,10 +253,11 @@ def catch(cls, handler): handler: as for :meth:`filter` """ - warnings.warn( - "MultiError.catch() has been deprecated. " - "Use except* or exceptiongroup.catch() instead.", - DeprecationWarning, + warn_deprecated( + "MultiError.catch", + "0.22.0", + instead="except* or exceptiongroup.catch()", + issue=2211, ) return MultiErrorCatcher(handler) From 1075ff038c53fdfd3baa0bba7497fb2607e64232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 13:48:16 +0300 Subject: [PATCH 1174/1498] Fixed the check for StopAsyncIteration --- trio/_core/tests/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 79f257918f..b4edc4bdb0 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1862,7 +1862,7 @@ async def __anext__(self): for i, f in enumerate(nexts): nursery.start_soon(self._accumulate, f, items, i) except ExceptionGroup as excgroup: - got_stop = bool(excgroup.split(StopAsyncIteration)) + got_stop = bool(excgroup.split(StopAsyncIteration)[0]) if got_stop: raise StopAsyncIteration From 458f5e2ebddc4ed3a6d1dde4fea508658432d3f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 13:51:16 +0300 Subject: [PATCH 1175/1498] Fixed comment in _nested_child_finished() --- trio/_core/_run.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 10a77d56c4..c369958ada 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -958,9 +958,8 @@ def _child_finished(self, task, outcome): self._check_nursery_closed() async def _nested_child_finished(self, nested_child_exc): - """ - Returns MultiError instance if there are pending exceptions. - """ + # Returns MultiError instance (or any exception if the nursery is in loose mode + # and there is just one contained exception) if there are pending exceptions if nested_child_exc is not None: self._add_exc(nested_child_exc) self._nested_child_running = False From 0f39b3f3551c3766ee4a16d2f07110de81e9617c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 14:42:31 +0300 Subject: [PATCH 1176/1498] Fixed deprecation checking in MultiError.catch() tests --- trio/_core/tests/test_multierror.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 1eaab6d4b3..503d85adc1 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -277,13 +277,13 @@ def test_MultiError_catch(): def noop(_): pass # pragma: no cover - with pytest.deprecated_call(MultiError.catch, noop): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(noop): pass # Simple pass-through of all exceptions m = make_tree() with pytest.raises(MultiError) as excinfo: - with pytest.deprecated_call(MultiError.catch, lambda exc: exc): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda exc: exc): raise m assert excinfo.value is m # Should be unchanged, except that we added a traceback frame by raising @@ -295,7 +295,7 @@ def noop(_): assert_tree_eq(m, make_tree()) # Swallows everything - with pytest.deprecated_call(MultiError.catch, lambda _: None): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda _: None): raise make_tree() def simple_filter(exc): @@ -306,7 +306,7 @@ def simple_filter(exc): return exc with pytest.raises(MultiError) as excinfo: - with pytest.deprecated_call(MultiError.catch, simple_filter): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(simple_filter): raise make_tree() new_m = excinfo.value assert isinstance(new_m, MultiError) @@ -324,7 +324,7 @@ def simple_filter(exc): v = ValueError() v.__cause__ = KeyError() with pytest.raises(ValueError) as excinfo: - with pytest.deprecated_call(MultiError.catch, lambda exc: exc): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda exc: exc): raise v assert isinstance(excinfo.value.__cause__, KeyError) @@ -332,7 +332,7 @@ def simple_filter(exc): context = KeyError() v.__context__ = context with pytest.raises(ValueError) as excinfo: - with pytest.deprecated_call(MultiError.catch, lambda exc: exc): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(lambda exc: exc): raise v assert excinfo.value.__context__ is context assert not excinfo.value.__suppress_context__ @@ -351,8 +351,9 @@ def catch_RuntimeError(exc): else: return exc - with pytest.deprecated_call(MultiError.catch, catch_RuntimeError): - raise MultiError([v, distractor]) + with pytest.warns(TrioDeprecationWarning): + with MultiError.catch(catch_RuntimeError): + raise MultiError([v, distractor]) assert excinfo.value.__context__ is context assert excinfo.value.__suppress_context__ == suppress_context @@ -380,7 +381,7 @@ def simple_filter(exc): gc.set_debug(gc.DEBUG_SAVEALL) with pytest.raises(MultiError): # covers MultiErrorCatcher.__exit__ and _multierror.copy_tb - with pytest.deprecated_call(MultiError.catch, simple_filter): + with pytest.warns(TrioDeprecationWarning), MultiError.catch(simple_filter): raise make_multi() gc.collect() assert not gc.garbage From dc35213466b22c2745bde3c69b5978e15046d350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 14:30:24 +0300 Subject: [PATCH 1177/1498] Reversed the order of tracebacks when concatenating --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index c369958ada..247a124ca9 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -135,7 +135,7 @@ def collapse_exception_group(excgroup): if len(exceptions) == 1: exceptions[0].__traceback__ = concat_tb( - exceptions[0].__traceback__, excgroup.__traceback__ + excgroup.__traceback__, exceptions[0].__traceback__ ) return exceptions[0] elif modified: From f9eff8e84b989df3f743e09ead2034ea5148fdc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 18 Sep 2022 23:44:08 +0300 Subject: [PATCH 1178/1498] Restored the IPython custom error handler Now it works with BaseExceptionGroup too. --- docs-requirements.in | 2 +- docs-requirements.txt | 12 +- setup.py | 2 +- test-requirements.in | 2 +- test-requirements.txt | 22 +++- trio/_core/_multierror.py | 28 ++++- trio/_core/tests/test_multierror.py | 108 ++++++++++++++++++ .../tests/test_multierror_scripts/__init__.py | 2 + .../tests/test_multierror_scripts/_common.py | 7 ++ .../ipython_custom_exc.py | 36 ++++++ .../simple_excepthook.py | 21 ++++ .../simple_excepthook_IPython.py | 7 ++ 12 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 trio/_core/tests/test_multierror_scripts/__init__.py create mode 100644 trio/_core/tests/test_multierror_scripts/_common.py create mode 100644 trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py create mode 100644 trio/_core/tests/test_multierror_scripts/simple_excepthook.py create mode 100644 trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py diff --git a/docs-requirements.in b/docs-requirements.in index 305aed888d..7d6abc0033 100644 --- a/docs-requirements.in +++ b/docs-requirements.in @@ -16,7 +16,7 @@ async_generator >= 1.9 idna outcome sniffio -exceptiongroup +exceptiongroup >= 1.0.0rc9 # See note in test-requirements.in immutables >= 0.6 diff --git a/docs-requirements.txt b/docs-requirements.txt index d9cbeb5e04..5e022f3226 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile docs-requirements.in @@ -28,7 +28,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -exceptiongroup==1.0.0rc8 +exceptiongroup==1.0.0rc9 # via -r docs-requirements.in idna==3.4 # via @@ -38,6 +38,8 @@ imagesize==1.4.1 # via sphinx immutables==0.18 # via -r docs-requirements.in +importlib-metadata==4.12.0 + # via click incremental==21.3.0 # via towncrier jinja2==3.0.3 @@ -90,8 +92,14 @@ tomli==2.0.1 # via towncrier towncrier==22.8.0 # via -r docs-requirements.in +typing-extensions==4.3.0 + # via + # immutables + # importlib-metadata urllib3==1.26.12 # via requests +zipp==3.8.1 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index d2f09d5055..00ba270764 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ # cffi 1.14 fixes memory leak inside ffi.getwinerror() # cffi is required on Windows, except on PyPy where it is built-in "cffi>=1.14; os_name == 'nt' and implementation_name != 'pypy'", - "exceptiongroup; python_version < '3.11'", + "exceptiongroup >= 1.0.0rc9; python_version < '3.11'", ], # This means, just install *everything* you see under trio/, even if it # doesn't look like a source file, so long as it appears in MANIFEST.in: diff --git a/test-requirements.in b/test-requirements.in index ae04d7bc70..cb9db8f894 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -30,5 +30,5 @@ async_generator >= 1.9 idna outcome sniffio -exceptiongroup; python_version < "3.11" +exceptiongroup >= 1.0.0rc9; python_version < "3.11" diff --git a/test-requirements.txt b/test-requirements.txt index 34768d31df..88123b0d69 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile test-requirements.in @@ -34,7 +34,7 @@ decorator==5.1.1 # via ipython dill==0.3.5.1 # via pylint -exceptiongroup==1.0.0rc8 ; python_version < "3.11" +exceptiongroup==1.0.0rc9 ; python_version < "3.11" # via -r test-requirements.in flake8==4.0.1 # via -r test-requirements.in @@ -42,6 +42,12 @@ idna==3.4 # via # -r test-requirements.in # trustme +importlib-metadata==4.2.0 + # via + # click + # flake8 + # pluggy + # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 @@ -130,6 +136,12 @@ traitlets==5.4.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in +typed-ast==1.5.4 ; implementation_name == "cpython" and python_version < "3.8" + # via + # -r test-requirements.in + # astroid + # black + # mypy types-cryptography==3.3.22 # via types-pyopenssl types-pyopenssl==22.0.9 ; implementation_name == "cpython" @@ -137,11 +149,17 @@ types-pyopenssl==22.0.9 ; implementation_name == "cpython" typing-extensions==4.3.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black + # importlib-metadata # mypy + # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.14.1 # via astroid +zipp==3.8.1 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 4a4a39ab90..ddeb768788 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -7,7 +7,9 @@ from trio._deprecate import warn_deprecated if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup, ExceptionGroup + from exceptiongroup import BaseExceptionGroup, ExceptionGroup, print_exception +else: + from traceback import print_exception ################################################################ # MultiError @@ -387,3 +389,27 @@ def concat_tb(head, tail): for head_tb in reversed(head_tbs): current_head = copy_tb(head_tb, tb_next=current_head) return current_head + + +# Remove when IPython gains support for exception groups +if "IPython" in sys.modules: + import IPython + + ip = IPython.get_ipython() + if ip is not None: + if ip.custom_exceptions != (): + warnings.warn( + "IPython detected, but you already have a custom exception " + "handler installed. I'll skip installing Trio's custom " + "handler, but this means exception groups will not show full " + "tracebacks.", + category=RuntimeWarning, + ) + else: + + def trio_show_traceback(self, etype, value, tb, tb_offset=None): + # XX it would be better to integrate with IPython's fancy + # exception formatting stuff (and not ignore tb_offset) + print_exception(value) + + ip.set_custom_exc((BaseExceptionGroup,), trio_show_traceback) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 503d85adc1..12f8606109 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -1,5 +1,9 @@ import gc import logging +import os +import subprocess +from pathlib import Path + import pytest from traceback import ( @@ -11,6 +15,7 @@ import sys import re +from .tutil import slow from .._multierror import MultiError, concat_tb, NonBaseMultiError from ... import TrioDeprecationWarning from ..._core import open_nursery @@ -427,3 +432,106 @@ def test_non_base_multierror(): exc = MultiError([ZeroDivisionError(), ValueError()]) assert type(exc) is NonBaseMultiError assert isinstance(exc, ExceptionGroup) + + +def run_script(name, use_ipython=False): + import trio + + trio_path = Path(trio.__file__).parent.parent + script_path = Path(__file__).parent / "test_multierror_scripts" / name + + env = dict(os.environ) + print("parent PYTHONPATH:", env.get("PYTHONPATH")) + if "PYTHONPATH" in env: # pragma: no cover + pp = env["PYTHONPATH"].split(os.pathsep) + else: + pp = [] + pp.insert(0, str(trio_path)) + pp.insert(0, str(script_path.parent)) + env["PYTHONPATH"] = os.pathsep.join(pp) + print("subprocess PYTHONPATH:", env.get("PYTHONPATH")) + + if use_ipython: + lines = [script_path.read_text(), "exit()"] + + cmd = [ + sys.executable, + "-u", + "-m", + "IPython", + # no startup files + "--quick", + "--TerminalIPythonApp.code_to_run=" + "\n".join(lines), + ] + else: + cmd = [sys.executable, "-u", str(script_path)] + print("running:", cmd) + completed = subprocess.run( + cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + print("process output:") + print(completed.stdout.decode("utf-8")) + return completed + + +def check_simple_excepthook(completed): + assert_match_in_seq( + [ + "in ", + "MultiError", + "--- 1 ---", + "in exc1_fn", + "ValueError", + "--- 2 ---", + "in exc2_fn", + "KeyError", + ], + completed.stdout.decode("utf-8"), + ) + + +try: + import IPython +except ImportError: # pragma: no cover + have_ipython = False +else: + have_ipython = True + +need_ipython = pytest.mark.skipif(not have_ipython, reason="need IPython") + + +@slow +@need_ipython +def test_ipython_exc_handler(): + completed = run_script("simple_excepthook.py", use_ipython=True) + check_simple_excepthook(completed) + + +@slow +@need_ipython +def test_ipython_imported_but_unused(): + completed = run_script("simple_excepthook_IPython.py") + check_simple_excepthook(completed) + + +@slow +@need_ipython +def test_ipython_custom_exc_handler(): + # Check we get a nice warning (but only one!) if the user is using IPython + # and already has some other set_custom_exc handler installed. + completed = run_script("ipython_custom_exc.py", use_ipython=True) + assert_match_in_seq( + [ + # The warning + "RuntimeWarning", + "IPython detected", + "skip installing Trio", + # The MultiError + "MultiError", + "ValueError", + "KeyError", + ], + completed.stdout.decode("utf-8"), + ) + # Make sure our other warning doesn't show up + assert "custom sys.excepthook" not in completed.stdout.decode("utf-8") diff --git a/trio/_core/tests/test_multierror_scripts/__init__.py b/trio/_core/tests/test_multierror_scripts/__init__.py new file mode 100644 index 0000000000..a1f6cb598d --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/__init__.py @@ -0,0 +1,2 @@ +# This isn't really a package, everything in here is a standalone script. This +# __init__.py is just to fool setup.py into actually installing the things. diff --git a/trio/_core/tests/test_multierror_scripts/_common.py b/trio/_core/tests/test_multierror_scripts/_common.py new file mode 100644 index 0000000000..0c70df1840 --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/_common.py @@ -0,0 +1,7 @@ +# https://coverage.readthedocs.io/en/latest/subprocess.html +try: + import coverage +except ImportError: # pragma: no cover + pass +else: + coverage.process_startup() diff --git a/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py b/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py new file mode 100644 index 0000000000..b3fd110e50 --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/ipython_custom_exc.py @@ -0,0 +1,36 @@ +import _common + +# Override the regular excepthook too -- it doesn't change anything either way +# because ipython doesn't use it, but we want to make sure Trio doesn't warn +# about it. +import sys + + +def custom_excepthook(*args): + print("custom running!") + return sys.__excepthook__(*args) + + +sys.excepthook = custom_excepthook + +import IPython + +ip = IPython.get_ipython() + + +# Set this to some random nonsense +class SomeError(Exception): + pass + + +def custom_exc_hook(etype, value, tb, tb_offset=None): + ip.showtraceback() + + +ip.set_custom_exc((SomeError,), custom_exc_hook) + +import trio + +# The custom excepthook should run, because Trio was polite and didn't +# override it +raise trio.MultiError([ValueError(), KeyError()]) diff --git a/trio/_core/tests/test_multierror_scripts/simple_excepthook.py b/trio/_core/tests/test_multierror_scripts/simple_excepthook.py new file mode 100644 index 0000000000..94004525db --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/simple_excepthook.py @@ -0,0 +1,21 @@ +import _common + +import trio + + +def exc1_fn(): + try: + raise ValueError + except Exception as exc: + return exc + + +def exc2_fn(): + try: + raise KeyError + except Exception as exc: + return exc + + +# This should be printed nicely, because Trio overrode sys.excepthook +raise trio.MultiError([exc1_fn(), exc2_fn()]) diff --git a/trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py b/trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py new file mode 100644 index 0000000000..6aa12493b0 --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/simple_excepthook_IPython.py @@ -0,0 +1,7 @@ +import _common + +# To tickle the "is IPython loaded?" logic, make sure that Trio tolerates +# IPython loaded but not actually in use +import IPython + +import simple_excepthook From 30b8f3b126050c2a7f71a6bac35472b809539170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 21 Sep 2022 02:22:50 +0300 Subject: [PATCH 1179/1498] Added link to specific IPython issue --- trio/_core/_multierror.py | 1 + 1 file changed, 1 insertion(+) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index ddeb768788..8a326cb162 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -392,6 +392,7 @@ def concat_tb(head, tail): # Remove when IPython gains support for exception groups +# (https://github.com/ipython/ipython/issues/13753) if "IPython" in sys.modules: import IPython From 232688b45d77fa81d52e562ce65892f57a9ba904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 21 Sep 2022 02:25:56 +0300 Subject: [PATCH 1180/1498] Updated wording about extracting submodules to independent packages --- docs/source/design.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 25647fe642..c3a47ab30c 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -461,12 +461,9 @@ of our public APIs without having to modify Trio internals. Inside ``trio._core`` ~~~~~~~~~~~~~~~~~~~~~ -There are two notable sub-modules that are largely independent of -the rest of Trio, and could (possibly should?) be extracted into their -own independent packages: - -* ``_ki.py``: Implements the core infrastructure for safe handling of - :class:`KeyboardInterrupt`. +The ``_ki.py`` module implements the core infrastructure for safe handling +of :class:`KeyboardInterrupt`. It's largely independent of the rest of Trio, +and could (possibly should?) be extracted into its own independent package. The most important submodule, where everything is integrated, is ``_run.py``. (This is also by far the largest submodule; it'd be nice From 0b8e908334a3367e2c299a9dca8e4b9df555da1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 25 Sep 2022 22:19:52 +0300 Subject: [PATCH 1181/1498] If present, patch Apport to support exception groups --- trio/_core/_multierror.py | 32 +++++++++++++++++++ trio/_core/tests/test_multierror.py | 19 +++++++++++ .../apport_excepthook.py | 13 ++++++++ 3 files changed, 64 insertions(+) create mode 100644 trio/_core/tests/test_multierror_scripts/apport_excepthook.py diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index 8a326cb162..ee9cbda416 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -414,3 +414,35 @@ def trio_show_traceback(self, etype, value, tb, tb_offset=None): print_exception(value) ip.set_custom_exc((BaseExceptionGroup,), trio_show_traceback) + + +# Ubuntu's system Python has a sitecustomize.py file that import +# apport_python_hook and replaces sys.excepthook. +# +# The custom hook captures the error for crash reporting, and then calls +# sys.__excepthook__ to actually print the error. +# +# We don't mind it capturing the error for crash reporting, but we want to +# take over printing the error. So we monkeypatch the apport_python_hook +# module so that instead of calling sys.__excepthook__, it calls our custom +# hook. +# +# More details: https://github.com/python-trio/trio/issues/1065 +if ( + sys.version_info < (3, 11) + and getattr(sys.excepthook, "__name__", None) == "apport_excepthook" +): + from types import ModuleType + + import apport_python_hook + from exceptiongroup import format_exception + + assert sys.excepthook is apport_python_hook.apport_excepthook + + def replacement_webhook(etype, value, tb): + sys.stderr.write("".join(format_exception(etype, value, tb))) + + fake_sys = ModuleType("trio_fake_sys") + fake_sys.__dict__.update(sys.__dict__) + fake_sys.__excepthook__ = replacement_webhook # type: ignore + apport_python_hook.sys = fake_sys diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 12f8606109..33e5afe925 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -535,3 +535,22 @@ def test_ipython_custom_exc_handler(): ) # Make sure our other warning doesn't show up assert "custom sys.excepthook" not in completed.stdout.decode("utf-8") + + +@slow +@pytest.mark.skipif( + not Path("/usr/lib/python3/dist-packages/apport_python_hook.py").exists(), + reason="need Ubuntu with python3-apport installed", +) +def test_apport_excepthook_monkeypatch_interaction(): + completed = run_script("apport_excepthook.py") + stdout = completed.stdout.decode("utf-8") + + # No warning + assert "custom sys.excepthook" not in stdout + + # Proper traceback + assert_match_in_seq( + ["Details of embedded", "KeyError", "Details of embedded", "ValueError"], + stdout, + ) diff --git a/trio/_core/tests/test_multierror_scripts/apport_excepthook.py b/trio/_core/tests/test_multierror_scripts/apport_excepthook.py new file mode 100644 index 0000000000..12e7fb0851 --- /dev/null +++ b/trio/_core/tests/test_multierror_scripts/apport_excepthook.py @@ -0,0 +1,13 @@ +# The apport_python_hook package is only installed as part of Ubuntu's system +# python, and not available in venvs. So before we can import it we have to +# make sure it's on sys.path. +import sys + +sys.path.append("/usr/lib/python3/dist-packages") +import apport_python_hook + +apport_python_hook.install() + +import trio + +raise trio.MultiError([KeyError("key_error"), ValueError("value_error")]) From 878af56191b813705558f690d37d43f541c47dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Sun, 25 Sep 2022 23:44:50 +0300 Subject: [PATCH 1182/1498] Added test coverage for collapse_exception_group() --- trio/_core/tests/test_run.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index b4edc4bdb0..bb3e8cc00c 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2371,3 +2371,17 @@ async def test_nursery_strict_exception_groups(): assert len(exc.value.exceptions) == 1 assert type(exc.value.exceptions[0]) is Exception assert exc.value.exceptions[0].args == ("foo",) + + +async def test_nursery_collapse(): + async def raise_error(): + raise RuntimeError("test error") + + with pytest.raises(MultiError) as exc: + async with _core.open_nursery() as nursery: + nursery.start_soon(raise_error) + async with _core.open_nursery(strict_exception_groups=True) as nursery2: + nursery2.start_soon(raise_error) + + assert len(exc.value.exceptions) == 2 + assert all(isinstance(e, RuntimeError) for e in exc.value.exceptions) From cc7d76a41b5cfa56624b4211ef75e55f614d688c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 26 Sep 2022 02:21:49 +0300 Subject: [PATCH 1183/1498] Changed cancel scopes to not collapse exceptions by default Instead, we mark collapsible MultiErrors and collapse only those when necessary. --- trio/_core/_multierror.py | 5 ++++- trio/_core/_run.py | 10 +++++----- trio/_core/tests/test_run.py | 18 +++++++++++++++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index ee9cbda416..e47701b3ad 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -187,6 +187,7 @@ def __init__(self, exceptions, *, _collapse=True): # This exception was already initialized. return + self.collapse = _collapse super().__init__("multiple tasks failed", exceptions) def __new__(cls, exceptions, *, _collapse=True): @@ -220,7 +221,9 @@ def __repr__(self): def derive(self, __excs): # We use _collapse=False here to get ExceptionGroup semantics, since derive() # is part of the PEP 654 API - return MultiError(__excs, _collapse=False) + exc = MultiError(__excs, _collapse=False) + exc.collapse = self.collapse + return exc @classmethod def filter(cls, handler, root_exc): diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 247a124ca9..78ca273820 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -127,13 +127,13 @@ def collapse_exception_group(excgroup): exceptions = list(excgroup.exceptions) modified = False for i, exc in enumerate(exceptions): - if isinstance(exc, MultiError): + if isinstance(exc, BaseExceptionGroup): new_exc = collapse_exception_group(exc) if new_exc is not exc: modified = True exceptions[i] = new_exc - if len(exceptions) == 1: + if len(exceptions) == 1 and isinstance(excgroup, MultiError) and excgroup.collapse: exceptions[0].__traceback__ = concat_tb( excgroup.__traceback__, exceptions[0].__traceback__ ) @@ -475,7 +475,7 @@ def __enter__(self): task._activate_cancel_status(self._cancel_status) return self - def _close(self, exc, collapse=True): + def _close(self, exc, collapse=False): if self._cancel_status is None: new_exc = RuntimeError( "Cancel scope stack corrupted: attempted to exit {!r} " @@ -540,8 +540,8 @@ def _close(self, exc, collapse=True): if matched: self.cancelled_caught = True - if collapse and isinstance(exc, MultiError): - exc = collapse_exception_group(exc) + if exc: + exc = collapse_exception_group(exc) self._cancel_status.close() with self._might_change_registered_deadline(): diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index bb3e8cc00c..7762c7963f 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2374,14 +2374,26 @@ async def test_nursery_strict_exception_groups(): async def test_nursery_collapse(): + """ + Test that a single exception from a nested nursery with strict semantics doesn't get + collapsed. + """ + async def raise_error(): raise RuntimeError("test error") with pytest.raises(MultiError) as exc: async with _core.open_nursery() as nursery: + nursery.start_soon(sleep_forever) nursery.start_soon(raise_error) async with _core.open_nursery(strict_exception_groups=True) as nursery2: + nursery2.start_soon(sleep_forever) nursery2.start_soon(raise_error) - - assert len(exc.value.exceptions) == 2 - assert all(isinstance(e, RuntimeError) for e in exc.value.exceptions) + nursery.cancel_scope.cancel() + + exceptions = exc.value.exceptions + assert len(exceptions) == 2 + assert isinstance(exceptions[0], RuntimeError) + assert isinstance(exceptions[1], MultiError) + assert len(exceptions[1].exceptions) == 1 + assert isinstance(exceptions[1].exceptions[0], RuntimeError) From a20e2d8466781c2c645c761df089a585ec2da9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 26 Sep 2022 09:57:28 +0300 Subject: [PATCH 1184/1498] Extended test coverage --- trio/_core/tests/test_run.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 7762c7963f..6aef1552ea 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2373,10 +2373,10 @@ async def test_nursery_strict_exception_groups(): assert exc.value.exceptions[0].args == ("foo",) -async def test_nursery_collapse(): +async def test_nursery_collapse_strict(): """ Test that a single exception from a nested nursery with strict semantics doesn't get - collapsed. + collapsed when CancelledErrors are stripped from it. """ async def raise_error(): @@ -2397,3 +2397,27 @@ async def raise_error(): assert isinstance(exceptions[1], MultiError) assert len(exceptions[1].exceptions) == 1 assert isinstance(exceptions[1].exceptions[0], RuntimeError) + + +async def test_nursery_collapse_loose(): + """ + Test that a single exception from a nested nursery with loose semantics gets + collapsed when CancelledErrors are stripped from it. + """ + + async def raise_error(): + raise RuntimeError("test error") + + with pytest.raises(MultiError) as exc: + async with _core.open_nursery() as nursery: + nursery.start_soon(sleep_forever) + nursery.start_soon(raise_error) + async with _core.open_nursery() as nursery2: + nursery2.start_soon(sleep_forever) + nursery2.start_soon(raise_error) + nursery.cancel_scope.cancel() + + exceptions = exc.value.exceptions + assert len(exceptions) == 2 + assert isinstance(exceptions[0], RuntimeError) + assert isinstance(exceptions[1], RuntimeError) From 1aba33e46b98ee7f524186ab1a0e7ab6efd9cd77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 26 Sep 2022 10:00:45 +0300 Subject: [PATCH 1185/1498] Fixed the apport script test --- trio/_core/tests/test_multierror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index 33e5afe925..edebf0dd59 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -551,6 +551,6 @@ def test_apport_excepthook_monkeypatch_interaction(): # Proper traceback assert_match_in_seq( - ["Details of embedded", "KeyError", "Details of embedded", "ValueError"], + ["--- 1 ---", "KeyError", "--- 2 ---", "ValueError"], stdout, ) From 0c283b7310d7df6f6bdaa60e2a498031eb3e44b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 26 Sep 2022 11:40:13 +0300 Subject: [PATCH 1186/1498] Fixed MultiError.collapse attribute missing on Python 3.11 --- trio/_core/_multierror.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index e47701b3ad..ef4c1592ae 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -181,13 +181,14 @@ class MultiError(BaseExceptionGroup): """ def __init__(self, exceptions, *, _collapse=True): + self.collapse = _collapse + # Avoid double initialization when _collapse is True and exceptions[0] returned # by __new__() happens to be a MultiError and subsequently __init__() is called. if _collapse and getattr(self, "exceptions", None) is not None: # This exception was already initialized. return - self.collapse = _collapse super().__init__("multiple tasks failed", exceptions) def __new__(cls, exceptions, *, _collapse=True): From 4a77603d2ec1d321d1972f4dd01b76a074c60c28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 10:50:49 +0000 Subject: [PATCH 1187/1498] Bump certifi from 2022.9.14 to 2022.9.24 Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.9.14 to 2022.9.24. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.09.14...2022.09.24) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 8c4c164bbb..a773e8dff1 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==22.1.0 # outcome babel==2.10.3 # via sphinx -certifi==2022.9.14 +certifi==2022.9.24 # via requests charset-normalizer==2.1.1 # via requests From 5a3e26c44f13b0e7af41fe2505a73a083287e9f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 10:51:34 +0000 Subject: [PATCH 1188/1498] Bump pyopenssl from 22.0.0 to 22.1.0 Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 22.0.0 to 22.1.0. - [Release notes](https://github.com/pyca/pyopenssl/releases) - [Changelog](https://github.com/pyca/pyopenssl/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/pyopenssl/compare/22.0.0...22.1.0) --- updated-dependencies: - dependency-name: pyopenssl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8780ddca67..1517407268 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -99,7 +99,7 @@ pygments==2.12.0 # via ipython pylint==2.14.5 # via -r test-requirements.in -pyopenssl==22.0.0 +pyopenssl==22.1.0 # via -r test-requirements.in pyparsing==3.0.9 # via packaging From 0364b4860fb313384940cb4906919977d48fe40b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 26 Sep 2022 22:47:36 +0300 Subject: [PATCH 1189/1498] Added test coverage for cancel scope encountering an excgroup w/o Cancelled --- trio/_core/tests/test_run.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 6aef1552ea..7b9af969d7 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2421,3 +2421,17 @@ async def raise_error(): assert len(exceptions) == 2 assert isinstance(exceptions[0], RuntimeError) assert isinstance(exceptions[1], RuntimeError) + + +async def test_cancel_scope_no_cancellederror(): + """ + Test that when a cancel scope encounters an exception group that does NOT contain + a Cancelled exception, it will NOT set the ``cancelled_caught`` flag. + """ + + with pytest.raises(ExceptionGroup): + with _core.CancelScope() as scope: + scope.cancel() + raise ExceptionGroup("test", [RuntimeError(), RuntimeError()]) + + assert not scope.cancelled_caught From bb31ed61434a007bc4d52ab04bae2e80cdcf5e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Mon, 26 Sep 2022 22:56:55 +0300 Subject: [PATCH 1190/1498] Removed an except block that was never entered --- trio/_core/tests/test_run.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 7b9af969d7..262ad6054b 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -1855,17 +1855,11 @@ def __aiter__(self): async def __anext__(self): nexts = self.nexts items = [None] * len(nexts) - got_stop = False - try: - async with _core.open_nursery() as nursery: - for i, f in enumerate(nexts): - nursery.start_soon(self._accumulate, f, items, i) - except ExceptionGroup as excgroup: - got_stop = bool(excgroup.split(StopAsyncIteration)[0]) + async with _core.open_nursery() as nursery: + for i, f in enumerate(nexts): + nursery.start_soon(self._accumulate, f, items, i) - if got_stop: - raise StopAsyncIteration return items result = [] From 2fd8309d235aacb8b264490437cb968a8e577016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 27 Sep 2022 10:29:25 +0300 Subject: [PATCH 1191/1498] Fixed excepthook wrapper name --- trio/_core/_multierror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index ef4c1592ae..c95f05b4ca 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -443,10 +443,10 @@ def trio_show_traceback(self, etype, value, tb, tb_offset=None): assert sys.excepthook is apport_python_hook.apport_excepthook - def replacement_webhook(etype, value, tb): + def replacement_excepthook(etype, value, tb): sys.stderr.write("".join(format_exception(etype, value, tb))) fake_sys = ModuleType("trio_fake_sys") fake_sys.__dict__.update(sys.__dict__) - fake_sys.__excepthook__ = replacement_webhook # type: ignore + fake_sys.__excepthook__ = replacement_excepthook # type: ignore apport_python_hook.sys = fake_sys From 80889b2818fb8637e3d478580f31d182746c42bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Tue, 27 Sep 2022 10:35:13 +0300 Subject: [PATCH 1192/1498] Removed unused parameter --- trio/_core/_run.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 78ca273820..7e57cfa445 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -475,7 +475,7 @@ def __enter__(self): task._activate_cancel_status(self._cancel_status) return self - def _close(self, exc, collapse=False): + def _close(self, exc): if self._cancel_status is None: new_exc = RuntimeError( "Cancel scope stack corrupted: attempted to exit {!r} " @@ -837,9 +837,7 @@ async def __aexit__(self, etype, exc, tb): new_exc = await self._nursery._nested_child_finished(exc) # Tracebacks show the 'raise' line below out of context, so let's give # this variable a name that makes sense out of context. - combined_error_from_nursery = self._scope._close( - new_exc, collapse=not self.strict_exception_groups - ) + combined_error_from_nursery = self._scope._close(new_exc) if combined_error_from_nursery is None: return True elif combined_error_from_nursery is exc: From 045f78bd06179888ab7a52d0e6b70295a6c734b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 10:20:55 +0000 Subject: [PATCH 1193/1498] Bump mypy from 0.971 to 0.981 Bumps [mypy](https://github.com/python/mypy) from 0.971 to 0.981. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.971...v0.981) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1517407268..c7d10c5349 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -58,7 +58,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.971 ; implementation_name == "cpython" +mypy==0.981 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From 23d12bdb662295eb5b8321a34445e1b3da701b98 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Tue, 27 Sep 2022 17:20:11 +0400 Subject: [PATCH 1194/1498] Remove now unused type: ignore --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 1eb957b8d8..749c1e11e8 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -655,7 +655,7 @@ def shield(self): """ return self._shield - @shield.setter # type: ignore # "decorated property not supported" + @shield.setter @enable_ki_protection def shield(self, new_value): if not isinstance(new_value, bool): From 45e9088c5ad67d9fc56777903d4de686e85ac3de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 17:13:43 +0000 Subject: [PATCH 1195/1498] Bump immutables from 0.18 to 0.19 Bumps [immutables](https://github.com/MagicStack/immutables) from 0.18 to 0.19. - [Release notes](https://github.com/MagicStack/immutables/releases) - [Commits](https://github.com/MagicStack/immutables/compare/v0.18...v0.19) --- updated-dependencies: - dependency-name: immutables dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index bacf886f8d..894438969a 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -36,10 +36,8 @@ idna==3.4 # requests imagesize==1.4.1 # via sphinx -immutables==0.18 +immutables==0.19 # via -r docs-requirements.in -importlib-metadata==4.12.0 - # via click incremental==21.3.0 # via towncrier jinja2==3.0.3 @@ -92,14 +90,8 @@ tomli==2.0.1 # via towncrier towncrier==22.8.0 # via -r docs-requirements.in -typing-extensions==4.3.0 - # via - # immutables - # importlib-metadata urllib3==1.26.12 # via requests -zipp==3.8.1 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From e8016ed61363eea07a04cd25b26cbdc77ef9423b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 17:14:02 +0000 Subject: [PATCH 1196/1498] Bump prompt-toolkit from 3.0.30 to 3.0.31 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.30 to 3.0.31. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.30...3.0.31) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index fd0154c6ad..7576544542 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -42,12 +42,6 @@ idna==3.4 # via # -r test-requirements.in # trustme -importlib-metadata==4.2.0 - # via - # click - # flake8 - # pluggy - # pytest iniconfig==1.1.1 # via pytest ipython==7.31.1 @@ -91,7 +85,7 @@ platformdirs==2.5.2 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython ptyprocess==0.7.0 # via pexpect @@ -136,12 +130,6 @@ traitlets==5.4.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -typed-ast==1.5.4 ; implementation_name == "cpython" and python_version < "3.8" - # via - # -r test-requirements.in - # astroid - # black - # mypy types-cryptography==3.3.22 # via types-pyopenssl types-pyopenssl==22.0.9 ; implementation_name == "cpython" @@ -149,17 +137,11 @@ types-pyopenssl==22.0.9 ; implementation_name == "cpython" typing-extensions==4.3.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black - # importlib-metadata # mypy - # pylint wcwidth==0.2.5 # via prompt-toolkit wrapt==1.14.1 # via astroid -zipp==3.8.1 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 87ea87018da2085180138bd952abcac67961be62 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 28 Sep 2022 10:04:13 +0400 Subject: [PATCH 1197/1498] Bump version to 0.22.0 --- docs/source/history.rst | 30 ++++++++++++++++++++++++++++++ newsfragments/2010.feature.rst | 4 ---- newsfragments/2211.headline.rst | 13 ------------- 3 files changed, 30 insertions(+), 17 deletions(-) delete mode 100644 newsfragments/2010.feature.rst delete mode 100644 newsfragments/2211.headline.rst diff --git a/docs/source/history.rst b/docs/source/history.rst index 783fae01ba..e93de7d3fc 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -5,6 +5,36 @@ Release history .. towncrier release notes start +Trio 0.22.0 (2022-09-28) +------------------------ + +Headline features +~~~~~~~~~~~~~~~~~ + +- ``MultiError`` has been deprecated in favor of the standard :exc:`BaseExceptionGroup` + (introduced in :pep:`654`). On Python versions below 3.11, this exception and its + derivative :exc:`ExceptionGroup` are provided by the backport_. Trio still raises + ``MultiError``, but it has been refactored into a subclass of :exc:`BaseExceptionGroup` + which users should catch instead of ``MultiError``. Uses of the ``MultiError.filter()`` + class method should be replaced with :meth:`BaseExceptionGroup.split`. Uses of the + ``MultiError.catch()`` class method should be replaced with either ``except*`` clauses + (on Python 3.11+) or the ``exceptiongroup.catch()`` context manager provided by the + backport_. + + See the :ref:`updated documentation ` for details. + + .. _backport: https://pypi.org/project/exceptiongroup/ (`#2211 `__) + + +Features +~~~~~~~~ + +- Added support for `Datagram TLS + `__, + for secure communication over UDP. Currently requires `PyOpenSSL + `__. (`#2010 `__) + + Trio 0.21.0 (2022-06-07) ---------------------------- diff --git a/newsfragments/2010.feature.rst b/newsfragments/2010.feature.rst deleted file mode 100644 index f99687652f..0000000000 --- a/newsfragments/2010.feature.rst +++ /dev/null @@ -1,4 +0,0 @@ -Added support for `Datagram TLS -`__, -for secure communication over UDP. Currently requires `PyOpenSSL -`__. diff --git a/newsfragments/2211.headline.rst b/newsfragments/2211.headline.rst deleted file mode 100644 index c4320ba37f..0000000000 --- a/newsfragments/2211.headline.rst +++ /dev/null @@ -1,13 +0,0 @@ -``MultiError`` has been deprecated in favor of the standard :exc:`BaseExceptionGroup` -(introduced in :pep:`654`). On Python versions below 3.11, this exception and its -derivative :exc:`ExceptionGroup` are provided by the backport_. Trio still raises -``MultiError``, but it has been refactored into a subclass of :exc:`BaseExceptionGroup` -which users should catch instead of ``MultiError``. Uses of the ``MultiError.filter()`` -class method should be replaced with :meth:`BaseExceptionGroup.split`. Uses of the -``MultiError.catch()`` class method should be replaced with either ``except*`` clauses -(on Python 3.11+) or the ``exceptiongroup.catch()`` context manager provided by the -backport_. - -See the :ref:`updated documentation ` for details. - -.. _backport: https://pypi.org/project/exceptiongroup/ From 8977afe1d547aaea64cb6ee2e3df65f3fd82c0c7 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 28 Sep 2022 10:18:06 +0400 Subject: [PATCH 1198/1498] Bump version to 0.22.0+dev --- trio/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_version.py b/trio/_version.py index d722bdc7f1..7111a4849d 100644 --- a/trio/_version.py +++ b/trio/_version.py @@ -1,3 +1,3 @@ # This file is imported from __init__.py and exec'd from setup.py -__version__ = "0.21.0+dev" +__version__ = "0.22.0+dev" From 33270da4ff9a4545fc9182d3cd69ec097689b9b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 10:17:20 +0000 Subject: [PATCH 1199/1498] Bump tomlkit from 0.11.4 to 0.11.5 Bumps [tomlkit](https://github.com/sdispater/tomlkit) from 0.11.4 to 0.11.5. - [Release notes](https://github.com/sdispater/tomlkit/releases) - [Changelog](https://github.com/sdispater/tomlkit/blob/master/CHANGELOG.md) - [Commits](https://github.com/sdispater/tomlkit/compare/0.11.4...0.11.5) --- updated-dependencies: - dependency-name: tomlkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7576544542..8f6fa93eb8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -122,7 +122,7 @@ tomli==2.0.1 # mypy # pylint # pytest -tomlkit==0.11.4 +tomlkit==0.11.5 # via pylint traitlets==5.4.0 # via From f9f8c6b730b17e4c8f2066c2579ddbc8955bcb49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= Date: Wed, 28 Sep 2022 14:39:39 +0300 Subject: [PATCH 1200/1498] Fixed link to exceptiongroup backport --- docs/source/history.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/history.rst b/docs/source/history.rst index e93de7d3fc..bcf90c79ab 100644 --- a/docs/source/history.rst +++ b/docs/source/history.rst @@ -22,8 +22,9 @@ Headline features backport_. See the :ref:`updated documentation ` for details. + (`#2211 `__) - .. _backport: https://pypi.org/project/exceptiongroup/ (`#2211 `__) + .. _backport: https://pypi.org/project/exceptiongroup/ Features From 65e93abe4095628e46033b19e6ce3b741bec1609 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Sep 2022 10:34:08 +0000 Subject: [PATCH 1201/1498] Bump pytest-cov from 3.0.0 to 4.0.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 3.0.0 to 4.0.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v3.0.0...v4.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8f6fa93eb8..a676a4cc9c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -109,7 +109,7 @@ pytest==7.1.2 # via # -r test-requirements.in # pytest-cov -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r test-requirements.in sniffio==1.3.0 # via -r test-requirements.in From 6df949c49e24d2919a7375d1fc1e8b7cc795181f Mon Sep 17 00:00:00 2001 From: John Belmonte Date: Fri, 30 Sep 2022 23:09:00 +0900 Subject: [PATCH 1202/1498] clarify RunVar test (#2431) --- trio/_core/tests/test_local.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/tests/test_local.py b/trio/_core/tests/test_local.py index 7f403168ea..619dcd20d4 100644 --- a/trio/_core/tests/test_local.py +++ b/trio/_core/tests/test_local.py @@ -80,7 +80,7 @@ async def task2(tok): with pytest.raises(LookupError): t1.get() - t1.set("cod") + t1.set("haddock") async with _core.open_nursery() as n: token = t1.set("cod") @@ -92,7 +92,7 @@ async def task2(tok): n.start_soon(task2, token) await _core.wait_all_tasks_blocked() - assert t1.get() == "cod" + assert t1.get() == "haddock" _core.run(sync_check) From 1c33474b550e1417ccda1a5b5216f81903c25b32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 10:17:48 +0000 Subject: [PATCH 1203/1498] Bump pytz from 2022.2.1 to 2022.4 Bumps [pytz](https://github.com/stub42/pytz) from 2022.2.1 to 2022.4. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/commits) --- updated-dependencies: - dependency-name: pytz dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 894438969a..cc34be48d6 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -55,7 +55,7 @@ pygments==2.12.0 # via sphinx pyparsing==3.0.9 # via packaging -pytz==2022.2.1 +pytz==2022.4 # via babel requests==2.28.1 # via sphinx From dfaf29fea12b1d2da097cd3091cf203f76a38c65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:48:33 +0000 Subject: [PATCH 1204/1498] Bump mypy from 0.981 to 0.982 Bumps [mypy](https://github.com/python/mypy) from 0.981 to 0.982. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.981...v0.982) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a676a4cc9c..b246b43557 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -60,7 +60,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.981 ; implementation_name == "cpython" +mypy==0.982 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From f4798ab474aabd290b9a21e8a7aff42d78df45c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:38:14 +0000 Subject: [PATCH 1205/1498] Bump typing-extensions from 4.3.0 to 4.4.0 Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.3.0 to 4.4.0. - [Release notes](https://github.com/python/typing_extensions/releases) - [Changelog](https://github.com/python/typing_extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/python/typing_extensions/compare/4.3.0...4.4.0) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b246b43557..6c7dcf9493 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -134,7 +134,7 @@ types-cryptography==3.3.22 # via types-pyopenssl types-pyopenssl==22.0.9 ; implementation_name == "cpython" # via -r test-requirements.in -typing-extensions==4.3.0 ; implementation_name == "cpython" +typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in # mypy From 2232426096af10a020be09c0d4ef1a5cd2b7f641 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:40:03 +0000 Subject: [PATCH 1206/1498] Bump black from 22.8.0 to 22.10.0 Bumps [black](https://github.com/psf/black) from 22.8.0 to 22.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.8.0...22.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b246b43557..7562674065 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==22.1.0 # pytest backcall==0.2.0 # via ipython -black==22.8.0 ; implementation_name == "cpython" +black==22.10.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.1 # via cryptography From fa1b2d96cf4201ff18eb3bb56bc9a7e3996c6708 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:17:47 +0000 Subject: [PATCH 1207/1498] Bump cryptography from 38.0.1 to 38.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 38.0.1 to 38.0.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/38.0.1...38.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b58b3e2d63..271eb9d5f3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.4.1 # via pytest-cov -cryptography==38.0.1 +cryptography==38.0.2 # via # -r test-requirements.in # pyopenssl From fdc2b3bf171cac744633cae10b1f60d641ac8ad2 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 13 Oct 2022 03:32:31 +0200 Subject: [PATCH 1208/1498] Introduce a gate/check GHA job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a GHA job that reliably determines if all the required dependencies have succeeded or not. It also allows to reduce the list of required branch protection CI statuses to just one — `check`. This reduces the maintenance burden by a lot and have been battle-tested across a small bunch of projects in its action form and in-house implementations of other people. It is now in use in aiohttp (and other aio-libs projects), CherryPy, attrs, coveragepy, some of the Ansible repositories, pip-tools, all of the jaraco's projects (like `setuptools`, `importlib_metadata`), some PyCQA, PyCA, PyPA and pytest projects, a few AWS Labs projects. The story behind this is explained in more detail at https://github.com/marketplace/actions/alls-green#why. --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 934fbbe026..92bc20539d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,16 @@ jobs: - python: '3.8' # <- not actually used pypy_nightly_branch: 'py3.8' extra_name: ', pypy 3.8 nightly' + continue-on-error: >- + ${{ + ( + matrix.check_formatting == '1' + || matrix.pypy_nightly_branch == 'py3.7' + || endsWith(matrix.python, '-dev') + ) + && true + || false + }} steps: - name: Checkout uses: actions/checkout@v2 @@ -138,3 +148,21 @@ jobs: env: # Should match 'name:' up above JOB_NAME: 'macOS (${{ matrix.python }})' + + # https://github.com/marketplace/actions/alls-green#why + check: # This job does nothing and is only used for the branch protection + + if: always() + + needs: + - Windows + - Ubuntu + - macOS + + runs-on: ubuntu-latest + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} From a4bcdaa25c4e50e62858402e8266fa596e860b5d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 13 Oct 2022 11:46:37 -0400 Subject: [PATCH 1209/1498] change the "python" link back to /3 when Python 3.11 is released --- docs/source/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 7859dd9e19..52872d0cb4 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -84,9 +84,8 @@ def setup(app): 'local_customization', ] -# FIXME: change the "python" link back to /3 when Python 3.11 is released intersphinx_mapping = { - "python": ('https://docs.python.org/3.11', None), + "python": ('https://docs.python.org/3', None), "outcome": ('https://outcome.readthedocs.io/en/latest/', None), "pyopenssl": ('https://www.pyopenssl.org/en/stable/', None), } From bb19e035d76326ef060c869bed62b18fef77eeec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 10:20:01 +0000 Subject: [PATCH 1210/1498] Bump incremental from 21.3.0 to 22.10.0 Bumps [incremental](https://github.com/twisted/incremental) from 21.3.0 to 22.10.0. - [Release notes](https://github.com/twisted/incremental/releases) - [Changelog](https://github.com/twisted/incremental/blob/trunk/NEWS.rst) - [Commits](https://github.com/twisted/incremental/compare/incremental-21.3.0...incremental-22.10.0) --- updated-dependencies: - dependency-name: incremental dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index cc34be48d6..6bdd2c3051 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -38,7 +38,7 @@ imagesize==1.4.1 # via sphinx immutables==0.19 # via -r docs-requirements.in -incremental==21.3.0 +incremental==22.10.0 # via towncrier jinja2==3.0.3 # via From 50cab3f66a5c5bcec708fcc3eac7d1ded3c1b31c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 10:21:23 +0000 Subject: [PATCH 1211/1498] Bump types-cryptography from 3.3.22 to 3.3.23.1 Bumps [types-cryptography](https://github.com/python/typeshed) from 3.3.22 to 3.3.23.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cryptography dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 271eb9d5f3..589ec3ac9d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -130,7 +130,7 @@ traitlets==5.4.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-cryptography==3.3.22 +types-cryptography==3.3.23.1 # via types-pyopenssl types-pyopenssl==22.0.9 ; implementation_name == "cpython" # via -r test-requirements.in From f3dbe5fe4d684170fd9b2bc27d8c86812607c034 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 10:22:12 +0000 Subject: [PATCH 1212/1498] Bump types-pyopenssl from 22.0.9 to 22.1.0.1 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 22.0.9 to 22.1.0.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 271eb9d5f3..d6537f399c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -132,7 +132,7 @@ trustme==0.9.0 # via -r test-requirements.in types-cryptography==3.3.22 # via types-pyopenssl -types-pyopenssl==22.0.9 ; implementation_name == "cpython" +types-pyopenssl==22.1.0.1 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.4.0 ; implementation_name == "cpython" # via From d80afb026e2d21a4f23ecf7b5a47d67c4c6e8e79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 10:27:40 +0000 Subject: [PATCH 1213/1498] Bump lazy-object-proxy from 1.7.1 to 1.8.0 Bumps [lazy-object-proxy](https://github.com/ionelmc/python-lazy-object-proxy) from 1.7.1 to 1.8.0. - [Release notes](https://github.com/ionelmc/python-lazy-object-proxy/releases) - [Changelog](https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-lazy-object-proxy/compare/v1.7.1...v1.8.0) --- updated-dependencies: - dependency-name: lazy-object-proxy dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d0b6992138..a306231cd5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -52,7 +52,7 @@ jedi==0.18.1 # via # -r test-requirements.in # ipython -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.8.0 # via astroid matplotlib-inline==0.1.6 # via ipython From d83242fde71fda78f57a2d7f7fc09ba2314a4ee8 Mon Sep 17 00:00:00 2001 From: Ganden Schaffner Date: Tue, 8 Nov 2022 00:00:00 -0800 Subject: [PATCH 1214/1498] Test host loop calling start_soon(needs_sniffio) during a guest run --- trio/_core/tests/test_guest_mode.py | 80 +++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index a5d6d78e18..ccb11a53a4 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -316,26 +316,38 @@ async def abandoned_main(in_host): trio.current_time() -def aiotrio_run(trio_fn, *, pass_not_threadsafe=True, **start_guest_run_kwargs): - loop = asyncio.new_event_loop() +async def start_guest_run_on_aio( + trio_fn, *, pass_not_threadsafe=True, **start_guest_run_kwargs +): + # This is asynchronous because asyncio documents that get_running_loop() + # can't be used from a sync function. + loop = asyncio.get_running_loop() + trio_done_fut = loop.create_future() - async def aio_main(): - trio_done_fut = loop.create_future() + def trio_done_callback(main_outcome): + print(f"trio_fn finished: {main_outcome!r}") + trio_done_fut.set_result(main_outcome) + + if pass_not_threadsafe: + start_guest_run_kwargs["run_sync_soon_not_threadsafe"] = loop.call_soon + + trio.lowlevel.start_guest_run( + trio_fn, + run_sync_soon_threadsafe=loop.call_soon_threadsafe, + done_callback=trio_done_callback, + **start_guest_run_kwargs, + ) - def trio_done_callback(main_outcome): - print(f"trio_fn finished: {main_outcome!r}") - trio_done_fut.set_result(main_outcome) + return trio_done_fut - if pass_not_threadsafe: - start_guest_run_kwargs["run_sync_soon_not_threadsafe"] = loop.call_soon - trio.lowlevel.start_guest_run( - trio_fn, - run_sync_soon_threadsafe=loop.call_soon_threadsafe, - done_callback=trio_done_callback, - **start_guest_run_kwargs, - ) +def aiotrio_run(trio_fn, *, pass_not_threadsafe=True, **start_guest_run_kwargs): + loop = asyncio.new_event_loop() + async def aio_main(): + trio_done_fut = await start_guest_run_on_aio( + trio_fn, pass_not_threadsafe=pass_not_threadsafe, **start_guest_run_kwargs + ) return (await trio_done_fut).unwrap() try: @@ -544,3 +556,41 @@ async def trio_main(): context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) assert record == {("asyncio", "asyncio"), ("trio", "trio")} + + +def test_guest_mode_host_calls_start_soon_sniffio(): + import sniffio + + async def aio_main(): + nursery = None + nursery_ready = asyncio.Event() + wait_in_nursery_done = asyncio.Event() + + async def wait_in_nursery(): + nonlocal nursery + try: + async with trio.open_nursery() as nursery: + nursery_ready.set() + await trio.sleep_forever() + finally: + wait_in_nursery_done.set() + + async def needs_sniffio(): + sniffio.current_async_library() + wait_in_nursery_done.set() + + wait_in_nursery_outcome_fut = await start_guest_run_on_aio( + wait_in_nursery, + # Not all versions of asyncio we test on use signal.set_wakeup_fd, + # but this test doesn't care about signal handling, and without + # host_uses_signal_set_wakeup_fd=True it can hang forever on some + # asyncio versions. + host_uses_signal_set_wakeup_fd=True, + ) + await nursery_ready.wait() + nursery.start_soon(needs_sniffio) + await wait_in_nursery_done.wait() + nursery.cancel_scope.cancel() + (await wait_in_nursery_outcome_fut).unwrap() + + asyncio.run(aio_main()) From 977a71c63ce810e67aed0a45e9029ee2ad6657c1 Mon Sep 17 00:00:00 2001 From: Ganden Schaffner Date: Fri, 4 Nov 2022 00:00:00 -0700 Subject: [PATCH 1215/1498] Fix nursery crashing during a guest run when the host loop calls start_soon(needs_sniffio) --- newsfragments/2183.bugfix.rst | 1 + trio/_core/_run.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 newsfragments/2183.bugfix.rst diff --git a/newsfragments/2183.bugfix.rst b/newsfragments/2183.bugfix.rst new file mode 100644 index 0000000000..d4f7558cd1 --- /dev/null +++ b/newsfragments/2183.bugfix.rst @@ -0,0 +1 @@ +Fixed nurseries crashing during a guest run when the host loop calls ``nursery.start_soon`` on a function that needs `sniffio `__. diff --git a/trio/_core/_run.py b/trio/_core/_run.py index be0497b635..d5c42b9feb 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1530,11 +1530,21 @@ def spawn_impl( if nursery is None: assert self.init_task is None + ###### + # Propagate contextvars, and make sure that async_fn can use sniffio. + ###### + if context is None: + if system_task: + context = self.system_context.copy() + else: + context = copy_context() + context.run(current_async_library_cvar.set, "trio") + ###### # Call the function and get the coroutine object, while giving helpful # errors for common mistakes. ###### - coro = coroutine_or_error(async_fn, *args) + coro = context.run(coroutine_or_error, async_fn, *args) if name is None: name = async_fn @@ -1546,12 +1556,6 @@ def spawn_impl( except AttributeError: name = repr(name) - if context is None: - if system_task: - context = self.system_context.copy() - else: - context = copy_context() - if not hasattr(coro, "cr_frame"): # This async function is implemented in C or Cython async def python_wrapper(orig_coro): @@ -1680,7 +1684,6 @@ def spawn_system_task(self, async_fn, *args, name=None, context=None): Task: the newly spawned task """ - current_async_library_cvar.set("trio") return self.spawn_impl( async_fn, args, @@ -1931,7 +1934,6 @@ def setup_runner( instruments = Instruments(instruments) io_manager = TheIOManager() system_context = copy_context() - system_context.run(current_async_library_cvar.set, "trio") ki_manager = KIManager() runner = Runner( From 2a4740b536e3b483b0e15407f18ee625cddd6e2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 10:13:38 +0000 Subject: [PATCH 1216/1498] Bump exceptiongroup from 1.0.0rc9 to 1.0.1 Bumps [exceptiongroup](https://github.com/agronholm/exceptiongroup) from 1.0.0rc9 to 1.0.1. - [Release notes](https://github.com/agronholm/exceptiongroup/releases) - [Changelog](https://github.com/agronholm/exceptiongroup/blob/main/CHANGES.rst) - [Commits](https://github.com/agronholm/exceptiongroup/compare/1.0.0rc9...1.0.1) --- updated-dependencies: - dependency-name: exceptiongroup dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 6bdd2c3051..df743b48df 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -28,7 +28,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -exceptiongroup==1.0.0rc9 +exceptiongroup==1.0.1 # via -r docs-requirements.in idna==3.4 # via diff --git a/test-requirements.txt b/test-requirements.txt index d0b6992138..376b6f906c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -34,8 +34,7 @@ decorator==5.1.1 # via ipython dill==0.3.5.1 # via pylint -exceptiongroup==1.0.0rc9 ; python_version < "3.11" - # via -r test-requirements.in +exceptiongroup==1.0.1 flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -116,12 +115,7 @@ sniffio==1.3.0 sortedcontainers==2.4.0 # via -r test-requirements.in tomli==2.0.1 - # via - # black - # coverage - # mypy - # pylint - # pytest + # via pytest tomlkit==0.11.5 # via pylint traitlets==5.4.0 From 18d74de3d6d59884513cfefb91d1bf86d789716b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 10:45:10 +0000 Subject: [PATCH 1217/1498] Bump tomlkit from 0.11.5 to 0.11.6 Bumps [tomlkit](https://github.com/sdispater/tomlkit) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/sdispater/tomlkit/releases) - [Changelog](https://github.com/sdispater/tomlkit/blob/master/CHANGELOG.md) - [Commits](https://github.com/sdispater/tomlkit/compare/0.11.5...0.11.6) --- updated-dependencies: - dependency-name: tomlkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 376b6f906c..747c720caf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -34,7 +34,6 @@ decorator==5.1.1 # via ipython dill==0.3.5.1 # via pylint -exceptiongroup==1.0.1 flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -116,7 +115,7 @@ sortedcontainers==2.4.0 # via -r test-requirements.in tomli==2.0.1 # via pytest -tomlkit==0.11.5 +tomlkit==0.11.6 # via pylint traitlets==5.4.0 # via From 8e27006505983234d64b50348f7f22178657ba58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 10:45:15 +0000 Subject: [PATCH 1218/1498] Bump dill from 0.3.5.1 to 0.3.6 Bumps [dill](https://github.com/uqfoundation/dill) from 0.3.5.1 to 0.3.6. - [Release notes](https://github.com/uqfoundation/dill/releases) - [Commits](https://github.com/uqfoundation/dill/compare/dill-0.3.5.1...dill-0.3.6) --- updated-dependencies: - dependency-name: dill dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 376b6f906c..6df2275090 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -32,9 +32,8 @@ cryptography==38.0.2 # trustme decorator==5.1.1 # via ipython -dill==0.3.5.1 +dill==0.3.6 # via pylint -exceptiongroup==1.0.1 flake8==4.0.1 # via -r test-requirements.in idna==3.4 From c259d507ff85d3b549897c09c59e54b910fe588d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 10:45:47 +0000 Subject: [PATCH 1219/1498] Bump traitlets from 5.4.0 to 5.5.0 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.4.0 to 5.5.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Changelog](https://github.com/ipython/traitlets/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipython/traitlets/compare/5.4.0...5.5.0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 376b6f906c..7e755642ae 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -34,7 +34,6 @@ decorator==5.1.1 # via ipython dill==0.3.5.1 # via pylint -exceptiongroup==1.0.1 flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -118,7 +117,7 @@ tomli==2.0.1 # via pytest tomlkit==0.11.5 # via pylint -traitlets==5.4.0 +traitlets==5.5.0 # via # ipython # matplotlib-inline From 10220eeb6b49fb9556ec38dafaa8f0a0d936dd05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 12:41:43 +0000 Subject: [PATCH 1220/1498] Bump pytest from 7.1.2 to 7.2.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.2 to 7.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.2...7.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6da3142fdf..381bbc6210 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -87,8 +87,6 @@ prompt-toolkit==3.0.31 # via ipython ptyprocess==0.7.0 # via pexpect -py==1.11.0 - # via pytest pycodestyle==2.8.0 # via flake8 pycparser==2.21 @@ -103,7 +101,7 @@ pyopenssl==22.1.0 # via -r test-requirements.in pyparsing==3.0.9 # via packaging -pytest==7.1.2 +pytest==7.2.0 # via # -r test-requirements.in # pytest-cov @@ -113,8 +111,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via pytest tomlkit==0.11.6 # via pylint traitlets==5.5.0 From c09dc3c1c55a588bb7ff36bb9caf59fe7e9bb754 Mon Sep 17 00:00:00 2001 From: AmericanY Date: Sun, 6 Nov 2022 22:41:01 +0200 Subject: [PATCH 1221/1498] Corrected Grammer (#2463) `handle_keyerror` and `handle_indexerror` should be represented with `(s)` e.g `handle_indexerrors` --- docs/source/reference-core.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 7fe2e04f6e..922ae4680e 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -748,8 +748,8 @@ callbacks:: ... # handle each IndexError with catch({ - KeyError: handle_keyerror, - IndexError: handle_indexerror + KeyError: handle_keyerrors, + IndexError: handle_indexerrors }): async with trio.open_nursery() as nursery: nursery.start_soon(broken1) @@ -764,7 +764,7 @@ inside the handler function(s) with the ``nonlocal`` keyword:: myflag = True myflag = False - with catch({KeyError: handle_keyerror}): + with catch({KeyError: handle_keyerrors}): async with trio.open_nursery() as nursery: nursery.start_soon(broken1) From 3e108dc18de750f24e464211dd89e607b2a04223 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 10:09:19 +0000 Subject: [PATCH 1222/1498] Bump sphinx-rtd-theme from 1.0.0 to 1.1.1 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 1.0.0 to 1.1.1. - [Release notes](https://github.com/readthedocs/sphinx_rtd_theme/releases) - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/1.0.0...1.1.1) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index df743b48df..a333e87825 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -70,7 +70,7 @@ sphinx==3.3.1 # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio -sphinx-rtd-theme==1.0.0 +sphinx-rtd-theme==1.1.1 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.2 # via sphinx From a2062982d5d9f45a34aca92ee82652c154e751c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 10:10:24 +0000 Subject: [PATCH 1223/1498] Bump types-pyopenssl from 22.1.0.1 to 22.1.0.2 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 22.1.0.1 to 22.1.0.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 381bbc6210..58d214a136 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -121,7 +121,7 @@ trustme==0.9.0 # via -r test-requirements.in types-cryptography==3.3.23.1 # via types-pyopenssl -types-pyopenssl==22.1.0.1 ; implementation_name == "cpython" +types-pyopenssl==22.1.0.2 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.4.0 ; implementation_name == "cpython" # via From 62484525617a097472feed5c9275764df48d02ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 10:10:51 +0000 Subject: [PATCH 1224/1498] Bump babel from 2.10.3 to 2.11.0 Bumps [babel](https://github.com/python-babel/babel) from 2.10.3 to 2.11.0. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES.rst) - [Commits](https://github.com/python-babel/babel/compare/v2.10.3...v2.11.0) --- updated-dependencies: - dependency-name: babel dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index df743b48df..c7839690cf 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -12,7 +12,7 @@ attrs==22.1.0 # via # -r docs-requirements.in # outcome -babel==2.10.3 +babel==2.11.0 # via sphinx certifi==2022.9.24 # via requests From 00cafe212803a40bf46b13a3379eb9ab5ab00b67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:05:10 +0000 Subject: [PATCH 1225/1498] Bump types-cryptography from 3.3.23.1 to 3.3.23.2 Bumps [types-cryptography](https://github.com/python/typeshed) from 3.3.23.1 to 3.3.23.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cryptography dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 381bbc6210..73c51d5513 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -119,7 +119,7 @@ traitlets==5.5.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-cryptography==3.3.23.1 +types-cryptography==3.3.23.2 # via types-pyopenssl types-pyopenssl==22.1.0.1 ; implementation_name == "cpython" # via -r test-requirements.in From a439605e117e5cfad3fc335bba06a23f58120bb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:06:05 +0000 Subject: [PATCH 1226/1498] Bump platformdirs from 2.5.2 to 2.5.3 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.5.2 to 2.5.3. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.5.2...2.5.3) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 381bbc6210..1fae27f623 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -77,7 +77,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.2 +platformdirs==2.5.3 # via # black # pylint From d262697ae445b2d646c9a2d75b02a15c034b5332 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:14:06 +0000 Subject: [PATCH 1227/1498] Bump pathspec from 0.9.0 to 0.10.2 Bumps [pathspec](https://github.com/cpburnz/python-pathspec) from 0.9.0 to 0.10.2. - [Release notes](https://github.com/cpburnz/python-pathspec/releases) - [Changelog](https://github.com/cpburnz/python-pathspec/blob/master/CHANGES.rst) - [Commits](https://github.com/cpburnz/python-pathspec/commits) --- updated-dependencies: - dependency-name: pathspec dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1fae27f623..5e239b82a0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -71,7 +71,7 @@ packaging==21.3 # via pytest parso==0.8.3 # via jedi -pathspec==0.9.0 +pathspec==0.10.2 # via black pexpect==4.8.0 # via ipython From 00cecb3e621e4105dba931b8d9fe961124f72822 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 14 Nov 2022 13:13:00 +0100 Subject: [PATCH 1228/1498] Add address in error message from trio.socket.socket upon connection failure --- newsfragments/1810.feature.rst | 1 + trio/_socket.py | 2 +- trio/tests/test_socket.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 newsfragments/1810.feature.rst diff --git a/newsfragments/1810.feature.rst b/newsfragments/1810.feature.rst new file mode 100644 index 0000000000..a2599d32b0 --- /dev/null +++ b/newsfragments/1810.feature.rst @@ -0,0 +1 @@ +`trio.socket.socket` now prints the address it tried to connect to upon failure. diff --git a/trio/_socket.py b/trio/_socket.py index bcff1ee9e7..8bfa6d265a 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -688,7 +688,7 @@ async def connect(self, address): # Okay, the connect finished, but it might have failed: err = self._sock.getsockopt(_stdlib_socket.SOL_SOCKET, _stdlib_socket.SO_ERROR) if err != 0: - raise OSError(err, "Error in connect: " + os.strerror(err)) + raise OSError(err, f"Error connecting to {address}: {os.strerror(err)}") ################################################################ # recv diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 1fa3721f91..ea5e48fe2d 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -723,6 +723,16 @@ def connect(self, *args, **kwargs): await sock.connect(("127.0.0.1", 2)) +# Fix issue #1810 +async def test_address_in_socket_error(): + address = "127.0.0.1" + with tsocket.socket() as sock: + try: + await sock.connect((address, 2)) + except OSError as e: + assert any(address in str(arg) for arg in e.args) + + async def test_resolve_address_exception_in_connect_closes_socket(): # Here we are testing issue 247, any cancellation will leave the socket closed with _core.CancelScope() as cancel_scope: From ebde15211dd4dad6cd1c122699028780cf2cc83d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Nov 2022 10:06:36 +0000 Subject: [PATCH 1229/1498] Bump mypy from 0.982 to 0.991 Bumps [mypy](https://github.com/python/mypy) from 0.982 to 0.991. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.982...v0.991) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1fae27f623..25c01be092 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -58,7 +58,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.982 ; implementation_name == "cpython" +mypy==0.991 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From fc6327086855eb77daf0a27b0e27e18c026a2dd3 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 11:13:36 +0100 Subject: [PATCH 1230/1498] Add some low-effort type annotations --- trio/_core/_run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 38359c3f7a..7ec9bb8c83 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1910,8 +1910,8 @@ def run( *args, clock=None, instruments=(), - restrict_keyboard_interrupt_to_checkpoints=False, - strict_exception_groups=False, + restrict_keyboard_interrupt_to_checkpoints: bool = False, + strict_exception_groups: bool = False, ): """Run a Trio-flavored async function, and return the result. @@ -2016,11 +2016,11 @@ def start_guest_run( run_sync_soon_threadsafe, done_callback, run_sync_soon_not_threadsafe=None, - host_uses_signal_set_wakeup_fd=False, + host_uses_signal_set_wakeup_fd: bool = False, clock=None, instruments=(), - restrict_keyboard_interrupt_to_checkpoints=False, - strict_exception_groups=False, + restrict_keyboard_interrupt_to_checkpoints: bool = False, + strict_exception_groups: bool = False, ): """Start a "guest" run of Trio on top of some other "host" event loop. @@ -2107,7 +2107,7 @@ def my_done_callback(run_outcome): # mode", where our core event loop gets unrolled into a series of callbacks on # the host loop. If you're doing a regular trio.run then this gets run # straight through. -def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): +def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd: bool = False): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True __tracebackhide__ = True From e306452de5287d11fb155fa4aa285caba853bfd9 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 16:20:42 +0100 Subject: [PATCH 1231/1498] Deal with mypy errors --- trio/_core/_run.py | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 7ec9bb8c83..46a898dd5f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -12,7 +12,7 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, TYPE_CHECKING +from typing import Callable, Final, NoReturn, TYPE_CHECKING from sniffio import current_async_library_cvar @@ -46,9 +46,9 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup -DEADLINE_HEAP_MIN_PRUNE_THRESHOLD = 1000 +DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: Final = 1000 -_NO_SEND = object() +_NO_SEND: Final = object() # Decorator to mark methods public. This does nothing by itself, but @@ -62,21 +62,21 @@ def _public(fn): # variable to True, and registers the Random instance _r for Hypothesis # to manage for each test case, which together should make Trio's task # scheduling loop deterministic. We have a test for that, of course. -_ALLOW_DETERMINISTIC_SCHEDULING = False +_ALLOW_DETERMINISTIC_SCHEDULING: Final = False _r = random.Random() # On CPython, Context.run() is implemented in C and doesn't show up in # tracebacks. On PyPy, it is implemented in Python and adds 1 frame to tracebacks. -def _count_context_run_tb_frames(): - def function_with_unique_name_xyzzy(): +def _count_context_run_tb_frames() -> int: + def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] 1 / 0 ctx = copy_context() try: ctx.run(function_with_unique_name_xyzzy) except ZeroDivisionError as exc: - tb = exc.__traceback__ + tb = cast(TracebackType, exc.__traceback__) # Skip the frame where we caught it tb = tb.tb_next count = 0 @@ -84,9 +84,14 @@ def function_with_unique_name_xyzzy(): tb = tb.tb_next count += 1 return count + else: + raise TrioInternalError( + f"The purpose of {function_with_unique_name_xyzzy.__name__} is " + "to raise a ZeroDivisionError, but it didn't." + ) -CONTEXT_RUN_TB_FRAMES = _count_context_run_tb_frames() +CONTEXT_RUN_TB_FRAMES: Final = _count_context_run_tb_frames() @attr.s(frozen=True, slots=True) @@ -1118,7 +1123,7 @@ class Task(metaclass=NoPublicConstructor): name = attr.ib() # PEP 567 contextvars context context = attr.ib() - _counter = attr.ib(init=False, factory=itertools.count().__next__) + _counter: int = attr.ib(init=False, factory=itertools.count().__next__) # Invariant: # - for unscheduled tasks, _next_send_fn and _next_send are both None @@ -1245,7 +1250,7 @@ class RunContext(threading.local): task: Task -GLOBAL_RUN_CONTEXT = RunContext() +GLOBAL_RUN_CONTEXT: Final = RunContext() @attr.s(frozen=True) @@ -2100,14 +2105,19 @@ def my_done_callback(run_outcome): # 24 hours is arbitrary, but it avoids issues like people setting timeouts of # 10**20 and then getting integer overflows in the underlying system calls. -_MAX_TIMEOUT = 24 * 60 * 60 +_MAX_TIMEOUT: Final = 24 * 60 * 60 # Weird quirk: this is written as a generator in order to support "guest # mode", where our core event loop gets unrolled into a series of callbacks on # the host loop. If you're doing a regular trio.run then this gets run # straight through. -def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd: bool = False): +def unrolled_run( + runner: Runner, + async_fn, + args, + host_uses_signal_set_wakeup_fd: bool = False, +): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True __tracebackhide__ = True @@ -2254,8 +2264,10 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd: bool = # frame we always remove, because it's this function # catching it, and then in addition we remove however many # more Context.run adds. - tb = task_exc.__traceback__.tb_next - for _ in range(CONTEXT_RUN_TB_FRAMES): + tb = task_exc.__traceback__ + for _ in range(1 + CONTEXT_RUN_TB_FRAMES): + if tb is None: + break tb = tb.tb_next final_outcome = Error(task_exc.with_traceback(tb)) # Remove local refs so that e.g. cancelled coroutine locals @@ -2350,7 +2362,7 @@ def started(self, value=None): pass -TASK_STATUS_IGNORED = _TaskStatusIgnored() +TASK_STATUS_IGNORED: Final = _TaskStatusIgnored() def current_task(): From 7027c2b317ce4650ab91634dbb4414e789820464 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 16:36:33 +0100 Subject: [PATCH 1232/1498] Remove cast --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 46a898dd5f..069e2e9a23 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -76,7 +76,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] try: ctx.run(function_with_unique_name_xyzzy) except ZeroDivisionError as exc: - tb = cast(TracebackType, exc.__traceback__) + tb = exc.__traceback__ # Skip the frame where we caught it tb = tb.tb_next count = 0 From 390976f108e723c249d511d112e3cf0e268aecad Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 17:05:01 +0100 Subject: [PATCH 1233/1498] Import Final from typing_extensions --- trio/_core/_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 069e2e9a23..c46644ef37 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -12,7 +12,8 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, Final, NoReturn, TYPE_CHECKING +from typing import Callable, NoReturn, TYPE_CHECKING +from typing_extensions import Final from sniffio import current_async_library_cvar From 5ac1ba59c34b9e6a18bb13c21122398440498f85 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 20:42:30 +0100 Subject: [PATCH 1234/1498] Ensure mypy succeeds --- mypy.ini | 3 +-- trio/_core/_run.py | 52 +++++++++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/mypy.ini b/mypy.ini index 31eeef1cd0..a4ff21dd91 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,5 @@ [mypy] -# TODO: run mypy against several OS/version combos in CI -# https://mypy.readthedocs.io/en/latest/command_line.html#platform-configuration +show_error_codes = True # Be flexible about dependencies that don't have stubs yet (like pytest) ignore_missing_imports = True diff --git a/trio/_core/_run.py b/trio/_core/_run.py index c46644ef37..5e17e4a595 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -13,7 +13,9 @@ from math import inf from time import perf_counter from typing import Callable, NoReturn, TYPE_CHECKING -from typing_extensions import Final +from typing import Deque + +from typing_extensions import Final as FinalT from sniffio import current_async_library_cvar @@ -47,9 +49,9 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup -DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: Final = 1000 +DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: FinalT = 1000 -_NO_SEND: Final = object() +_NO_SEND: FinalT = object() # Decorator to mark methods public. This does nothing by itself, but @@ -63,15 +65,31 @@ def _public(fn): # variable to True, and registers the Random instance _r for Hypothesis # to manage for each test case, which together should make Trio's task # scheduling loop deterministic. We have a test for that, of course. -_ALLOW_DETERMINISTIC_SCHEDULING: Final = False +_ALLOW_DETERMINISTIC_SCHEDULING: FinalT = False _r = random.Random() -# On CPython, Context.run() is implemented in C and doesn't show up in -# tracebacks. On PyPy, it is implemented in Python and adds 1 frame to tracebacks. def _count_context_run_tb_frames() -> int: - def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] - 1 / 0 + """Count implementation dependent traceback frames from Context.run() + + On CPython, Context.run() is implemented in C and doesn't show up in + tracebacks. On PyPy, it is implemented in Python and adds 1 frame to + tracebacks. + + Returns: + int: Traceback frame count + + """ + + def function_with_unique_name_xyzzy() -> NoReturn: + try: + 1 / 0 + except ZeroDivisionError: + raise + else: + raise TrioInternalError( + "A ZeroDivisionError should have been raised, but it wasn't." + ) ctx = copy_context() try: @@ -79,10 +97,10 @@ def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] except ZeroDivisionError as exc: tb = exc.__traceback__ # Skip the frame where we caught it - tb = tb.tb_next + tb = tb.tb_next # type: ignore[union-attr] count = 0 - while tb.tb_frame.f_code.co_name != "function_with_unique_name_xyzzy": - tb = tb.tb_next + while tb.tb_frame.f_code.co_name != "function_with_unique_name_xyzzy": # type: ignore[union-attr] + tb = tb.tb_next # type: ignore[union-attr] count += 1 return count else: @@ -92,7 +110,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] ) -CONTEXT_RUN_TB_FRAMES: Final = _count_context_run_tb_frames() +CONTEXT_RUN_TB_FRAMES: FinalT = _count_context_run_tb_frames() @attr.s(frozen=True, slots=True) @@ -1251,7 +1269,7 @@ class RunContext(threading.local): task: Task -GLOBAL_RUN_CONTEXT: Final = RunContext() +GLOBAL_RUN_CONTEXT: FinalT = RunContext() @attr.s(frozen=True) @@ -1338,7 +1356,7 @@ class Runner: # Run-local values, see _local.py _locals = attr.ib(factory=dict) - runq = attr.ib(factory=deque) + runq: Deque[Task] = attr.ib(factory=deque) tasks = attr.ib(factory=set) deadlines = attr.ib(factory=Deadlines) @@ -2106,7 +2124,7 @@ def my_done_callback(run_outcome): # 24 hours is arbitrary, but it avoids issues like people setting timeouts of # 10**20 and then getting integer overflows in the underlying system calls. -_MAX_TIMEOUT: Final = 24 * 60 * 60 +_MAX_TIMEOUT: FinalT = 24 * 60 * 60 # Weird quirk: this is written as a generator in order to support "guest @@ -2137,7 +2155,7 @@ def unrolled_run( # here is our event loop: while runner.tasks: if runner.runq: - timeout = 0 + timeout: float = 0 else: deadline = runner.deadlines.next_deadline() timeout = runner.clock.deadline_to_sleep_time(deadline) @@ -2363,7 +2381,7 @@ def started(self, value=None): pass -TASK_STATUS_IGNORED: Final = _TaskStatusIgnored() +TASK_STATUS_IGNORED: FinalT = _TaskStatusIgnored() def current_task(): From 532f5e4519897296541dc091e2317fe52a0e3fc5 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 21:14:29 +0100 Subject: [PATCH 1235/1498] Use pragmas to ignore coverage for defensive code --- trio/_core/_run.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 5e17e4a595..3ce85b5cac 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -12,9 +12,10 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, NoReturn, TYPE_CHECKING +from typing import Callable, NoReturn, TypeVar, TYPE_CHECKING from typing import Deque +# An unfortunate name collision here with trio._util.Final from typing_extensions import Final as FinalT from sniffio import current_async_library_cvar @@ -49,6 +50,8 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup +T = TypeVar("T") + DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: FinalT = 1000 _NO_SEND: FinalT = object() @@ -56,7 +59,7 @@ # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. -def _public(fn): +def _public(fn: T) -> T: return fn @@ -86,7 +89,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: 1 / 0 except ZeroDivisionError: raise - else: + else: # pragma: no cover raise TrioInternalError( "A ZeroDivisionError should have been raised, but it wasn't." ) @@ -103,7 +106,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: tb = tb.tb_next # type: ignore[union-attr] count += 1 return count - else: + else: # pragma: no cover raise TrioInternalError( f"The purpose of {function_with_unique_name_xyzzy.__name__} is " "to raise a ZeroDivisionError, but it didn't." From bbd77c6f69ef3014bc1e8841c2aec525eed53c0e Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 21:16:05 +0100 Subject: [PATCH 1236/1498] Minor cleanup of utils --- trio/_util.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index 539fc540ff..7631080adc 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -5,9 +5,7 @@ from abc import ABCMeta import os import signal -import sys -import pathlib -from functools import wraps, update_wrapper +from functools import update_wrapper import typing as t import threading import collections @@ -18,7 +16,7 @@ # Equivalent to the C function raise(), which Python doesn't wrap if os.name == "nt": - # On windows, os.kill exists but is really weird. + # On Windows, os.kill exists but is really weird. # # If you give it CTRL_C_EVENT or CTRL_BREAK_EVENT, it tries to deliver # those using GenerateConsoleCtrlEvent. But I found that when I tried @@ -37,7 +35,7 @@ # OTOH, if you pass os.kill any *other* signal number... then CPython # just calls TerminateProcess (wtf). # - # So, anyway, os.kill is not so useful for testing purposes. Instead + # So, anyway, os.kill is not so useful for testing purposes. Instead, # we use raise(): # # https://msdn.microsoft.com/en-us/library/dwwzkt4c.aspx @@ -226,7 +224,7 @@ def fix_one(qualname, name, obj): mod = getattr(obj, "__module__", None) if mod is not None and mod.startswith("trio."): obj.__module__ = module_name - # Modules, unlike everything else in Python, put fully-qualitied + # Modules, unlike everything else in Python, put fully-qualified # names into their __name__ attribute. We check for "." to avoid # rewriting these. if hasattr(obj, "__name__") and "." not in obj.__name__: @@ -277,11 +275,11 @@ class Final(ABCMeta): class SomeClass(metaclass=Final): pass - The metaclass will ensure that no sub class can be created. + The metaclass will ensure that no subclass can be created. Raises ------ - - TypeError if a sub class is created + - TypeError if a subclass is created """ def __new__(cls, name, bases, cls_namespace): @@ -305,14 +303,14 @@ class NoPublicConstructor(Final): class SomeClass(metaclass=NoPublicConstructor): pass - The metaclass will ensure that no sub class can be created, and that no instance + The metaclass will ensure that no subclass can be created, and that no instance can be initialized. If you try to instantiate your class (SomeClass()), a TypeError will be thrown. Raises ------ - - TypeError if a sub class or an instance is created. + - TypeError if a subclass or an instance is created. """ def __call__(cls, *args, **kwargs): From a58a58c57f2c7015851c0510a7822c3b797ab71a Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 14 Nov 2022 12:07:35 +0100 Subject: [PATCH 1237/1498] remove async_generator as dependency, add it as test dependency but also guard all revelant tests with skips --- setup.py | 1 - test-requirements.in | 3 +- trio/_core/_ki.py | 13 +++++-- trio/_core/tests/test_asyncgen.py | 8 ++-- trio/_core/tests/test_ki.py | 62 ++++++++++++++++--------------- trio/_util.py | 6 +-- trio/testing/_sequencer.py | 2 +- trio/tests/test_dtls.py | 2 +- trio/tests/test_ssl.py | 3 +- trio/tests/test_subprocess.py | 2 +- 10 files changed, 56 insertions(+), 46 deletions(-) diff --git a/setup.py b/setup.py index 00ba270764..b2e53826e6 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,6 @@ install_requires=[ "attrs >= 19.2.0", # for eq "sortedcontainers", - "async_generator >= 1.9", "idna", "outcome", "sniffio", diff --git a/test-requirements.in b/test-requirements.in index cb9db8f894..6e94409005 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -1,6 +1,7 @@ # For tests pytest >= 5.0 # for faulthandler in core pytest-cov >= 2.6.0 +async_generator >= 1.9 # ipython 7.x is the last major version supporting Python 3.7 ipython < 7.32 # for the IPython traceback integration tests pyOpenSSL >= 22.0.0 # for the ssl + DTLS tests @@ -26,9 +27,7 @@ typing-extensions; implementation_name == "cpython" cffi; os_name == "nt" attrs >= 19.2.0 sortedcontainers -async_generator >= 1.9 idna outcome sniffio exceptiongroup >= 1.0.0rc9; python_version < "3.11" - diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index 36aacecd96..180faee5a9 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -2,9 +2,8 @@ import signal import sys from functools import wraps -import attr -import async_generator +import attr from .._util import is_main_thread @@ -109,6 +108,14 @@ def currently_ki_protected(): return ki_protection_enabled(sys._getframe()) +# This is to support the async_generator package necessary for aclosing on <3.10 +# functions decorated @async_generator are given this magic property that's a +# reference to the object itself +# see python-trio/async_generator/async_generator/_impl.py +def legacy_isasyncgenfunction(obj): + return getattr(obj, "_async_gen_function", None) == id(obj) + + def _ki_protection_decorator(enabled): def decorator(fn): # In some version of Python, isgeneratorfunction returns true for @@ -141,7 +148,7 @@ def wrapper(*args, **kwargs): return gen return wrapper - elif async_generator.isasyncgenfunction(fn): + elif inspect.isasyncgenfunction(fn) or legacy_isasyncgenfunction(fn): @wraps(fn) def wrapper(*args, **kwargs): diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index 6ce0af366f..ef837469d4 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -1,13 +1,15 @@ import sys import weakref import pytest +import contextlib from math import inf from functools import partial -from async_generator import aclosing + from ... import _core from .tutil import gc_collect_harder, buggy_pypy_asyncgens, restore_unraisablehook +@pytest.mark.skipif(sys.version_info < (3, 10), reason="no aclosing() in stdlib<3.10") def test_asyncgen_basics(): collected = [] @@ -46,7 +48,7 @@ async def async_main(): assert collected.pop() == "abandoned" # aclosing() ensures it's cleaned up at point of use - async with aclosing(example("exhausted 1")) as aiter: + async with contextlib.aclosing(example("exhausted 1")) as aiter: assert 42 == await aiter.asend(None) assert collected.pop() == "exhausted 1" @@ -58,7 +60,7 @@ async def async_main(): gc_collect_harder() # No problems saving the geniter when using either of these patterns - async with aclosing(example("exhausted 3")) as aiter: + async with contextlib.aclosing(example("exhausted 3")) as aiter: saved.append(aiter) assert 42 == await aiter.asend(None) assert collected.pop() == "exhausted 3" diff --git a/trio/_core/tests/test_ki.py b/trio/_core/tests/test_ki.py index 430352ab6d..101e21441d 100644 --- a/trio/_core/tests/test_ki.py +++ b/trio/_core/tests/test_ki.py @@ -6,13 +6,12 @@ import threading import contextlib import time +import inspect -from async_generator import ( - async_generator, - yield_, - isasyncgenfunction, - asynccontextmanager, -) +try: + from async_generator import yield_, async_generator +except ImportError: # pragma: no cover + async_generator = yield_ = None from ... import _core from ...testing import wait_all_tasks_blocked @@ -142,7 +141,8 @@ def protected_manager(): raise KeyError -async def test_agen_protection(): +@pytest.mark.skipif(async_generator is None, reason="async_generator not installed") +async def test_async_generator_agen_protection(): @_core.enable_ki_protection @async_generator async def agen_protected1(): @@ -180,9 +180,16 @@ async def agen_unprotected2(): finally: assert not _core.currently_ki_protected() + await _check_agen(agen_protected1) + await _check_agen(agen_protected2) + await _check_agen(agen_unprotected1) + await _check_agen(agen_unprotected2) + + +async def test_native_agen_protection(): # Native async generators @_core.enable_ki_protection - async def agen_protected3(): + async def agen_protected(): assert _core.currently_ki_protected() try: yield @@ -190,35 +197,32 @@ async def agen_protected3(): assert _core.currently_ki_protected() @_core.disable_ki_protection - async def agen_unprotected3(): + async def agen_unprotected(): assert not _core.currently_ki_protected() try: yield finally: assert not _core.currently_ki_protected() - for agen_fn in [ - agen_protected1, - agen_protected2, - agen_protected3, - agen_unprotected1, - agen_unprotected2, - agen_unprotected3, - ]: - async for _ in agen_fn(): # noqa + await _check_agen(agen_protected) + await _check_agen(agen_unprotected) + + +async def _check_agen(agen_fn): + async for _ in agen_fn(): # noqa + assert not _core.currently_ki_protected() + + # asynccontextmanager insists that the function passed must itself be an + # async gen function, not a wrapper around one + if inspect.isasyncgenfunction(agen_fn): + async with contextlib.asynccontextmanager(agen_fn)(): assert not _core.currently_ki_protected() - # asynccontextmanager insists that the function passed must itself be an - # async gen function, not a wrapper around one - if isasyncgenfunction(agen_fn): - async with asynccontextmanager(agen_fn)(): - assert not _core.currently_ki_protected() - - # Another case that's tricky due to: - # https://bugs.python.org/issue29590 - with pytest.raises(KeyError): - async with asynccontextmanager(agen_fn)(): - raise KeyError + # Another case that's tricky due to: + # https://bugs.python.org/issue29590 + with pytest.raises(KeyError): + async with contextlib.asynccontextmanager(agen_fn)(): + raise KeyError # Test the case where there's no magic local anywhere in the call stack diff --git a/trio/_util.py b/trio/_util.py index 539fc540ff..58e6eddf29 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -11,8 +11,7 @@ import typing as t import threading import collections - -from async_generator import isasyncgen +import inspect import trio @@ -143,6 +142,7 @@ def _return_value_looks_like_wrong_library(value): # for things like functools.partial objects wrapping an async # function. So we have to just call it and then check whether the # return value is a coroutine object. + # Note: will not be necessary on python>=3.8, see https://bugs.python.org/issue34890 if not isinstance(coro, collections.abc.Coroutine): # Give good error for: nursery.start_soon(func_returning_future) if _return_value_looks_like_wrong_library(coro): @@ -152,7 +152,7 @@ def _return_value_looks_like_wrong_library(value): "That won't work without some sort of compatibility shim.".format(coro) ) - if isasyncgen(coro): + if inspect.isasyncgen(coro): raise TypeError( "start_soon expected an async function but got an async " "generator {!r}".format(coro) diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index a7e6e50ff0..c4e34717a1 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -1,7 +1,7 @@ from collections import defaultdict +from contextlib import asynccontextmanager import attr -from async_generator import asynccontextmanager from .. import _core from .. import _util diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index 8968d9a601..dbb489b9a8 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -4,7 +4,7 @@ from trio import DTLSEndpoint import random import attr -from async_generator import asynccontextmanager +from contextlib import asynccontextmanager from itertools import count import trustme diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 7825e319c2..fb119a754c 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -7,12 +7,11 @@ import threading import socket as stdlib_socket import ssl -from contextlib import contextmanager +from contextlib import asynccontextmanager, contextmanager from functools import partial from OpenSSL import SSL import trustme -from async_generator import asynccontextmanager import trio from .. import _core diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 061a715104..23aeebc587 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -3,11 +3,11 @@ import signal import subprocess import sys +from contextlib import asynccontextmanager from functools import partial from pathlib import Path as SyncPath import pytest -from async_generator import asynccontextmanager from .. import ( ClosedResourceError, From 8415ece0476ea2a812b16b0fc628060b076bd7f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 10:03:57 +0000 Subject: [PATCH 1238/1498] Bump exceptiongroup from 1.0.1 to 1.0.4 Bumps [exceptiongroup](https://github.com/agronholm/exceptiongroup) from 1.0.1 to 1.0.4. - [Release notes](https://github.com/agronholm/exceptiongroup/releases) - [Changelog](https://github.com/agronholm/exceptiongroup/blob/main/CHANGES.rst) - [Commits](https://github.com/agronholm/exceptiongroup/compare/1.0.1...1.0.4) --- updated-dependencies: - dependency-name: exceptiongroup dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 324babe3f8..b7a43a28bb 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -28,7 +28,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -exceptiongroup==1.0.1 +exceptiongroup==1.0.4 # via -r docs-requirements.in idna==3.4 # via From 9607e5b769a9b6c1bd3ffda98057a94da548d3ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 10:22:01 +0000 Subject: [PATCH 1239/1498] Bump platformdirs from 2.5.3 to 2.5.4 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.5.3 to 2.5.4. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.5.3...2.5.4) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 25c01be092..cfbad0da87 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -77,7 +77,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.3 +platformdirs==2.5.4 # via # black # pylint From 8f7937d64d2e7a9c3abcde605d4ede70dec8fde3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Nov 2022 10:37:10 +0000 Subject: [PATCH 1240/1498] Bump prompt-toolkit from 3.0.31 to 3.0.33 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.31 to 3.0.33. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.31...3.0.33) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 25c01be092..0575959428 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -83,7 +83,7 @@ platformdirs==2.5.3 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.31 +prompt-toolkit==3.0.33 # via ipython ptyprocess==0.7.0 # via pexpect From b57faf63063637180359d1a57c6ea3b33496edcd Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 22 Nov 2022 13:25:17 +0100 Subject: [PATCH 1241/1498] Modernization --- trio/_util.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index 12b7050cb5..387ef8785e 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -1,5 +1,3 @@ -# coding: utf-8 - # Little utilities we use internally from abc import ABCMeta @@ -331,7 +329,7 @@ def name_asyncgen(agen): try: module = agen.ag_frame.f_globals["__name__"] except (AttributeError, KeyError): - module = "<{}>".format(agen.ag_code.co_filename) + module = f"<{agen.ag_code.co_filename}>" try: qualname = agen.__qualname__ except AttributeError: From 9be6779d62180aac190b6a6fa7326c631d0724db Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 22 Nov 2022 13:31:13 +0100 Subject: [PATCH 1242/1498] Update mypy config given defaults in new version of mypy --- mypy.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy.ini b/mypy.ini index a4ff21dd91..6ccc92f5a4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,4 @@ [mypy] -show_error_codes = True - # Be flexible about dependencies that don't have stubs yet (like pytest) ignore_missing_imports = True From 6a02da70586d5f515529eecbff156cf83931a755 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 22 Nov 2022 13:53:40 +0100 Subject: [PATCH 1243/1498] Update old style type hints --- trio/_core/_ki.py | 4 ++-- trio/_subprocess.py | 14 +++++++------- trio/testing/_sequencer.py | 17 ++++++++++------- trio/tests/test_windows_pipes.py | 11 +++++++---- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index 180faee5a9..37fdc9b927 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -170,10 +170,10 @@ def wrapper(*args, **kwargs): return decorator -enable_ki_protection = _ki_protection_decorator(True) # type: Callable[[F], F] +enable_ki_protection: Callable[[F], F] = _ki_protection_decorator(True) enable_ki_protection.__name__ = "enable_ki_protection" -disable_ki_protection = _ki_protection_decorator(False) # type: Callable[[F], F] +disable_ki_protection: Callable[[F], F] = _ki_protection_decorator(False) disable_ki_protection.__name__ = "disable_ki_protection" diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 2bb0adcbdc..36f3fe9321 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -121,11 +121,11 @@ class Process(AsyncResource, metaclass=NoPublicConstructor): def __init__(self, popen, stdin, stdout, stderr): self._proc = popen - self.stdin = stdin # type: Optional[SendStream] - self.stdout = stdout # type: Optional[ReceiveStream] - self.stderr = stderr # type: Optional[ReceiveStream] + self.stdin: Optional[SendStream] = stdin + self.stdout: Optional[ReceiveStream] = stdout + self.stderr: Optional[ReceiveStream] = stderr - self.stdio = None # type: Optional[StapledStream] + self.stdio: Optional[StapledStream] = None if self.stdin is not None and self.stdout is not None: self.stdio = StapledStream(self.stdin, self.stdout) @@ -368,9 +368,9 @@ async def open_process( "on UNIX systems" ) - trio_stdin = None # type: Optional[ClosableSendStream] - trio_stdout = None # type: Optional[ClosableReceiveStream] - trio_stderr = None # type: Optional[ClosableReceiveStream] + trio_stdin: Optional[ClosableSendStream] = None + trio_stdout: Optional[ClosableReceiveStream] = None + trio_stderr: Optional[ClosableReceiveStream] = None # Close the parent's handle for each child side of a pipe; we want the child to # have the only copy, so that when it exits we can read EOF on our side. The # trio ends of pipes will be transferred to the Process object, which will be diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index c4e34717a1..00c0b40b64 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -1,5 +1,8 @@ +from __future__ import annotations + from collections import defaultdict from contextlib import asynccontextmanager +from typing import TYPE_CHECKING import attr @@ -7,8 +10,8 @@ from .. import _util from .. import Event -if False: - from typing import DefaultDict, Set +if TYPE_CHECKING: + from collections.abc import AsyncIterator @attr.s(eq=False, hash=False) @@ -52,14 +55,14 @@ async def main(): """ - _sequence_points = attr.ib( + _sequence_points: defaultdict[int, Event] = attr.ib( factory=lambda: defaultdict(Event), init=False - ) # type: DefaultDict[int, Event] - _claimed = attr.ib(factory=set, init=False) # type: Set[int] - _broken = attr.ib(default=False, init=False) + ) + _claimed: set[int] = attr.ib(factory=set, init=False) + _broken: bool = attr.ib(default=False, init=False) @asynccontextmanager - async def __call__(self, position: int): + async def __call__(self, position: int) -> AsyncIterator[None]: if position in self._claimed: raise RuntimeError("Attempted to re-use sequence point {}".format(position)) if self._broken: diff --git a/trio/tests/test_windows_pipes.py b/trio/tests/test_windows_pipes.py index 0a6b3516b0..2bcc64a072 100644 --- a/trio/tests/test_windows_pipes.py +++ b/trio/tests/test_windows_pipes.py @@ -3,6 +3,9 @@ import os import sys +from typing import Any +from typing import Tuple + import pytest from .._core.tests.tutil import gc_collect_harder @@ -15,12 +18,12 @@ from asyncio.windows_utils import pipe else: pytestmark = pytest.mark.skip(reason="windows only") - pipe = None # type: Any - PipeSendStream = None # type: Any - PipeReceiveStream = None # type: Any + pipe: Any = None + PipeSendStream: Any = None + PipeReceiveStream: Any = None -async def make_pipe() -> "Tuple[PipeSendStream, PipeReceiveStream]": +async def make_pipe() -> Tuple[PipeSendStream, PipeReceiveStream]: """Makes a new pair of pipes.""" (r, w) = pipe() return PipeSendStream(w), PipeReceiveStream(r) From d56b5bd98911cf26a2f8f889a659a1bbc682637c Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 22 Nov 2022 14:05:36 +0100 Subject: [PATCH 1244/1498] Fix missing import --- trio/_core/_ki.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index 37fdc9b927..fec23863f1 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import inspect import signal import sys from functools import wraps +from typing import TYPE_CHECKING import attr from .._util import is_main_thread -if False: +if TYPE_CHECKING: from typing import Any, TypeVar, Callable F = TypeVar("F", bound=Callable[..., Any]) @@ -54,7 +57,7 @@ # # If this raises a KeyboardInterrupt, it might be because the coroutine got # interrupted and has unwound... or it might be the KeyboardInterrupt -# arrived just *after* 'send' returned, so the coroutine is still running +# arrived just *after* 'send' returned, so the coroutine is still running, # but we just lost the message it sent. (And worse, in our actual task # runner, the send is hidden inside a utility function etc.) # From 2ae6c0cfa506a0a3cb98269c6fc19a87e46c997b Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Thu, 24 Nov 2022 09:50:26 +0100 Subject: [PATCH 1245/1498] Run black and pyupgrade --- docs/source/conf.py | 78 ++++++++++--------- docs/source/local_customization.py | 9 ++- .../reference-core/channels-backpressure.py | 4 + .../reference-core/channels-mpmc-broken.py | 4 + .../reference-core/channels-mpmc-fixed.py | 4 + .../reference-core/channels-shutdown.py | 4 + docs/source/reference-core/channels-simple.py | 4 + .../reference-testing/across-realtime.py | 17 ++-- docs/source/tutorial/echo-client.py | 4 + docs/source/tutorial/echo-server.py | 1 + docs/source/tutorial/tasks-intro.py | 4 + docs/source/tutorial/tasks-with-trace.py | 5 ++ trio/_abc.py | 2 - trio/_core/_asyncgens.py | 10 +-- trio/_core/_local.py | 2 +- trio/_core/_multierror.py | 2 +- trio/_core/_run.py | 9 +-- trio/_core/_unbounded_queue.py | 2 +- trio/_core/tests/test_asyncgen.py | 2 +- trio/_core/tests/test_io.py | 4 +- trio/_core/tests/test_mock_clock.py | 4 +- trio/_core/tests/test_multierror.py | 1 - trio/_core/tests/test_parking_lot.py | 10 +-- trio/_core/tests/test_run.py | 1 - trio/_deprecate.py | 16 ++-- trio/_highlevel_open_tcp_listeners.py | 2 +- trio/_highlevel_open_tcp_stream.py | 6 +- trio/_highlevel_socket.py | 4 +- trio/_path.py | 2 +- trio/_ssl.py | 4 +- trio/_subprocess.py | 14 ++-- trio/_sync.py | 7 +- trio/_threads.py | 2 - trio/_tools/gen_exports.py | 3 +- trio/_util.py | 7 +- trio/_wait_for_object.py | 2 +- trio/testing/_check_streams.py | 2 +- trio/testing/_sequencer.py | 2 +- trio/tests/test_channel.py | 1 - trio/tests/test_dtls.py | 6 +- trio/tests/test_exports.py | 6 +- trio/tests/test_file_io.py | 2 +- .../test_highlevel_open_tcp_listeners.py | 2 +- trio/tests/test_highlevel_open_tcp_stream.py | 4 +- trio/tests/test_path.py | 2 - trio/tests/test_socket.py | 4 +- trio/tests/test_subprocess.py | 7 +- trio/tests/test_testing.py | 4 +- trio/tests/test_threads.py | 8 +- trio/tests/tools/test_gen_exports.py | 2 +- 50 files changed, 166 insertions(+), 142 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 52872d0cb4..7200d91375 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # Trio documentation build configuration file, created by # sphinx-quickstart on Sat Jan 21 19:11:14 2017. @@ -19,17 +18,20 @@ # import os import sys + # For our local_customization module -sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath(".")) # For trio itself -sys.path.insert(0, os.path.abspath('../..')) +sys.path.insert(0, os.path.abspath("../..")) # https://docs.readthedocs.io/en/stable/builds.html#build-environment if "READTHEDOCS" in os.environ: import glob + if glob.glob("../../newsfragments/*.*.rst"): print("-- Found newsfragments; running towncrier --", flush=True) import subprocess + subprocess.run( ["towncrier", "--yes", "--date", "not released yet"], cwd="../..", @@ -66,6 +68,7 @@ def setup(app): app.add_css_file("hackrtd.css") + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -76,38 +79,38 @@ def setup(app): # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', - 'sphinx.ext.napoleon', - 'sphinxcontrib_trio', - 'local_customization', + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinxcontrib_trio", + "local_customization", ] intersphinx_mapping = { - "python": ('https://docs.python.org/3', None), - "outcome": ('https://outcome.readthedocs.io/en/latest/', None), - "pyopenssl": ('https://www.pyopenssl.org/en/stable/', None), + "python": ("https://docs.python.org/3", None), + "outcome": ("https://outcome.readthedocs.io/en/latest/", None), + "pyopenssl": ("https://www.pyopenssl.org/en/stable/", None), } autodoc_member_order = "bysource" # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Trio' -copyright = '2017, Nathaniel J. Smith' -author = 'Nathaniel J. Smith' +project = "Trio" +copyright = "2017, Nathaniel J. Smith" +author = "Nathaniel J. Smith" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -115,6 +118,7 @@ def setup(app): # # The short X.Y version. import trio + version = trio.__version__ # The full version, including alpha/beta/rc tags. release = version @@ -136,9 +140,9 @@ def setup(app): exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'default' +pygments_style = "default" -highlight_language = 'python3' +highlight_language = "python3" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -153,13 +157,14 @@ def setup(app): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -#html_theme = 'alabaster' +# html_theme = 'alabaster' # We have to set this ourselves, not only because it's useful for local # testing, but also because if we don't then RTD will throw away our # html_theme_options. import sphinx_rtd_theme -html_theme = 'sphinx_rtd_theme' + +html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme @@ -174,19 +179,19 @@ def setup(app): # versions/settings... "navigation_depth": 4, "logo_only": True, - 'prev_next_buttons_location': 'both' + "prev_next_buttons_location": "both", } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'Triodoc' +htmlhelp_basename = "Triodoc" # -- Options for LaTeX output --------------------------------------------- @@ -195,15 +200,12 @@ def setup(app): # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -213,8 +215,7 @@ def setup(app): # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'Trio.tex', 'Trio Documentation', - 'Nathaniel J. Smith', 'manual'), + (master_doc, "Trio.tex", "Trio Documentation", "Nathaniel J. Smith", "manual"), ] @@ -222,10 +223,7 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'trio', 'Trio Documentation', - [author], 1) -] +man_pages = [(master_doc, "trio", "Trio Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -234,7 +232,13 @@ def setup(app): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'Trio', 'Trio Documentation', - author, 'Trio', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "Trio", + "Trio Documentation", + author, + "Trio", + "One line description of project.", + "Miscellaneous", + ), ] diff --git a/docs/source/local_customization.py b/docs/source/local_customization.py index abade0ea8d..a970ad6e22 100644 --- a/docs/source/local_customization.py +++ b/docs/source/local_customization.py @@ -2,7 +2,10 @@ from sphinx import addnodes from sphinx.domains.python import PyClasslike from sphinx.ext.autodoc import ( - FunctionDocumenter, MethodDocumenter, ClassLevelDocumenter, Options, + FunctionDocumenter, + MethodDocumenter, + ClassLevelDocumenter, + Options, ) """ @@ -13,13 +16,15 @@ """ + class Interface(PyClasslike): def handle_signature(self, sig, signode): signode += addnodes.desc_name(sig, sig) return sig, "" def get_index_text(self, modname, name_cls): - return "{} (interface in {})".format(name_cls[0], modname) + return f"{name_cls[0]} (interface in {modname})" + def setup(app): app.add_directive_to_domain("py", "interface", Interface) diff --git a/docs/source/reference-core/channels-backpressure.py b/docs/source/reference-core/channels-backpressure.py index 50ac67f20a..72cb6900c2 100644 --- a/docs/source/reference-core/channels-backpressure.py +++ b/docs/source/reference-core/channels-backpressure.py @@ -4,6 +4,7 @@ import trio import math + async def producer(send_channel): count = 0 while True: @@ -14,6 +15,7 @@ async def producer(send_channel): print("Sent message:", count) count += 1 + async def consumer(receive_channel): async for value in receive_channel: print("Received message:", value) @@ -21,10 +23,12 @@ async def consumer(receive_channel): # takes 1 second await trio.sleep(1) + async def main(): send_channel, receive_channel = trio.open_memory_channel(math.inf) async with trio.open_nursery() as nursery: nursery.start_soon(producer, send_channel) nursery.start_soon(consumer, receive_channel) + trio.run(main) diff --git a/docs/source/reference-core/channels-mpmc-broken.py b/docs/source/reference-core/channels-mpmc-broken.py index 6b4adf298a..7043f0aafa 100644 --- a/docs/source/reference-core/channels-mpmc-broken.py +++ b/docs/source/reference-core/channels-mpmc-broken.py @@ -3,6 +3,7 @@ import trio import random + async def main(): async with trio.open_nursery() as nursery: send_channel, receive_channel = trio.open_memory_channel(0) @@ -13,6 +14,7 @@ async def main(): nursery.start_soon(consumer, "X", receive_channel) nursery.start_soon(consumer, "Y", receive_channel) + async def producer(name, send_channel): async with send_channel: for i in range(3): @@ -20,6 +22,7 @@ async def producer(name, send_channel): # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) + async def consumer(name, receive_channel): async with receive_channel: async for value in receive_channel: @@ -27,4 +30,5 @@ async def consumer(name, receive_channel): # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) + trio.run(main) diff --git a/docs/source/reference-core/channels-mpmc-fixed.py b/docs/source/reference-core/channels-mpmc-fixed.py index 0233938058..986e0d0c31 100644 --- a/docs/source/reference-core/channels-mpmc-fixed.py +++ b/docs/source/reference-core/channels-mpmc-fixed.py @@ -1,6 +1,7 @@ import trio import random + async def main(): async with trio.open_nursery() as nursery: send_channel, receive_channel = trio.open_memory_channel(0) @@ -12,6 +13,7 @@ async def main(): nursery.start_soon(consumer, "X", receive_channel.clone()) nursery.start_soon(consumer, "Y", receive_channel.clone()) + async def producer(name, send_channel): async with send_channel: for i in range(3): @@ -19,6 +21,7 @@ async def producer(name, send_channel): # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) + async def consumer(name, receive_channel): async with receive_channel: async for value in receive_channel: @@ -26,4 +29,5 @@ async def consumer(name, receive_channel): # Random sleeps help trigger the problem more reliably await trio.sleep(random.random()) + trio.run(main) diff --git a/docs/source/reference-core/channels-shutdown.py b/docs/source/reference-core/channels-shutdown.py index 8cb8e28225..cba4e43801 100644 --- a/docs/source/reference-core/channels-shutdown.py +++ b/docs/source/reference-core/channels-shutdown.py @@ -1,19 +1,23 @@ import trio + async def main(): async with trio.open_nursery() as nursery: send_channel, receive_channel = trio.open_memory_channel(0) nursery.start_soon(producer, send_channel) nursery.start_soon(consumer, receive_channel) + async def producer(send_channel): async with send_channel: for i in range(3): await send_channel.send(f"message {i}") + async def consumer(receive_channel): async with receive_channel: async for value in receive_channel: print(f"got value {value!r}") + trio.run(main) diff --git a/docs/source/reference-core/channels-simple.py b/docs/source/reference-core/channels-simple.py index c00fb145cf..e89347379d 100644 --- a/docs/source/reference-core/channels-simple.py +++ b/docs/source/reference-core/channels-simple.py @@ -1,5 +1,6 @@ import trio + async def main(): async with trio.open_nursery() as nursery: # Open a channel: @@ -9,15 +10,18 @@ async def main(): nursery.start_soon(producer, send_channel) nursery.start_soon(consumer, receive_channel) + async def producer(send_channel): # Producer sends 3 messages for i in range(3): # The producer sends using 'await send_channel.send(...)' await send_channel.send(f"message {i}") + async def consumer(receive_channel): # The consumer uses an 'async for' loop to receive the values: async for value in receive_channel: print(f"got value {value!r}") + trio.run(main) diff --git a/docs/source/reference-testing/across-realtime.py b/docs/source/reference-testing/across-realtime.py index 41bf502611..300810c065 100644 --- a/docs/source/reference-testing/across-realtime.py +++ b/docs/source/reference-testing/across-realtime.py @@ -6,6 +6,7 @@ YEAR = 365 * 24 * 60 * 60 # seconds + async def task1(): start = trio.current_time() @@ -13,15 +14,15 @@ async def task1(): await trio.sleep(YEAR) duration = trio.current_time() - start - print("task1: woke up; clock says I've slept {} years" - .format(duration / YEAR)) + print(f"task1: woke up; clock says I've slept {duration / YEAR} years") print("task1: sleeping for 1 year, 100 times") for _ in range(100): await trio.sleep(YEAR) duration = trio.current_time() - start - print("task1: slept {} years total".format(duration / YEAR)) + print(f"task1: slept {duration / YEAR} years total") + async def task2(): start = trio.current_time() @@ -30,25 +31,27 @@ async def task2(): await trio.sleep(5 * YEAR) duration = trio.current_time() - start - print("task2: woke up; clock says I've slept {} years" - .format(duration / YEAR)) + print(f"task2: woke up; clock says I've slept {duration / YEAR} years") print("task2: sleeping for 500 years") await trio.sleep(500 * YEAR) duration = trio.current_time() - start - print("task2: slept {} years total".format(duration / YEAR)) + print(f"task2: slept {duration / YEAR} years total") + async def main(): async with trio.open_nursery() as nursery: nursery.start_soon(task1) nursery.start_soon(task2) + def run_example(clock): real_start = time.perf_counter() trio.run(main, clock=clock) real_duration = time.perf_counter() - real_start - print("Total real time elapsed: {} seconds".format(real_duration)) + print(f"Total real time elapsed: {real_duration} seconds") + print("Clock where time passes at 100 years per second:\n") run_example(trio.testing.MockClock(rate=100 * YEAR)) diff --git a/docs/source/tutorial/echo-client.py b/docs/source/tutorial/echo-client.py index 8ee3e2a7ca..244f2831f5 100644 --- a/docs/source/tutorial/echo-client.py +++ b/docs/source/tutorial/echo-client.py @@ -9,6 +9,7 @@ # - must match what we set in our echo server PORT = 12345 + async def sender(client_stream): print("sender: started!") while True: @@ -17,6 +18,7 @@ async def sender(client_stream): await client_stream.send_all(data) await trio.sleep(1) + async def receiver(client_stream): print("receiver: started!") async for data in client_stream: @@ -24,6 +26,7 @@ async def receiver(client_stream): print("receiver: connection closed") sys.exit() + async def parent(): print(f"parent: connecting to 127.0.0.1:{PORT}") client_stream = await trio.open_tcp_stream("127.0.0.1", PORT) @@ -35,4 +38,5 @@ async def parent(): print("parent: spawning receiver...") nursery.start_soon(receiver, client_stream) + trio.run(parent) diff --git a/docs/source/tutorial/echo-server.py b/docs/source/tutorial/echo-server.py index d37e509af4..3751cadd73 100644 --- a/docs/source/tutorial/echo-server.py +++ b/docs/source/tutorial/echo-server.py @@ -11,6 +11,7 @@ CONNECTION_COUNTER = count() + async def echo_server(server_stream): # Assign each connection a unique number to make our debug prints easier # to understand when there are multiple simultaneous connections. diff --git a/docs/source/tutorial/tasks-intro.py b/docs/source/tutorial/tasks-intro.py index a316cb933d..e00de363b1 100644 --- a/docs/source/tutorial/tasks-intro.py +++ b/docs/source/tutorial/tasks-intro.py @@ -2,16 +2,19 @@ import trio + async def child1(): print(" child1: started! sleeping now...") await trio.sleep(1) print(" child1: exiting!") + async def child2(): print(" child2: started! sleeping now...") await trio.sleep(1) print(" child2: exiting!") + async def parent(): print("parent: started!") async with trio.open_nursery() as nursery: @@ -25,4 +28,5 @@ async def parent(): # -- we exit the nursery block here -- print("parent: all done!") + trio.run(parent) diff --git a/docs/source/tutorial/tasks-with-trace.py b/docs/source/tutorial/tasks-with-trace.py index 26f8fb205d..a6e40ec8ee 100644 --- a/docs/source/tutorial/tasks-with-trace.py +++ b/docs/source/tutorial/tasks-with-trace.py @@ -2,16 +2,19 @@ import trio + async def child1(): print(" child1: started! sleeping now...") await trio.sleep(1) print(" child1: exiting!") + async def child2(): print(" child2 started! sleeping now...") await trio.sleep(1) print(" child2 exiting!") + async def parent(): print("parent: started!") async with trio.open_nursery() as nursery: @@ -25,6 +28,7 @@ async def parent(): # -- we exit the nursery block here -- print("parent: all done!") + class Tracer(trio.abc.Instrument): def before_run(self): print("!!! run started") @@ -63,4 +67,5 @@ def after_io_wait(self, timeout): def after_run(self): print("!!! run finished") + trio.run(parent, instruments=[Tracer()]) diff --git a/trio/_abc.py b/trio/_abc.py index 4da9977393..c085c82b89 100644 --- a/trio/_abc.py +++ b/trio/_abc.py @@ -1,5 +1,3 @@ -# coding: utf-8 - from abc import ABCMeta, abstractmethod from typing import Generic, TypeVar import trio diff --git a/trio/_core/_asyncgens.py b/trio/_core/_asyncgens.py index cb88a1d57b..1eab150488 100644 --- a/trio/_core/_asyncgens.py +++ b/trio/_core/_asyncgens.py @@ -78,9 +78,9 @@ def finalizer(agen): # ignored, since we're running in GC context.) warnings.warn( f"Async generator {agen_name!r} was garbage collected before it " - f"had been exhausted. Surround its use in 'async with " - f"aclosing(...):' to ensure that it gets cleaned up as soon as " - f"you're done using it.", + "had been exhausted. Surround its use in 'async with " + "aclosing(...):' to ensure that it gets cleaned up as soon as " + "you're done using it.", ResourceWarning, stacklevel=2, source=agen, @@ -106,8 +106,8 @@ def finalizer(agen): # error than the default "async generator ignored GeneratorExit" raise RuntimeError( f"Non-Trio async generator {agen_name!r} awaited something " - f"during finalization; install a finalization hook to " - f"support this, or wrap it in 'async with aclosing(...):'" + "during finalization; install a finalization hook to " + "support this, or wrap it in 'async with aclosing(...):'" ) self.prev_hooks = sys.get_asyncgen_hooks() diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 139fb7830a..f898a13cff 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -92,4 +92,4 @@ def reset(self, token): token.redeemed = True def __repr__(self): - return "".format(self._name) + return f"" diff --git a/trio/_core/_multierror.py b/trio/_core/_multierror.py index c95f05b4ca..a9778fd244 100644 --- a/trio/_core/_multierror.py +++ b/trio/_core/_multierror.py @@ -217,7 +217,7 @@ def __str__(self): return ", ".join(repr(exc) for exc in self.exceptions) def __repr__(self): - return "".format(self) + return f"" def derive(self, __excs): # We use _collapse=False here to get ExceptionGroup semantics, since derive() diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 22d4513969..be0497b635 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -606,7 +606,7 @@ def __repr__(self): "from now" if self._deadline >= now else "ago", ) - return "".format(id(self), binding, state) + return f"" @contextmanager @enable_ki_protection @@ -756,7 +756,7 @@ class _TaskStatus: _value = attr.ib(default=None) def __repr__(self): - return "".format(id(self)) + return f"" def started(self, value=None): if self._called_started: @@ -1148,7 +1148,7 @@ class Task(metaclass=NoPublicConstructor): _schedule_points = attr.ib(default=0) def __repr__(self): - return "".format(self.name, id(self)) + return f"" @property def parent_nursery(self): @@ -1518,7 +1518,6 @@ def reschedule(self, task, next_send=_NO_SEND): def spawn_impl( self, async_fn, args, nursery, name, *, system_task=False, context=None ): - ###### # Make sure the nursery is in working order ###### @@ -1543,7 +1542,7 @@ def spawn_impl( name = name.func if not isinstance(name, str): try: - name = "{}.{}".format(name.__module__, name.__qualname__) + name = f"{name.__module__}.{name.__qualname__}" except AttributeError: name = repr(name) diff --git a/trio/_core/_unbounded_queue.py b/trio/_core/_unbounded_queue.py index f877e42a0c..9c747749b4 100644 --- a/trio/_core/_unbounded_queue.py +++ b/trio/_core/_unbounded_queue.py @@ -54,7 +54,7 @@ def __init__(self): self._can_get = False def __repr__(self): - return "".format(len(self._data)) + return f"" def qsize(self): """Returns the number of items currently in the queue.""" diff --git a/trio/_core/tests/test_asyncgen.py b/trio/_core/tests/test_asyncgen.py index ef837469d4..65bde5857f 100644 --- a/trio/_core/tests/test_asyncgen.py +++ b/trio/_core/tests/test_asyncgen.py @@ -226,7 +226,7 @@ async def async_main(): break else: # pragma: no cover pytest.fail( - f"Didn't manage to hit the trailing_finalizer_asyncgens case " + "Didn't manage to hit the trailing_finalizer_asyncgens case " f"despite trying {attempt} times" ) diff --git a/trio/_core/tests/test_io.py b/trio/_core/tests/test_io.py index 397375503d..916ba6cd6f 100644 --- a/trio/_core/tests/test_io.py +++ b/trio/_core/tests/test_io.py @@ -43,7 +43,7 @@ def using_fileno(fn): def fileno_wrapper(fileobj): return fn(fileobj.fileno()) - name = "<{} on fileno>".format(fn.__name__) + name = f"<{fn.__name__} on fileno>" fileno_wrapper.__name__ = fileno_wrapper.__qualname__ = name return fileno_wrapper @@ -417,7 +417,7 @@ async def allow_OSError(async_func, *args): # handle_io context rather than abort context. a, b = stdlib_socket.socketpair() with a, b, a.dup() as a2: # noqa: F841 - print("a={}, b={}, a2={}".format(a.fileno(), b.fileno(), a2.fileno())) + print(f"a={a.fileno()}, b={b.fileno()}, a2={a2.fileno()}") a.setblocking(False) b.setblocking(False) fill_socket(a) diff --git a/trio/_core/tests/test_mock_clock.py b/trio/_core/tests/test_mock_clock.py index bea9509686..e5b2373ca5 100644 --- a/trio/_core/tests/test_mock_clock.py +++ b/trio/_core/tests/test_mock_clock.py @@ -64,14 +64,14 @@ async def test_mock_clock_autojump(mock_clock): virtual_start = _core.current_time() for i in range(10): - print("sleeping {} seconds".format(10 * i)) + print(f"sleeping {10 * i} seconds") await sleep(10 * i) print("woke up!") assert virtual_start + 10 * i == _core.current_time() virtual_start = _core.current_time() real_duration = time.perf_counter() - real_start - print("Slept {} seconds in {} seconds".format(10 * sum(range(10)), real_duration)) + print(f"Slept {10 * sum(range(10))} seconds in {real_duration} seconds") assert real_duration < 1 mock_clock.autojump_threshold = 0.02 diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index edebf0dd59..a15ba3f2c9 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -77,7 +77,6 @@ def get_tb(raiser): def test_concat_tb(): - tb1 = get_tb(raiser1) tb2 = get_tb(raiser2) diff --git a/trio/_core/tests/test_parking_lot.py b/trio/_core/tests/test_parking_lot.py index 13ffe0c066..db3fc76709 100644 --- a/trio/_core/tests/test_parking_lot.py +++ b/trio/_core/tests/test_parking_lot.py @@ -10,9 +10,9 @@ async def test_parking_lot_basic(): record = [] async def waiter(i, lot): - record.append("sleep {}".format(i)) + record.append(f"sleep {i}") await lot.park() - record.append("wake {}".format(i)) + record.append(f"wake {i}") async with _core.open_nursery() as nursery: lot = ParkingLot() @@ -76,13 +76,13 @@ async def waiter(i, lot): async def cancellable_waiter(name, lot, scopes, record): with _core.CancelScope() as scope: scopes[name] = scope - record.append("sleep {}".format(name)) + record.append(f"sleep {name}") try: await lot.park() except _core.Cancelled: - record.append("cancelled {}".format(name)) + record.append(f"cancelled {name}") else: - record.append("wake {}".format(name)) + record.append(f"wake {name}") async def test_parking_lot_cancel(): diff --git a/trio/_core/tests/test_run.py b/trio/_core/tests/test_run.py index 262ad6054b..17a9cfad58 100644 --- a/trio/_core/tests/test_run.py +++ b/trio/_core/tests/test_run.py @@ -2245,7 +2245,6 @@ async def crasher(): old_flags = gc.get_debug() try: - with pytest.raises(ValueError), _core.CancelScope() as outer: async with _core.open_nursery() as nursery: gc.collect() diff --git a/trio/_deprecate.py b/trio/_deprecate.py index 4f9f15ec35..7641baefd3 100644 --- a/trio/_deprecate.py +++ b/trio/_deprecate.py @@ -30,24 +30,24 @@ class TrioDeprecationWarning(FutureWarning): def _url_for_issue(issue): - return "https://github.com/python-trio/trio/issues/{}".format(issue) + return f"https://github.com/python-trio/trio/issues/{issue}" def _stringify(thing): if hasattr(thing, "__module__") and hasattr(thing, "__qualname__"): - return "{}.{}".format(thing.__module__, thing.__qualname__) + return f"{thing.__module__}.{thing.__qualname__}" return str(thing) def warn_deprecated(thing, version, *, issue, instead, stacklevel=2): stacklevel += 1 - msg = "{} is deprecated since Trio {}".format(_stringify(thing), version) + msg = f"{_stringify(thing)} is deprecated since Trio {version}" if instead is None: msg += " with no replacement" else: - msg += "; use {} instead".format(_stringify(instead)) + msg += f"; use {_stringify(instead)} instead" if issue is not None: - msg += " ({})".format(_url_for_issue(issue)) + msg += f" ({_url_for_issue(issue)})" warnings.warn(TrioDeprecationWarning(msg), stacklevel=stacklevel) @@ -72,9 +72,9 @@ def wrapper(*args, **kwargs): doc = wrapper.__doc__ doc = doc.rstrip() doc += "\n\n" - doc += ".. deprecated:: {}\n".format(version) + doc += f".. deprecated:: {version}\n" if instead is not None: - doc += " Use {} instead.\n".format(_stringify(instead)) + doc += f" Use {_stringify(instead)} instead.\n" if issue is not None: doc += " For details, see `issue #{} <{}>`__.\n".format( issue, _url_for_issue(issue) @@ -116,7 +116,7 @@ def __getattr__(self, name): instead = info.instead if instead is DeprecatedAttribute._not_set: instead = info.value - thing = "{}.{}".format(self.__name__, name) + thing = f"{self.__name__}.{name}" warn_deprecated(thing, info.version, issue=info.issue, instead=instead) return info.value diff --git a/trio/_highlevel_open_tcp_listeners.py b/trio/_highlevel_open_tcp_listeners.py index b650ac973f..2028d30766 100644 --- a/trio/_highlevel_open_tcp_listeners.py +++ b/trio/_highlevel_open_tcp_listeners.py @@ -90,7 +90,7 @@ async def open_tcp_listeners(port, *, host=None, backlog=None): # doesn't: # http://klickverbot.at/blog/2012/01/getaddrinfo-edge-case-behavior-on-windows-linux-and-osx/ if not isinstance(port, int): - raise TypeError("port must be an int not {!r}".format(port)) + raise TypeError(f"port must be an int not {port!r}") backlog = _compute_backlog(backlog) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 740c0b1752..0fcffbcb06 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -147,9 +147,9 @@ def reorder_for_rfc_6555_section_5_4(targets): def format_host_port(host, port): host = host.decode("ascii") if isinstance(host, bytes) else host if ":" in host: - return "[{}]:{}".format(host, port) + return f"[{host}]:{port}" else: - return "{}:{}".format(host, port) + return f"{host}:{port}" # Twisted's HostnameEndpoint has a good set of configurables: @@ -251,7 +251,7 @@ async def open_tcp_stream( if host is None: raise ValueError("host cannot be None") if not isinstance(port, int): - raise TypeError("port must be int, not {!r}".format(port)) + raise TypeError(f"port must be int, not {port!r}") if happy_eyeballs_delay is None: happy_eyeballs_delay = DEFAULT_DELAY diff --git a/trio/_highlevel_socket.py b/trio/_highlevel_socket.py index 0de76aa54f..1e8dc16ebc 100644 --- a/trio/_highlevel_socket.py +++ b/trio/_highlevel_socket.py @@ -30,9 +30,7 @@ def _translate_socket_errors_to_stream_errors(): if exc.errno in _closed_stream_errnos: raise trio.ClosedResourceError("this socket was already closed") from None else: - raise trio.BrokenResourceError( - "socket connection broken: {}".format(exc) - ) from exc + raise trio.BrokenResourceError(f"socket connection broken: {exc}") from exc class SocketStream(HalfCloseableStream, metaclass=Final): diff --git a/trio/_path.py b/trio/_path.py index 4077c449d7..ea8cf98c34 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -166,7 +166,7 @@ def __dir__(self): return super().__dir__() + self._forward def __repr__(self): - return "trio.Path({})".format(repr(str(self))) + return f"trio.Path({repr(str(self))})" def __fspath__(self): return os.fspath(self._wrapped) diff --git a/trio/_ssl.py b/trio/_ssl.py index f9bb90afd8..8f005c2c9a 100644 --- a/trio/_ssl.py +++ b/trio/_ssl.py @@ -402,9 +402,7 @@ def __init__( def __getattr__(self, name): if name in self._forwarded: if name in self._after_handshake and not self._handshook.done: - raise NeedHandshakeError( - "call do_handshake() before calling {!r}".format(name) - ) + raise NeedHandshakeError(f"call do_handshake() before calling {name!r}") return getattr(self._ssl_object, name) else: diff --git a/trio/_subprocess.py b/trio/_subprocess.py index 2bb0adcbdc..75c94c731d 100644 --- a/trio/_subprocess.py +++ b/trio/_subprocess.py @@ -1,5 +1,3 @@ -# coding: utf-8 - import os import subprocess import sys @@ -152,13 +150,13 @@ def __init__(self, popen, stdin, stdout, stderr): def __repr__(self): returncode = self.returncode if returncode is None: - status = "running with PID {}".format(self.pid) + status = f"running with PID {self.pid}" else: if returncode < 0: - status = "exited with signal {}".format(-returncode) + status = f"exited with signal {-returncode}" else: - status = "exited with status {}".format(returncode) - return "".format(self.args, status) + status = f"exited with status {returncode}" + return f"" @property def returncode(self): @@ -430,8 +428,8 @@ async def _posix_deliver_cancel(p): warnings.warn( RuntimeWarning( f"process {p!r} ignored SIGTERM for 5 seconds. " - f"(Maybe you should pass a custom deliver_cancel?) " - f"Trying SIGKILL." + "(Maybe you should pass a custom deliver_cancel?) " + "Trying SIGKILL." ) ) p.kill() diff --git a/trio/_sync.py b/trio/_sync.py index c8c6d0dfe5..8d2fdc0a2d 100644 --- a/trio/_sync.py +++ b/trio/_sync.py @@ -245,8 +245,7 @@ def acquire_on_behalf_of_nowait(self, borrower): """ if borrower in self._borrowers: raise RuntimeError( - "this borrower is already holding one of this " - "CapacityLimiter's tokens" + "this borrower is already holding one of this CapacityLimiter's tokens" ) if len(self._borrowers) < self._total_tokens and not self._lot: self._borrowers.add(borrower) @@ -396,7 +395,7 @@ def __repr__(self): if self._max_value is None: max_value_str = "" else: - max_value_str = ", max_value={}".format(self._max_value) + max_value_str = f", max_value={self._max_value}" return "".format( self._value, max_value_str, id(self) ) @@ -484,7 +483,7 @@ class _LockImpl(AsyncContextManagerMixin): def __repr__(self): if self.locked(): s1 = "locked" - s2 = " with {} waiters".format(len(self._lot)) + s2 = f" with {len(self._lot)} waiters" else: s1 = "unlocked" s2 = "" diff --git a/trio/_threads.py b/trio/_threads.py index c8cab2ee76..80f9492036 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -1,5 +1,3 @@ -# coding: utf-8 - import contextvars import threading import queue as stdlib_queue diff --git a/trio/_tools/gen_exports.py b/trio/_tools/gen_exports.py index d7e6326ce6..3c18a86298 100755 --- a/trio/_tools/gen_exports.py +++ b/trio/_tools/gen_exports.py @@ -1,5 +1,4 @@ #! /usr/bin/env python3 -# -*- coding: utf-8 -`- """ Code generation script for class methods to be exported as public API @@ -134,7 +133,7 @@ def matches_disk_files(new_files): for new_path, new_source in new_files.items(): if not os.path.exists(new_path): return False - with open(new_path, "r", encoding="utf-8") as old_file: + with open(new_path, encoding="utf-8") as old_file: old_source = old_file.read() if old_source != new_source: return False diff --git a/trio/_util.py b/trio/_util.py index 58e6eddf29..063d6fcf9a 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -1,5 +1,3 @@ -# coding: utf-8 - # Little utilities we use internally from abc import ABCMeta @@ -288,7 +286,8 @@ def __new__(cls, name, bases, cls_namespace): for base in bases: if isinstance(base, Final): raise TypeError( - f"{base.__module__}.{base.__qualname__} does not support subclassing" + f"{base.__module__}.{base.__qualname__} does not support" + " subclassing" ) return super().__new__(cls, name, bases, cls_namespace) @@ -333,7 +332,7 @@ def name_asyncgen(agen): try: module = agen.ag_frame.f_globals["__name__"] except (AttributeError, KeyError): - module = "<{}>".format(agen.ag_code.co_filename) + module = f"<{agen.ag_code.co_filename}>" try: qualname = agen.__qualname__ except AttributeError: diff --git a/trio/_wait_for_object.py b/trio/_wait_for_object.py index 07c0461429..2e24682444 100644 --- a/trio/_wait_for_object.py +++ b/trio/_wait_for_object.py @@ -53,7 +53,7 @@ async def WaitForSingleObject(obj): def WaitForMultipleObjects_sync(*handles): """Wait for any of the given Windows handles to be signaled.""" n = len(handles) - handle_arr = ffi.new("HANDLE[{}]".format(n)) + handle_arr = ffi.new(f"HANDLE[{n}]") for i in range(n): handle_arr[i] = handles[i] timeout = 0xFFFFFFFF # INFINITE diff --git a/trio/testing/_check_streams.py b/trio/testing/_check_streams.py index de1f5fe687..0206f1f737 100644 --- a/trio/testing/_check_streams.py +++ b/trio/testing/_check_streams.py @@ -31,7 +31,7 @@ def _assert_raises(exc): except exc: pass else: - raise AssertionError("expected exception: {}".format(exc)) + raise AssertionError(f"expected exception: {exc}") async def check_one_way_stream(stream_maker, clogged_stream_maker): diff --git a/trio/testing/_sequencer.py b/trio/testing/_sequencer.py index c4e34717a1..84eb217601 100644 --- a/trio/testing/_sequencer.py +++ b/trio/testing/_sequencer.py @@ -61,7 +61,7 @@ async def main(): @asynccontextmanager async def __call__(self, position: int): if position in self._claimed: - raise RuntimeError("Attempted to re-use sequence point {}".format(position)) + raise RuntimeError(f"Attempted to re-use sequence point {position}") if self._broken: raise RuntimeError("sequence broken!") self._claimed.add(position) diff --git a/trio/tests/test_channel.py b/trio/tests/test_channel.py index fd990fb3e3..aabb368799 100644 --- a/trio/tests/test_channel.py +++ b/trio/tests/test_channel.py @@ -346,7 +346,6 @@ async def test_statistics(): async def test_channel_fairness(): - # We can remove an item we just sent, and send an item back in after, if # no-one else is waiting. s, r = open_memory_channel(1) diff --git a/trio/tests/test_dtls.py b/trio/tests/test_dtls.py index dbb489b9a8..680a8793eb 100644 --- a/trio/tests/test_dtls.py +++ b/trio/tests/test_dtls.py @@ -50,7 +50,7 @@ async def dtls_echo_server(*, autocancel=True, mtu=None, ipv6=False): async def echo_handler(dtls_channel): print( - f"echo handler started: " + "echo handler started: " f"server {dtls_channel.endpoint.socket.getsockname()} " f"client {dtls_channel.peer_address}" ) @@ -148,7 +148,8 @@ async def route_packet(packet): else: assert op == "deliver" print( - f"{packet.source} -> {packet.destination}: delivered {packet.payload.hex()}" + f"{packet.source} -> {packet.destination}: delivered" + f" {packet.payload.hex()}" ) fn.deliver_packet(packet) break @@ -446,7 +447,6 @@ async def test_invalid_cookie_rejected(autojump_clock): from trio._dtls import decode_client_hello_untrusted, BadPacket with trio.CancelScope() as cscope: - # the first 11 bytes of ClientHello aren't protected by the cookie, so only test # corrupting bytes after that. offset_to_corrupt = count(11) diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 8d6b2d6131..026d6f5efa 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -93,7 +93,7 @@ def no_underscores(symbols): import jedi # Simulate typing "import trio; trio." - script = jedi.Script("import {}; {}.".format(modname, modname)) + script = jedi.Script(f"import {modname}; {modname}.") completions = script.complete() static_names = no_underscores(c.name for c in completions) else: # pragma: no cover @@ -107,10 +107,10 @@ def no_underscores(symbols): # So we check that the runtime names are a subset of the static names. missing_names = runtime_names - static_names if missing_names: # pragma: no cover - print("{} can't see the following names in {}:".format(tool, modname)) + print(f"{tool} can't see the following names in {modname}:") print() for name in sorted(missing_names): - print(" {}".format(name)) + print(f" {name}") assert False diff --git a/trio/tests/test_file_io.py b/trio/tests/test_file_io.py index b40f7518a9..dcbd1a63bb 100644 --- a/trio/tests/test_file_io.py +++ b/trio/tests/test_file_io.py @@ -27,7 +27,7 @@ def async_file(wrapped): def test_wrap_invalid(): with pytest.raises(TypeError): - trio.wrap_file(str()) + trio.wrap_file("") def test_wrap_non_iobase(): diff --git a/trio/tests/test_highlevel_open_tcp_listeners.py b/trio/tests/test_highlevel_open_tcp_listeners.py index 103984739e..0c38b4ca69 100644 --- a/trio/tests/test_highlevel_open_tcp_listeners.py +++ b/trio/tests/test_highlevel_open_tcp_listeners.py @@ -288,7 +288,7 @@ async def test_open_tcp_listeners_socket_fails_not_afnosupport(): async def test_open_tcp_listeners_backlog(): fsf = FakeSocketFactory(99) tsocket.set_custom_socket_factory(fsf) - for (given, expected) in [ + for given, expected in [ (None, 0xFFFF), (99999999, 0xFFFF), (10, 10), diff --git a/trio/tests/test_highlevel_open_tcp_stream.py b/trio/tests/test_highlevel_open_tcp_stream.py index 0f3b6a0baf..35ddd3e118 100644 --- a/trio/tests/test_highlevel_open_tcp_stream.py +++ b/trio/tests/test_highlevel_open_tcp_stream.py @@ -55,11 +55,11 @@ def fake4(i): SOCK_STREAM, IPPROTO_TCP, "", - ("10.0.0.{}".format(i), 80), + (f"10.0.0.{i}", 80), ) def fake6(i): - return (AF_INET6, SOCK_STREAM, IPPROTO_TCP, "", ("::{}".format(i), 80)) + return (AF_INET6, SOCK_STREAM, IPPROTO_TCP, "", (f"::{i}", 80)) for fake in fake4, fake6: # No effect on homogeneous lists diff --git a/trio/tests/test_path.py b/trio/tests/test_path.py index 284bcf82dd..b4345e4d55 100644 --- a/trio/tests/test_path.py +++ b/trio/tests/test_path.py @@ -104,7 +104,6 @@ async def test_async_method_signature(path): @pytest.mark.parametrize("method_name", ["is_dir", "is_file"]) async def test_compare_async_stat_methods(method_name): - method, async_method = method_pair(".", method_name) result = method() @@ -120,7 +119,6 @@ async def test_invalid_name_not_wrapped(path): @pytest.mark.parametrize("method_name", ["absolute", "resolve"]) async def test_async_methods_rewrap(method_name): - method, async_method = method_pair(".", method_name) result = method() diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index ea5e48fe2d..db21096fac 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -46,7 +46,7 @@ def getaddrinfo(self, *args, **kwargs): elif bound[-1] & stdlib_socket.AI_NUMERICHOST: return self._orig_getaddrinfo(*args, **kwargs) else: - raise RuntimeError("gai called with unexpected arguments {}".format(bound)) + raise RuntimeError(f"gai called with unexpected arguments {bound}") @pytest.fixture @@ -965,7 +965,7 @@ async def check_AF_UNIX(path): # Can't use tmpdir fixture, because we can exceed the maximum AF_UNIX path # length on macOS. with tempfile.TemporaryDirectory() as tmpdir: - path = "{}/sock".format(tmpdir) + path = f"{tmpdir}/sock" await check_AF_UNIX(path) try: diff --git a/trio/tests/test_subprocess.py b/trio/tests/test_subprocess.py index 23aeebc587..e2d66f654d 100644 --- a/trio/tests/test_subprocess.py +++ b/trio/tests/test_subprocess.py @@ -45,7 +45,7 @@ def python(code): if posix: SLEEP = lambda seconds: ["/bin/sleep", str(seconds)] else: - SLEEP = lambda seconds: python("import time; time.sleep({})".format(seconds)) + SLEEP = lambda seconds: python(f"import time; time.sleep({seconds})") def got_signal(proc, sig): @@ -224,7 +224,6 @@ async def test_interactive(background_process): stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) as proc: - newline = b"\n" if posix else b"\r\n" async def expect(idx, request): @@ -233,9 +232,7 @@ async def expect(idx, request): async def drain_one(stream, count, digit): while count > 0: result = await stream.receive_some(count) - assert result == ( - "{}".format(digit).encode("utf-8") * len(result) - ) + assert result == (f"{digit}".encode() * len(result)) count -= len(result) assert count == 0 assert await stream.receive_some(len(newline)) == newline diff --git a/trio/tests/test_testing.py b/trio/tests/test_testing.py index 0b10ae71e1..a2dba728d5 100644 --- a/trio/tests/test_testing.py +++ b/trio/tests/test_testing.py @@ -213,7 +213,7 @@ async def child(i): async with seq(i): pass # pragma: no cover except RuntimeError: - record.append("seq({}) RuntimeError".format(i)) + record.append(f"seq({i}) RuntimeError") async with _core.open_nursery() as nursery: nursery.start_soon(child, 1) @@ -651,7 +651,7 @@ async def check(listener): # can't use pytest's tmpdir; if we try then macOS says "OSError: # AF_UNIX path too long" with tempfile.TemporaryDirectory() as tmpdir: - path = "{}/sock".format(tmpdir) + path = f"{tmpdir}/sock" await sock.bind(path) sock.listen(10) await check(SocketListener(sock)) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index baff18278d..9b05a526ff 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -509,9 +509,10 @@ def g(): sniffio_outer_value = current_async_library_cvar.get() assert parent_value == "main" assert inner_value == "worker" - assert ( - current_value == "main" - ), "The contextvar value set on the worker would not propagate back to the main thread" + assert current_value == "main", ( + "The contextvar value set on the worker would not propagate back to the main" + " thread" + ) assert sniffio_cvar_value is None assert sniffio_outer_value == "trio" @@ -741,7 +742,6 @@ async def test_trio_token_weak_referenceable(): async def test_unsafe_cancellable_kwarg(): - # This is a stand in for a numpy ndarray or other objects # that (maybe surprisingly) lack a notion of truthiness class BadBool: diff --git a/trio/tests/tools/test_gen_exports.py b/trio/tests/tools/test_gen_exports.py index e4e388c226..73eacc098a 100644 --- a/trio/tests/tools/test_gen_exports.py +++ b/trio/tests/tools/test_gen_exports.py @@ -49,7 +49,7 @@ def test_create_pass_through_args(): ), ] - for (funcdef, expected) in testcases: + for funcdef, expected in testcases: func_node = ast.parse(funcdef + ":\n pass").body[0] assert isinstance(func_node, ast.FunctionDef) assert create_passthrough_args(func_node) == expected From 9c86e9c8cdea5b655f4488535d788cd0caaeef01 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Tue, 8 Nov 2022 15:42:04 +0100 Subject: [PATCH 1246/1498] Add name parameter to to_thread_run_sync, setting thread name with pthread if available --- newsfragments/1148.feature.rst | 1 + trio/_core/_thread_cache.py | 76 ++++++++++++++++++++++++++++---- trio/_threads.py | 21 ++++++++- trio/tests/test_threads.py | 79 +++++++++++++++++++++++++++++++--- 4 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 newsfragments/1148.feature.rst diff --git a/newsfragments/1148.feature.rst b/newsfragments/1148.feature.rst new file mode 100644 index 0000000000..7d9ead3819 --- /dev/null +++ b/newsfragments/1148.feature.rst @@ -0,0 +1 @@ +Added support for naming threads created with `to_thread_run_sync`, requires pthreads so is only available on POSIX platforms with glibc installed. diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index d11a23d27a..593b6f9e6f 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -2,8 +2,47 @@ import traceback from threading import Thread, Lock import outcome +import sys +import ctypes +import ctypes.util from itertools import count +from typing import Callable, Optional, Tuple +from functools import partial + + +def get_os_thread_name_func() -> Optional[Callable[[Optional[int], str], None]]: + def namefunc(setname: Callable[[int, bytes], int], ident: Optional[int], name: str): + if ident is not None: + setname(ident, bytes(name[:15], "ascii", "replace")) + + libpthread_path = ctypes.util.find_library("pthread") + if not libpthread_path: + return None + libpthread = ctypes.CDLL(libpthread_path) + + pthread_setname_np = getattr(libpthread, "pthread_setname_np", None) + if pthread_setname_np is None: + return None + + # specify function prototype + pthread_setname_np.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + pthread_setname_np.restype = ctypes.c_int + + return partial(namefunc, pthread_setname_np) + ## set the name + # try: + # bname = name.encode("ascii", "replace") + # thread = threading.current_thread() + # # if thread is not None: + # thread.name = name + # pthread_setname_np(thread.ident, bname[:15]) + # except Exception: + # return nofunc + + +set_os_thread_name = get_os_thread_name_func() + # The "thread cache" is a simple unbounded thread pool, i.e., it automatically # spawns as many threads as needed to handle all the requests its given. Its # only purpose is to cache worker threads so that they don't have to be @@ -44,7 +83,7 @@ class WorkerThread: def __init__(self, thread_cache): - self._job = None + self._job: Optional[Tuple[Callable, Callable, str]] = None self._thread_cache = thread_cache # This Lock is used in an unconventional way. # @@ -54,16 +93,35 @@ def __init__(self, thread_cache): # Initially we have no job, so it starts out in locked state. self._worker_lock = Lock() self._worker_lock.acquire() - thread = Thread(target=self._work, daemon=True) - thread.name = f"Trio worker thread {next(name_counter)}" - thread.start() + self._default_name = f"Trio thread {next(name_counter)}" + + self._thread = Thread(target=self._work, daemon=True) + self._thread.name = self._default_name + + if set_os_thread_name: + set_os_thread_name(self._thread.ident, self._default_name) + self._thread.start() def _handle_job(self): # Handle job in a separate method to ensure user-created # objects are cleaned up in a consistent manner. - fn, deliver = self._job + assert self._job is not None + fn, deliver, name = self._job self._job = None + + # set name + if name is not None: + self._thread.name = name + if set_os_thread_name: + set_os_thread_name(self._thread.ident, name) result = outcome.capture(fn) + + # reset name if it was changed + if name is not None: + self._thread.name = self._default_name + if set_os_thread_name: + set_os_thread_name(self._thread.ident, self._default_name) + # Tell the cache that we're available to be assigned a new # job. We do this *before* calling 'deliver', so that if # 'deliver' triggers a new job, it can be assigned to us @@ -102,19 +160,19 @@ class ThreadCache: def __init__(self): self._idle_workers = {} - def start_thread_soon(self, fn, deliver): + def start_thread_soon(self, fn, deliver, name: Optional[str] = None): try: worker, _ = self._idle_workers.popitem() except KeyError: worker = WorkerThread(self) - worker._job = (fn, deliver) + worker._job = (fn, deliver, name) worker._worker_lock.release() THREAD_CACHE = ThreadCache() -def start_thread_soon(fn, deliver): +def start_thread_soon(fn, deliver, name: Optional[str] = None): """Runs ``deliver(outcome.capture(fn))`` in a worker thread. Generally ``fn`` does some blocking work, and ``deliver`` delivers the @@ -174,4 +232,4 @@ def start_thread_soon(fn, deliver): limit how many threads they're using then it's polite to respect that. """ - THREAD_CACHE.start_thread_soon(fn, deliver) + THREAD_CACHE.start_thread_soon(fn, deliver, name) diff --git a/trio/_threads.py b/trio/_threads.py index c8cab2ee76..e4ec7dcec2 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -58,8 +58,13 @@ class ThreadPlaceholder: name = attr.ib() +from typing import Optional + + @enable_ki_protection -async def to_thread_run_sync(sync_fn, *args, cancellable=False, limiter=None): +async def to_thread_run_sync( + sync_fn, *args, thread_name: Optional[str] = None, cancellable=False, limiter=None +): """Convert a blocking operation into an async operation using a thread. These two lines are equivalent:: @@ -81,6 +86,12 @@ async def to_thread_run_sync(sync_fn, *args, cancellable=False, limiter=None): arguments, use :func:`functools.partial`. cancellable (bool): Whether to allow cancellation of this operation. See discussion below. + name (str): Optional string to set the name of the thread. + Will always set `threading.Thread.name`, but only set the os name + if pthread.h is available (i.e. most POSIX installations). + pthread names are limited to 15 characters, and can be read from + ``/proc//task//comm`` or with ``ps -eT``, among others. + Defaults to ``Thread for {trio.lowlevel.current_task().name}``. limiter (None, or CapacityLimiter-like object): An object used to limit the number of simultaneous threads. Most commonly this will be a `~trio.CapacityLimiter`, but it could be @@ -168,6 +179,10 @@ def do_release_then_return_result(): current_trio_token = trio.lowlevel.current_trio_token() + if thread_name is None: + # TODO, better default since it caps at 15 chars + thread_name = f"Thread for {trio.lowlevel.current_task().name}" + def worker_fn(): current_async_library_cvar.set(None) TOKEN_LOCAL.token = current_trio_token @@ -200,7 +215,9 @@ def deliver_worker_fn_result(result): await limiter.acquire_on_behalf_of(placeholder) try: - start_thread_soon(contextvars_aware_worker_fn, deliver_worker_fn_result) + start_thread_soon( + contextvars_aware_worker_fn, deliver_worker_fn_result, thread_name + ) except: limiter.release_on_behalf_of(placeholder) raise diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index baff18278d..7b3418a74c 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -1,25 +1,28 @@ import contextvars -import threading import queue as stdlib_queue +import re +import sys +import threading import time import weakref +from functools import partial +from typing import Callable, Optional import pytest from sniffio import current_async_library_cvar + from trio._core import TrioToken, current_trio_token -from .. import _core -from .. import Event, CapacityLimiter, sleep -from ..testing import wait_all_tasks_blocked +from .. import CapacityLimiter, Event, _core, sleep +from .._core.tests.test_ki import ki_self from .._core.tests.tutil import buggy_pypy_asyncgens from .._threads import ( - to_thread_run_sync, current_default_thread_limiter, from_thread_run, from_thread_run_sync, + to_thread_run_sync, ) - -from .._core.tests.test_ki import ki_self +from ..testing import wait_all_tasks_blocked async def test_do_in_trio_thread(): @@ -164,6 +167,68 @@ async def main(): assert record == ["sleeping", "cancelled"] +def _get_thread_name(ident: Optional[int] = None) -> Optional[str]: + import ctypes + import ctypes.util + + # afaik pthread should be available on MacOS as well, but it's apparently + # not on the CI machines. + libpthread_path = ctypes.util.find_library("pthread") + assert libpthread_path, sys.platform + libpthread = ctypes.CDLL(libpthread_path) + + pthread_getname_np = getattr(libpthread, "pthread_getname_np", None) + assert pthread_getname_np is not None + + pthread_getname_np.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t, + ] + pthread_getname_np.restype = ctypes.c_int + + name_buffer = ctypes.create_string_buffer(b"", size=16) + if ident is None: + ident = threading.get_ident() + assert pthread_getname_np(ident, name_buffer, 16) == 0 + return name_buffer.value.decode() + + +async def test_named_thread(): + def inner(name, os_name) -> threading.Thread: + assert threading.current_thread().name == name + if sys.platform == "linux": + assert _get_thread_name() == os_name[:15] + return threading.current_thread() + + def f(name: str, os_name: str) -> Callable[[None], threading.Thread]: + return partial(inner, name, os_name) + + # test defaults + default = "Thread for trio.tests.test_threads.test_named_thread" + await to_thread_run_sync(f(default, default)) + await to_thread_run_sync(f(default, default), thread_name=None) + + # test that you can set a custom name, and that it's reset afterwards + async def test_thread_name(name: str, os_name: Optional[str] = None): + if os_name is None: + os_name = name + thread = await to_thread_run_sync(f(name, os_name), thread_name=name) + assert re.match("Trio thread [0-9]*", thread.name) + + if sys.platform == "linux": + os_thread_name = _get_thread_name(thread.ident) + assert os_thread_name is not None and re.match( + "Trio thread [0-9]*", os_thread_name + ) + + await test_thread_name("") + await test_thread_name("fobiedoo") + await test_thread_name("name_longer_than_15_characters") + + await test_thread_name("💙", os_name="?") + + async def test_run_in_worker_thread(): trio_thread = threading.current_thread() From 4690bb982388dd3d7bb7865973c87e3a8ad808fe Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:13:41 +0100 Subject: [PATCH 1247/1498] Apply suggestions from code review Co-authored-by: Zac Hatfield-Dodds --- newsfragments/1148.feature.rst | 2 +- trio/_core/_thread_cache.py | 12 +----------- trio/_threads.py | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/newsfragments/1148.feature.rst b/newsfragments/1148.feature.rst index 7d9ead3819..f01eae7777 100644 --- a/newsfragments/1148.feature.rst +++ b/newsfragments/1148.feature.rst @@ -1 +1 @@ -Added support for naming threads created with `to_thread_run_sync`, requires pthreads so is only available on POSIX platforms with glibc installed. +Added support for naming threads created with `trio.to_thread_run_sync`, requires pthreads so is only available on POSIX platforms with glibc installed. diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index 593b6f9e6f..acacdff662 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -30,15 +30,6 @@ def namefunc(setname: Callable[[int, bytes], int], ident: Optional[int], name: s pthread_setname_np.restype = ctypes.c_int return partial(namefunc, pthread_setname_np) - ## set the name - # try: - # bname = name.encode("ascii", "replace") - # thread = threading.current_thread() - # # if thread is not None: - # thread.name = name - # pthread_setname_np(thread.ident, bname[:15]) - # except Exception: - # return nofunc set_os_thread_name = get_os_thread_name_func() @@ -95,8 +86,7 @@ def __init__(self, thread_cache): self._worker_lock.acquire() self._default_name = f"Trio thread {next(name_counter)}" - self._thread = Thread(target=self._work, daemon=True) - self._thread.name = self._default_name + self._thread = Thread(target=self._work, name=self._default_name, daemon=True) if set_os_thread_name: set_os_thread_name(self._thread.ident, self._default_name) diff --git a/trio/_threads.py b/trio/_threads.py index e4ec7dcec2..33517440c4 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -86,7 +86,7 @@ async def to_thread_run_sync( arguments, use :func:`functools.partial`. cancellable (bool): Whether to allow cancellation of this operation. See discussion below. - name (str): Optional string to set the name of the thread. + thread_name (str): Optional string to set the name of the thread. Will always set `threading.Thread.name`, but only set the os name if pthread.h is available (i.e. most POSIX installations). pthread names are limited to 15 characters, and can be read from From dc744bf4228db630fc9245b60fbfd249b081d20f Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 14 Nov 2022 11:01:38 +0100 Subject: [PATCH 1248/1498] split out os thread name testing --- trio/tests/test_threads.py | 74 ++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 7b3418a74c..5c2aa51b5c 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -167,18 +167,45 @@ async def main(): assert record == ["sleeping", "cancelled"] +async def test_named_thread(): + def inner(name) -> threading.Thread: + assert threading.current_thread().name == name + return threading.current_thread() + + def f(name: str) -> Callable[[None], threading.Thread]: + return partial(inner, name) + + # test defaults + default = "Thread for trio.tests.test_threads.test_named_thread" + await to_thread_run_sync(f(default)) + await to_thread_run_sync(f(default), thread_name=None) + + # test that you can set a custom name, and that it's reset afterwards + async def test_thread_name(name: str): + thread = await to_thread_run_sync(f(name), thread_name=name) + assert re.match("Trio thread [0-9]*", thread.name) + + await test_thread_name("") + await test_thread_name("fobiedoo") + await test_thread_name("name_longer_than_15_characters") + + await test_thread_name("💙") + + def _get_thread_name(ident: Optional[int] = None) -> Optional[str]: import ctypes import ctypes.util - # afaik pthread should be available on MacOS as well, but it's apparently - # not on the CI machines. libpthread_path = ctypes.util.find_library("pthread") - assert libpthread_path, sys.platform + if not libpthread_path: + print(f"no pthread on {sys.platform})") + return None libpthread = ctypes.CDLL(libpthread_path) pthread_getname_np = getattr(libpthread, "pthread_getname_np", None) - assert pthread_getname_np is not None + if not pthread_getname_np: + print(f"no pthread_getname_np on {sys.platform})") + return None pthread_getname_np.argtypes = [ ctypes.c_void_p, @@ -194,15 +221,19 @@ def _get_thread_name(ident: Optional[int] = None) -> Optional[str]: return name_buffer.value.decode() -async def test_named_thread(): - def inner(name, os_name) -> threading.Thread: - assert threading.current_thread().name == name - if sys.platform == "linux": - assert _get_thread_name() == os_name[:15] +# test os thread naming +# afaik pthread should be available on MacOS as well, but it's apparently +# not on the CI machines. +async def test_named_thread_os(): + def inner(name) -> threading.Thread: + if not _get_thread_name() == name[:15]: + assert sys.platform != "linux", "os threads should work on linux" + pytest.skip(f"no pthread OS support in {sys.platform}") + return threading.current_thread() - def f(name: str, os_name: str) -> Callable[[None], threading.Thread]: - return partial(inner, name, os_name) + def f(name: str, expected: str) -> Callable[[None], threading.Thread]: + return partial(inner, name) # test defaults default = "Thread for trio.tests.test_threads.test_named_thread" @@ -210,23 +241,22 @@ def f(name: str, os_name: str) -> Callable[[None], threading.Thread]: await to_thread_run_sync(f(default, default), thread_name=None) # test that you can set a custom name, and that it's reset afterwards - async def test_thread_name(name: str, os_name: Optional[str] = None): - if os_name is None: - os_name = name - thread = await to_thread_run_sync(f(name, os_name), thread_name=name) - assert re.match("Trio thread [0-9]*", thread.name) + async def test_thread_name(name: str, expected: Optional[str] = None): + if expected is None: + expected = name + thread = await to_thread_run_sync(f(name, expected), thread_name=name) - if sys.platform == "linux": - os_thread_name = _get_thread_name(thread.ident) - assert os_thread_name is not None and re.match( - "Trio thread [0-9]*", os_thread_name - ) + os_thread_name = _get_thread_name(thread.ident) + if os_thread_name is None: + assert sys.platform == "linux", "os threads should work on linux" + pytest.skip(f"no pthread OS support in {sys.platform}") + assert re.match("Trio thread [0-9]*", os_thread_name) await test_thread_name("") await test_thread_name("fobiedoo") await test_thread_name("name_longer_than_15_characters") - await test_thread_name("💙", os_name="?") + await test_thread_name("💙", expected="?") async def test_run_in_worker_thread(): From de1ffed26a813079e0d50abb1cd00092dfb50d14 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 14 Nov 2022 11:07:30 +0100 Subject: [PATCH 1249/1498] fix doc build warning --- newsfragments/1148.feature.rst | 2 +- trio/_threads.py | 19 ++++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/newsfragments/1148.feature.rst b/newsfragments/1148.feature.rst index f01eae7777..51f2b792c3 100644 --- a/newsfragments/1148.feature.rst +++ b/newsfragments/1148.feature.rst @@ -1 +1 @@ -Added support for naming threads created with `trio.to_thread_run_sync`, requires pthreads so is only available on POSIX platforms with glibc installed. +Added support for naming threads created with `trio.to_thread.run_sync`, requires pthreads so is only available on POSIX platforms with glibc installed. diff --git a/trio/_threads.py b/trio/_threads.py index 33517440c4..e493281a7e 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -1,26 +1,22 @@ # coding: utf-8 import contextvars -import threading -import queue as stdlib_queue import functools +import inspect +import queue as stdlib_queue +import threading from itertools import count +from typing import Optional import attr -import inspect import outcome from sniffio import current_async_library_cvar import trio +from ._core import (RunVar, TrioToken, disable_ki_protection, + enable_ki_protection, start_thread_soon) from ._sync import CapacityLimiter -from ._core import ( - enable_ki_protection, - disable_ki_protection, - RunVar, - TrioToken, - start_thread_soon, -) from ._util import coroutine_or_error # Global due to Threading API, thread local storage for trio token @@ -58,9 +54,6 @@ class ThreadPlaceholder: name = attr.ib() -from typing import Optional - - @enable_ki_protection async def to_thread_run_sync( sync_fn, *args, thread_name: Optional[str] = None, cancellable=False, limiter=None From 85510db1b7a8525021d332e926c7053918c53603 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 14 Nov 2022 11:15:49 +0100 Subject: [PATCH 1250/1498] fix tests --- trio/_threads.py | 9 +++++++-- trio/tests/test_threads.py | 22 +++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/trio/_threads.py b/trio/_threads.py index e493281a7e..1a7d7bebf9 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -14,8 +14,13 @@ import trio -from ._core import (RunVar, TrioToken, disable_ki_protection, - enable_ki_protection, start_thread_soon) +from ._core import ( + RunVar, + TrioToken, + disable_ki_protection, + enable_ki_protection, + start_thread_soon, +) from ._sync import CapacityLimiter from ._util import coroutine_or_error diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 5c2aa51b5c..6588d59cd4 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -218,7 +218,10 @@ def _get_thread_name(ident: Optional[int] = None) -> Optional[str]: if ident is None: ident = threading.get_ident() assert pthread_getname_np(ident, name_buffer, 16) == 0 - return name_buffer.value.decode() + try: + return name_buffer.value.decode() + except UnicodeDecodeError as e: + pytest.fail(f"value: {name_buffer.value!r}, exception: {e}") # test os thread naming @@ -232,19 +235,19 @@ def inner(name) -> threading.Thread: return threading.current_thread() - def f(name: str, expected: str) -> Callable[[None], threading.Thread]: + def f(name: str) -> Callable[[None], threading.Thread]: return partial(inner, name) # test defaults default = "Thread for trio.tests.test_threads.test_named_thread" - await to_thread_run_sync(f(default, default)) - await to_thread_run_sync(f(default, default), thread_name=None) + await to_thread_run_sync(f(default)) + await to_thread_run_sync(f(default), thread_name=None) # test that you can set a custom name, and that it's reset afterwards async def test_thread_name(name: str, expected: Optional[str] = None): if expected is None: expected = name - thread = await to_thread_run_sync(f(name, expected), thread_name=name) + thread = await to_thread_run_sync(f(expected), thread_name=name) os_thread_name = _get_thread_name(thread.ident) if os_thread_name is None: @@ -259,6 +262,15 @@ async def test_thread_name(name: str, expected: Optional[str] = None): await test_thread_name("💙", expected="?") +async def test_has_pthread_setname_np(): + from trio._core._thread_cache import get_os_thread_name_func + + k = get_os_thread_name_func() + if k is None: + assert sys.platform != "linux" + pytest.skip(f"no pthread_setname_np on {sys.platform}") + + async def test_run_in_worker_thread(): trio_thread = threading.current_thread() From f355c65fb05d1e6400cb16297f51a0d9f54ef580 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 14 Nov 2022 12:58:54 +0100 Subject: [PATCH 1251/1498] debugging Mac OSX with CI --- trio/_core/_thread_cache.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index acacdff662..a80b866b8b 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -16,6 +16,12 @@ def namefunc(setname: Callable[[int, bytes], int], ident: Optional[int], name: s if ident is not None: setname(ident, bytes(name[:15], "ascii", "replace")) + def darwin_namefunc( + setname: Callable[[bytes], int], ident: Optional[int], name: str + ): + if ident is not None: + setname(bytes(name[:15], "ascii", "replace")) + libpthread_path = ctypes.util.find_library("pthread") if not libpthread_path: return None @@ -26,9 +32,14 @@ def namefunc(setname: Callable[[int, bytes], int], ident: Optional[int], name: s return None # specify function prototype - pthread_setname_np.argtypes = [ctypes.c_void_p, ctypes.c_char_p] pthread_setname_np.restype = ctypes.c_int + # on mac OSX pthread_setname_np does not take a thread id + if sys.platform == "darwin": + pthread_setname_np.argtypes = [ctypes.c_char_p] + return partial(darwin_namefunc, pthread_setname_np) + + pthread_setname_np.argtypes = [ctypes.c_void_p, ctypes.c_char_p] return partial(namefunc, pthread_setname_np) From 4cb23deb555715f9f981f3c0d6547386157f06e9 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Thu, 24 Nov 2022 11:35:54 +0100 Subject: [PATCH 1252/1498] add comments, clean up, change default name to include function name --- trio/_core/_thread_cache.py | 34 +++++++++++++++++++++++------ trio/_threads.py | 5 ++--- trio/tests/test_threads.py | 43 ++++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 27 deletions(-) diff --git a/trio/_core/_thread_cache.py b/trio/_core/_thread_cache.py index a80b866b8b..a36181ee36 100644 --- a/trio/_core/_thread_cache.py +++ b/trio/_core/_thread_cache.py @@ -2,7 +2,6 @@ import traceback from threading import Thread, Lock import outcome -import sys import ctypes import ctypes.util from itertools import count @@ -11,38 +10,59 @@ from functools import partial +def _to_os_thread_name(name: str) -> bytes: + # ctypes handles the trailing \00 + return name.encode("ascii", errors="replace")[:15] + + +# used to construct the method used to set os thread name, or None, depending on platform. +# called once on import def get_os_thread_name_func() -> Optional[Callable[[Optional[int], str], None]]: def namefunc(setname: Callable[[int, bytes], int], ident: Optional[int], name: str): - if ident is not None: - setname(ident, bytes(name[:15], "ascii", "replace")) + # Thread.ident is None "if it has not been started". Unclear if that can happen + # with current usage. + if ident is not None: # pragma: no cover + setname(ident, _to_os_thread_name(name)) + # namefunc on mac also takes an ident, even if pthread_setname_np doesn't/can't use it + # so the caller don't need to care about platform. def darwin_namefunc( setname: Callable[[bytes], int], ident: Optional[int], name: str ): - if ident is not None: - setname(bytes(name[:15], "ascii", "replace")) + # I don't know if Mac can rename threads that hasn't been started, but default + # to no to be on the safe side. + if ident is not None: # pragma: no cover + setname(_to_os_thread_name(name)) + # find the pthread library + # this will fail on windows libpthread_path = ctypes.util.find_library("pthread") if not libpthread_path: return None libpthread = ctypes.CDLL(libpthread_path) + # get the setname method from it + # afaik this should never fail pthread_setname_np = getattr(libpthread, "pthread_setname_np", None) - if pthread_setname_np is None: + if pthread_setname_np is None: # pragma: no cover return None # specify function prototype pthread_setname_np.restype = ctypes.c_int - # on mac OSX pthread_setname_np does not take a thread id + # on mac OSX pthread_setname_np does not take a thread id, + # it only lets threads name themselves, which is not a problem for us. + # Just need to make sure to call it correctly if sys.platform == "darwin": pthread_setname_np.argtypes = [ctypes.c_char_p] return partial(darwin_namefunc, pthread_setname_np) + # otherwise assume linux parameter conventions. Should also work on *BSD pthread_setname_np.argtypes = [ctypes.c_void_p, ctypes.c_char_p] return partial(namefunc, pthread_setname_np) +# construct os thread name method set_os_thread_name = get_os_thread_name_func() # The "thread cache" is a simple unbounded thread pool, i.e., it automatically diff --git a/trio/_threads.py b/trio/_threads.py index 1a7d7bebf9..04a1798b16 100644 --- a/trio/_threads.py +++ b/trio/_threads.py @@ -89,7 +89,7 @@ async def to_thread_run_sync( if pthread.h is available (i.e. most POSIX installations). pthread names are limited to 15 characters, and can be read from ``/proc//task//comm`` or with ``ps -eT``, among others. - Defaults to ``Thread for {trio.lowlevel.current_task().name}``. + Defaults to ``{sync_fn.__name__|None} from {trio.lowlevel.current_task().name}``. limiter (None, or CapacityLimiter-like object): An object used to limit the number of simultaneous threads. Most commonly this will be a `~trio.CapacityLimiter`, but it could be @@ -178,8 +178,7 @@ def do_release_then_return_result(): current_trio_token = trio.lowlevel.current_trio_token() if thread_name is None: - # TODO, better default since it caps at 15 chars - thread_name = f"Thread for {trio.lowlevel.current_task().name}" + thread_name = f"{getattr(sync_fn, '__name__', None)} from {trio.lowlevel.current_task().name}" def worker_fn(): current_async_library_cvar.set(None) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 6588d59cd4..68cf6e85ea 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -168,7 +168,9 @@ async def main(): async def test_named_thread(): - def inner(name) -> threading.Thread: + ending = " from trio.tests.test_threads.test_named_thread" + + def inner(name="inner" + ending) -> threading.Thread: assert threading.current_thread().name == name return threading.current_thread() @@ -176,9 +178,11 @@ def f(name: str) -> Callable[[None], threading.Thread]: return partial(inner, name) # test defaults - default = "Thread for trio.tests.test_threads.test_named_thread" - await to_thread_run_sync(f(default)) - await to_thread_run_sync(f(default), thread_name=None) + await to_thread_run_sync(inner) + await to_thread_run_sync(inner, thread_name=None) + + # functools.partial doesn't have __name__, so defaults to None + await to_thread_run_sync(f("None" + ending)) # test that you can set a custom name, and that it's reset afterwards async def test_thread_name(name: str): @@ -203,10 +207,13 @@ def _get_thread_name(ident: Optional[int] = None) -> Optional[str]: libpthread = ctypes.CDLL(libpthread_path) pthread_getname_np = getattr(libpthread, "pthread_getname_np", None) - if not pthread_getname_np: - print(f"no pthread_getname_np on {sys.platform})") - return None + # this should never fail afaik, but if so just switch the below lines + assert pthread_getname_np + # if not pthread_getname_np: # pragma: no cover + # print(f"no pthread_getname_np on {sys.platform})") + # return None + # thankfully getname signature doesn't differ between platforms pthread_getname_np.argtypes = [ ctypes.c_void_p, ctypes.c_char_p, @@ -220,18 +227,22 @@ def _get_thread_name(ident: Optional[int] = None) -> Optional[str]: assert pthread_getname_np(ident, name_buffer, 16) == 0 try: return name_buffer.value.decode() - except UnicodeDecodeError as e: + except UnicodeDecodeError as e: # pragma: no cover + # used for debugging when testing via CI pytest.fail(f"value: {name_buffer.value!r}, exception: {e}") # test os thread naming -# afaik pthread should be available on MacOS as well, but it's apparently -# not on the CI machines. +# this depends on pthread being available, which is the case on 99.9% of linux machines +# and most mac machines. So unless the platform is linux it will just skip +# in case it fails to fetch the os thread name. async def test_named_thread_os(): def inner(name) -> threading.Thread: - if not _get_thread_name() == name[:15]: - assert sys.platform != "linux", "os threads should work on linux" - pytest.skip(f"no pthread OS support in {sys.platform}") + os_thread_name = _get_thread_name() + if os_thread_name is None and sys.platform != "linux": + pytest.skip(f"no pthread OS support on {sys.platform}") + else: + assert os_thread_name == name[:15] return threading.current_thread() @@ -239,7 +250,7 @@ def f(name: str) -> Callable[[None], threading.Thread]: return partial(inner, name) # test defaults - default = "Thread for trio.tests.test_threads.test_named_thread" + default = "None from trio.tests.test_threads.test_named_thread" await to_thread_run_sync(f(default)) await to_thread_run_sync(f(default), thread_name=None) @@ -250,9 +261,7 @@ async def test_thread_name(name: str, expected: Optional[str] = None): thread = await to_thread_run_sync(f(expected), thread_name=name) os_thread_name = _get_thread_name(thread.ident) - if os_thread_name is None: - assert sys.platform == "linux", "os threads should work on linux" - pytest.skip(f"no pthread OS support in {sys.platform}") + assert os_thread_name is not None, "should skip earlier if this is the case" assert re.match("Trio thread [0-9]*", os_thread_name) await test_thread_name("") From 0dc1808be9ecdc4f8b5e21a83d4f8a9c7758e36b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 10:11:34 +0000 Subject: [PATCH 1253/1498] Bump cryptography from 38.0.2 to 38.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 38.0.2 to 38.0.4. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/38.0.2...38.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 25c01be092..db1ba71b0b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.4.1 # via pytest-cov -cryptography==38.0.2 +cryptography==38.0.4 # via # -r test-requirements.in # pyopenssl From 05d53b33b796d8d4c638ec161e990e8e8d10bff5 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 29 Nov 2022 10:57:46 +0100 Subject: [PATCH 1254/1498] Disable failing tests for Python 3.7 on Ubuntu --- trio/tests/test_ssl.py | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index fb119a754c..ad188a83d9 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1,6 +1,6 @@ import os -import re import sys +from typing_extensions import Final import pytest @@ -12,6 +12,7 @@ from OpenSSL import SSL import trustme +from _pytest.mark import MarkDecorator import trio from .. import _core @@ -35,7 +36,7 @@ # We have two different kinds of echo server fixtures we use for testing. The # first is a real server written using the stdlib ssl module and blocking -# sockets. It runs in a thread and we talk to it over a real socketpair(), to +# sockets. It runs in a thread, and we talk to it over a real socketpair(), to # validate interoperability in a semi-realistic setting. # # The second is a very weird virtual echo server that lives inside a custom @@ -45,21 +46,27 @@ # the server-side TLS state engine to decrypt, then takes that data, feeds it # back through to get the encrypted response, and returns it from 'receive_some'. This # gives us full control and reproducibility. This server is written using -# PyOpenSSL, so that we can trigger renegotiations on demand. It also allows +# PyOpenSSL, so that we can trigger re-negotiations on demand. It also allows # us to insert random (virtual) delays, to really exercise all the weird paths # in SSLStream's state engine. # # Both present a certificate for "trio-test-1.example.org". -TRIO_TEST_CA = trustme.CA() -TRIO_TEST_1_CERT = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") +TRIO_TEST_CA: Final = trustme.CA() +TRIO_TEST_1_CERT: Final = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") -SERVER_CTX = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) +SERVER_CTX: Final = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): SERVER_CTX.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) +skip_on_python_37_linux: MarkDecorator = pytest.mark.skipif( + sys.platform == "linux", + sys.version_info[0:2] == (3, 7), + reason="Certain SSL Tests are not passing on Ubuntu while running Python 3.7", +) + # TLS 1.3 has a lot of changes from previous versions. So we want to run tests # with both TLS 1.3, and TLS 1.2. @@ -68,7 +75,7 @@ # downgrade on the server side. "tls12" means we refuse to negotiate TLS # 1.3, so we'll almost certainly use TLS 1.2. @pytest.fixture(scope="module", params=["tls13", "tls12"]) -def client_ctx(request): +def client_ctx(request) -> ssl.SSLContext: ctx = ssl.create_default_context() if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): @@ -85,7 +92,11 @@ def client_ctx(request): # The blocking socket server. -def ssl_echo_serve_sync(sock, *, expect_fail=False): +def ssl_echo_serve_sync( + sock: stdlib_socket.socket, + *, + expect_fail: bool = False, +) -> None: try: wrapped = SERVER_CTX.wrap_socket( sock, server_side=True, suppress_ragged_eofs=False @@ -96,7 +107,7 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): data = wrapped.recv(4096) if not data: # other side has initiated a graceful shutdown; we try to - # respond in kind but it's legal for them to have already + # respond in kind, but it's legal for them to have already # gone away. exceptions = (BrokenPipeError, ssl.SSLZeroReturnError) try: @@ -107,7 +118,7 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): # Under unclear conditions, CPython sometimes raises # SSLWantWriteError here. This is a bug (bpo-32219), # but it's not our bug. Christian Heimes thinks - # it's fixed in 'recent' CPython versions so we fail + # it's fixed in 'recent' CPython versions, so we fail # the test for those and ignore it for earlier # versions. if ( @@ -128,7 +139,7 @@ def ssl_echo_serve_sync(sock, *, expect_fail=False): # the OS to report a ECONNREST or even ECONNABORTED (which is just wrong, # since ECONNABORTED is supposed to mean that connect() failed, but what # can you do). In this case the other side did nothing wrong, but there's - # no way to recover, so we let it pass, and just cross our fingers its not + # no way to recover, so we let it pass, and just cross our fingers it's not # hiding any (other) real bugs. For more details see: # # https://github.com/python-trio/trio/issues/1293 @@ -186,7 +197,7 @@ def __init__(self, sleeper=None): # we still have to support versions before that, and that means we # need to test renegotiation support, which means we need to force this # to use a lower version where this test server can trigger - # renegotiations. Of course TLS 1.3 support isn't released yet, but + # re-negotiations. Of course TLS 1.3 support isn't released yet, but # I'm told that this will work once it is. (And once it is we can # remove the pragma: no cover too.) Alternatively, we could switch to # using TLSv1_2_METHOD. @@ -519,7 +530,7 @@ async def test_attributes(client_ctx): # also seen cases where our send_all blocks waiting to write, and then our receive_some # also blocks waiting to write, and they never wake up again. It looks like # some kind of deadlock. I suspect there may be an issue where we've filled up -# the send buffers, and the remote side is trying to handle the renegotiation +# the send-buffers, and the remote side is trying to handle the renegotiation # from inside a write() call, so it has a problem: there's all this application # data clogging up the pipe, but it can't process and return it to the # application because it's in write(), and it doesn't want to buffer infinite @@ -809,6 +820,7 @@ async def test_send_all_empty_string(client_ctx): await s.aclose() +@skip_on_python_37_linux @pytest.mark.parametrize("https_compatible", [False, True]) async def test_SSLStream_generic(client_ctx, https_compatible): async def stream_maker(): @@ -1024,6 +1036,7 @@ async def test_ssl_bad_shutdown(client_ctx): await server.aclose() +@skip_on_python_37_linux async def test_ssl_bad_shutdown_but_its_ok(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, @@ -1088,6 +1101,7 @@ def close_hook(): assert transport_close_count == 1 +@skip_on_python_37_linux async def test_ssl_https_compatibility_disagreement(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, @@ -1112,6 +1126,7 @@ async def receive_and_expect_error(): nursery.start_soon(receive_and_expect_error) +@skip_on_python_37_linux async def test_https_mode_eof_before_handshake(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, From 9078da3186a69a2c9e5afb7cb4f916ac989a91e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 10:12:13 +0000 Subject: [PATCH 1255/1498] Bump urllib3 from 1.26.12 to 1.26.13 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.12 to 1.26.13. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.12...1.26.13) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index b7a43a28bb..cb335ef4b8 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -90,7 +90,7 @@ tomli==2.0.1 # via towncrier towncrier==22.8.0 # via -r docs-requirements.in -urllib3==1.26.12 +urllib3==1.26.13 # via requests # The following packages are considered to be unsafe in a requirements file: From d0999e262622bc9f429a7fc8026b59eda65bb118 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 29 Nov 2022 12:38:32 +0100 Subject: [PATCH 1256/1498] Revert changes not needed for PR --- trio/tests/test_ssl.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index ad188a83d9..0fa8d92bbc 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1,6 +1,7 @@ import os +import re import sys -from typing_extensions import Final +from typing import TYPE_CHECKING import pytest @@ -12,7 +13,6 @@ from OpenSSL import SSL import trustme -from _pytest.mark import MarkDecorator import trio from .. import _core @@ -34,9 +34,13 @@ check_two_way_stream, ) +if TYPE_CHECKING: + from _pytest.mark import MarkDecorator + + # We have two different kinds of echo server fixtures we use for testing. The # first is a real server written using the stdlib ssl module and blocking -# sockets. It runs in a thread, and we talk to it over a real socketpair(), to +# sockets. It runs in a thread and we talk to it over a real socketpair(), to # validate interoperability in a semi-realistic setting. # # The second is a very weird virtual echo server that lives inside a custom @@ -46,16 +50,16 @@ # the server-side TLS state engine to decrypt, then takes that data, feeds it # back through to get the encrypted response, and returns it from 'receive_some'. This # gives us full control and reproducibility. This server is written using -# PyOpenSSL, so that we can trigger re-negotiations on demand. It also allows +# PyOpenSSL, so that we can trigger renegotiations on demand. It also allows # us to insert random (virtual) delays, to really exercise all the weird paths # in SSLStream's state engine. # # Both present a certificate for "trio-test-1.example.org". -TRIO_TEST_CA: Final = trustme.CA() -TRIO_TEST_1_CERT: Final = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") +TRIO_TEST_CA = trustme.CA() +TRIO_TEST_1_CERT = TRIO_TEST_CA.issue_server_cert("trio-test-1.example.org") -SERVER_CTX: Final = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) +SERVER_CTX = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): SERVER_CTX.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF @@ -75,7 +79,7 @@ # downgrade on the server side. "tls12" means we refuse to negotiate TLS # 1.3, so we'll almost certainly use TLS 1.2. @pytest.fixture(scope="module", params=["tls13", "tls12"]) -def client_ctx(request) -> ssl.SSLContext: +def client_ctx(request): ctx = ssl.create_default_context() if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): @@ -92,11 +96,7 @@ def client_ctx(request) -> ssl.SSLContext: # The blocking socket server. -def ssl_echo_serve_sync( - sock: stdlib_socket.socket, - *, - expect_fail: bool = False, -) -> None: +def ssl_echo_serve_sync(sock, *, expect_fail=False): try: wrapped = SERVER_CTX.wrap_socket( sock, server_side=True, suppress_ragged_eofs=False @@ -107,7 +107,7 @@ def ssl_echo_serve_sync( data = wrapped.recv(4096) if not data: # other side has initiated a graceful shutdown; we try to - # respond in kind, but it's legal for them to have already + # respond in kind but it's legal for them to have already # gone away. exceptions = (BrokenPipeError, ssl.SSLZeroReturnError) try: @@ -118,7 +118,7 @@ def ssl_echo_serve_sync( # Under unclear conditions, CPython sometimes raises # SSLWantWriteError here. This is a bug (bpo-32219), # but it's not our bug. Christian Heimes thinks - # it's fixed in 'recent' CPython versions, so we fail + # it's fixed in 'recent' CPython versions so we fail # the test for those and ignore it for earlier # versions. if ( @@ -139,7 +139,7 @@ def ssl_echo_serve_sync( # the OS to report a ECONNREST or even ECONNABORTED (which is just wrong, # since ECONNABORTED is supposed to mean that connect() failed, but what # can you do). In this case the other side did nothing wrong, but there's - # no way to recover, so we let it pass, and just cross our fingers it's not + # no way to recover, so we let it pass, and just cross our fingers its not # hiding any (other) real bugs. For more details see: # # https://github.com/python-trio/trio/issues/1293 @@ -197,7 +197,7 @@ def __init__(self, sleeper=None): # we still have to support versions before that, and that means we # need to test renegotiation support, which means we need to force this # to use a lower version where this test server can trigger - # re-negotiations. Of course TLS 1.3 support isn't released yet, but + # renegotiations. Of course TLS 1.3 support isn't released yet, but # I'm told that this will work once it is. (And once it is we can # remove the pragma: no cover too.) Alternatively, we could switch to # using TLSv1_2_METHOD. @@ -530,7 +530,7 @@ async def test_attributes(client_ctx): # also seen cases where our send_all blocks waiting to write, and then our receive_some # also blocks waiting to write, and they never wake up again. It looks like # some kind of deadlock. I suspect there may be an issue where we've filled up -# the send-buffers, and the remote side is trying to handle the renegotiation +# the send buffers, and the remote side is trying to handle the renegotiation # from inside a write() call, so it has a problem: there's all this application # data clogging up the pipe, but it can't process and return it to the # application because it's in write(), and it doesn't want to buffer infinite From cfe9abf4067d7d635fd5188b1063b6ab1038e7cf Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 29 Nov 2022 12:42:09 +0100 Subject: [PATCH 1257/1498] Defer annotation evaluation --- trio/tests/test_ssl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 0fa8d92bbc..8bc83e171e 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re import sys From 0e47d3c27b29f4b73b960fb82be76441962e4996 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 29 Nov 2022 16:11:15 +0100 Subject: [PATCH 1258/1498] Update trio/tests/test_ssl.py Co-authored-by: Quentin Pradet --- trio/tests/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 8bc83e171e..d561565d53 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -69,7 +69,7 @@ skip_on_python_37_linux: MarkDecorator = pytest.mark.skipif( sys.platform == "linux", - sys.version_info[0:2] == (3, 7), + sys.version_info < (3, 8), reason="Certain SSL Tests are not passing on Ubuntu while running Python 3.7", ) From da945d84b1a1b4f98aa41e1094fde09101a10175 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 29 Nov 2022 16:18:13 +0100 Subject: [PATCH 1259/1498] Add reference to github comment --- trio/tests/test_ssl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index d561565d53..12f00071dd 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -67,6 +67,8 @@ TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) + +# See: https://github.com/python-trio/trio/pull/2480#issuecomment-1330135171 skip_on_python_37_linux: MarkDecorator = pytest.mark.skipif( sys.platform == "linux", sys.version_info < (3, 8), From 4289e7eb8d9f0f9999d508b8c642ba642ca1b4bd Mon Sep 17 00:00:00 2001 From: jakkdl Date: Wed, 30 Nov 2022 11:34:13 +0100 Subject: [PATCH 1260/1498] update comment --- trio/tests/test_threads.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/trio/tests/test_threads.py b/trio/tests/test_threads.py index 68cf6e85ea..77b7a0831a 100644 --- a/trio/tests/test_threads.py +++ b/trio/tests/test_threads.py @@ -207,11 +207,9 @@ def _get_thread_name(ident: Optional[int] = None) -> Optional[str]: libpthread = ctypes.CDLL(libpthread_path) pthread_getname_np = getattr(libpthread, "pthread_getname_np", None) - # this should never fail afaik, but if so just switch the below lines + + # this should never fail on any platforms afaik assert pthread_getname_np - # if not pthread_getname_np: # pragma: no cover - # print(f"no pthread_getname_np on {sys.platform})") - # return None # thankfully getname signature doesn't differ between platforms pthread_getname_np.argtypes = [ From 74832d0d9b042d460c5aae669ddc68d6658d299e Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 30 Nov 2022 13:11:27 +0100 Subject: [PATCH 1261/1498] Improve skip decorator Co-authored-by: Quentin Pradet --- trio/tests/test_ssl.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 12f00071dd..6a652b144d 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -68,11 +68,9 @@ TRIO_TEST_1_CERT.configure_cert(SERVER_CTX) -# See: https://github.com/python-trio/trio/pull/2480#issuecomment-1330135171 -skip_on_python_37_linux: MarkDecorator = pytest.mark.skipif( - sys.platform == "linux", - sys.version_info < (3, 8), - reason="Certain SSL Tests are not passing on Ubuntu while running Python 3.7", +skip_on_broken_openssl: MarkDecorator = pytest.mark.skipif( + sys.version_info < (3, 8) and ssl.OPENSSL_VERSION_INFO[0] > 1, + reason="Python 3.7 does not work with OpenSSL versions higher than 1.X", ) From c80440fedbaaa2c48624471ef4b1afa4f9d98ba6 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 30 Nov 2022 13:12:01 +0100 Subject: [PATCH 1262/1498] Update trio/tests/test_ssl.py Co-authored-by: Quentin Pradet --- trio/tests/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 6a652b144d..c849a14c0a 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -822,7 +822,7 @@ async def test_send_all_empty_string(client_ctx): await s.aclose() -@skip_on_python_37_linux +@skip_on_broken_openssl @pytest.mark.parametrize("https_compatible", [False, True]) async def test_SSLStream_generic(client_ctx, https_compatible): async def stream_maker(): From 9885dc8d8da98e4b6b2616fc9d86e1a936396ef7 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 30 Nov 2022 13:12:10 +0100 Subject: [PATCH 1263/1498] Update trio/tests/test_ssl.py Co-authored-by: Quentin Pradet --- trio/tests/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index c849a14c0a..fe8697a5e6 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1038,7 +1038,7 @@ async def test_ssl_bad_shutdown(client_ctx): await server.aclose() -@skip_on_python_37_linux +@skip_on_broken_openssl async def test_ssl_bad_shutdown_but_its_ok(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, From e1726742b16aa20481fea4c292cd100104ec1b6f Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 30 Nov 2022 13:12:18 +0100 Subject: [PATCH 1264/1498] Update trio/tests/test_ssl.py Co-authored-by: Quentin Pradet --- trio/tests/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index fe8697a5e6..0c2bf12282 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1103,7 +1103,7 @@ def close_hook(): assert transport_close_count == 1 -@skip_on_python_37_linux +@skip_on_broken_openssl async def test_ssl_https_compatibility_disagreement(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, From 2802ed80980b94cfbc207a89e19527a221dcc270 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 30 Nov 2022 13:12:26 +0100 Subject: [PATCH 1265/1498] Update trio/tests/test_ssl.py Co-authored-by: Quentin Pradet --- trio/tests/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/tests/test_ssl.py b/trio/tests/test_ssl.py index 0c2bf12282..26e107e08f 100644 --- a/trio/tests/test_ssl.py +++ b/trio/tests/test_ssl.py @@ -1128,7 +1128,7 @@ async def receive_and_expect_error(): nursery.start_soon(receive_and_expect_error) -@skip_on_python_37_linux +@skip_on_broken_openssl async def test_https_mode_eof_before_handshake(client_ctx): client, server = ssl_memory_stream_pair( client_ctx, From 54c579ea65c8e5741dda4429968b2111f1a47bd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 10:16:07 +0000 Subject: [PATCH 1266/1498] Bump jedi from 0.18.1 to 0.18.2 Bumps [jedi](https://github.com/davidhalter/jedi) from 0.18.1 to 0.18.2. - [Release notes](https://github.com/davidhalter/jedi/releases) - [Changelog](https://github.com/davidhalter/jedi/blob/master/CHANGELOG.rst) - [Commits](https://github.com/davidhalter/jedi/compare/v0.18.1...v0.18.2) --- updated-dependencies: - dependency-name: jedi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6d1d7a5a06..b420eb9135 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -46,7 +46,7 @@ ipython==7.31.1 # via -r test-requirements.in isort==5.10.1 # via pylint -jedi==0.18.1 +jedi==0.18.2 # via # -r test-requirements.in # ipython From 720f8066afa5d3f0217ff44d3d823b8faaaa2551 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 10:14:36 +0000 Subject: [PATCH 1267/1498] Bump certifi from 2022.9.24 to 2022.12.7 Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.9.24 to 2022.12.7. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.09.24...2022.12.07) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index cb335ef4b8..64c59caba2 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -14,7 +14,7 @@ attrs==22.1.0 # outcome babel==2.11.0 # via sphinx -certifi==2022.9.24 +certifi==2022.12.7 # via requests charset-normalizer==2.1.1 # via requests From c79ee0fc1ca4b0db808f576713641e251c15be3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 10:16:05 +0000 Subject: [PATCH 1268/1498] Bump platformdirs from 2.5.4 to 2.6.0 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.5.4 to 2.6.0. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.5.4...2.6.0) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2e12b2c8a6..9a0eb421e9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -77,7 +77,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.4 +platformdirs==2.6.0 # via # black # pylint From 6fb73eaee8ac38884c00fa8b582d46f9129f631f Mon Sep 17 00:00:00 2001 From: Faster Speeding Date: Sun, 27 Nov 2022 12:29:27 +0000 Subject: [PATCH 1269/1498] Bump the minimum attrs ver for @frozen --- setup.py | 4 +++- test-requirements.in | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b2e53826e6..a8e1154dc6 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,9 @@ license="MIT OR Apache-2.0", packages=find_packages(), install_requires=[ - "attrs >= 19.2.0", # for eq + # attrs 19.2.0 adds `eq` option to decorators + # attrs 20.1.0 adds @frozen + "attrs >= 20.1.0", "sortedcontainers", "idna", "outcome", diff --git a/test-requirements.in b/test-requirements.in index 6e94409005..e8c73aed3f 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -25,7 +25,7 @@ typing-extensions; implementation_name == "cpython" # Trio's own dependencies cffi; os_name == "nt" -attrs >= 19.2.0 +attrs >= 20.1.0 sortedcontainers idna outcome From c94d5e1db1d71c2c656b050bde6378b2c2753d16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 10:21:32 +0000 Subject: [PATCH 1270/1498] Bump black from 22.10.0 to 22.12.0 Bumps [black](https://github.com/psf/black) from 22.10.0 to 22.12.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.10.0...22.12.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e727d88e18..687311ad66 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==22.1.0 # pytest backcall==0.2.0 # via ipython -black==22.10.0 ; implementation_name == "cpython" +black==22.12.0 ; implementation_name == "cpython" # via -r test-requirements.in cffi==1.15.1 # via cryptography From f207b2b0ef7a0fef582e9f02aed5788b0cc26c37 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 13 Dec 2022 09:01:54 +0100 Subject: [PATCH 1271/1498] Add types to _unix_pipes --- trio/_unix_pipes.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index 4b3637dc32..50c6ad2aac 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import os import errno +from typing_extensions import Final as FinalType from ._abc import Stream from ._util import ConflictDetector, Final @@ -13,7 +16,7 @@ # XX TODO: is this a good number? who knows... it does match the default Linux # pipe capacity though. -DEFAULT_RECEIVE_SIZE = 65536 +DEFAULT_RECEIVE_SIZE: FinalType = 65536 class _FdHolder: @@ -34,7 +37,9 @@ class _FdHolder: # impossible to make this mistake – we'll just get an EBADF. # # (This trick was copied from the stdlib socket module.) - def __init__(self, fd: int): + fd: int + + def __init__(self, fd: int) -> None: # make sure self.fd is always initialized to *something*, because even # if we error out here then __del__ will run and access it. self.fd = -1 @@ -46,10 +51,10 @@ def __init__(self, fd: int): os.set_blocking(fd, False) @property - def closed(self): + def closed(self) -> bool: return self.fd == -1 - def _raw_close(self): + def _raw_close(self) -> None: # This doesn't assume it's in a Trio context, so it can be called from # __del__. You should never call it from Trio context, because it # skips calling notify_fd_close. But from __del__, skipping that is @@ -64,10 +69,10 @@ def _raw_close(self): os.set_blocking(fd, self._original_is_blocking) os.close(fd) - def __del__(self): + def __del__(self) -> None: self._raw_close() - def close(self): + def close(self) -> None: if not self.closed: trio.lowlevel.notify_closing(self.fd) self._raw_close() @@ -93,7 +98,7 @@ class FdStream(Stream, metaclass=Final): thrust upon them. For example, you can use ``FdStream(os.dup(sys.stdin.fileno()))`` to obtain a stream for reading from standard input, but it is only safe to do so with heavy caveats: your - stdin must not be shared by any other processes and you must not make any + stdin must not be shared by any other processes, and you must not make any calls to synchronous methods of `sys.stdin` until the stream returned by `FdStream` is closed. See `issue #174 `__ for a discussion of the @@ -106,7 +111,7 @@ class FdStream(Stream, metaclass=Final): A new `FdStream` object. """ - def __init__(self, fd: int): + def __init__(self, fd: int) -> None: self._fd_holder = _FdHolder(fd) self._send_conflict_detector = ConflictDetector( "another task is using this stream for send" @@ -115,7 +120,7 @@ def __init__(self, fd: int): "another task is using this stream for receive" ) - async def send_all(self, data: bytes): + async def send_all(self, data: bytes) -> None: with self._send_conflict_detector: # have to check up front, because send_all(b"") on a closed pipe # should raise @@ -151,7 +156,7 @@ async def wait_send_all_might_not_block(self) -> None: # of sending, which is annoying raise trio.BrokenResourceError from e - async def receive_some(self, max_bytes=None) -> bytes: + async def receive_some(self, max_bytes: int | None = None) -> bytes: with self._receive_conflict_detector: if max_bytes is None: max_bytes = DEFAULT_RECEIVE_SIZE @@ -179,12 +184,12 @@ async def receive_some(self, max_bytes=None) -> bytes: return data - def close(self): + def close(self) -> None: self._fd_holder.close() - async def aclose(self): + async def aclose(self) -> None: self.close() await trio.lowlevel.checkpoint() - def fileno(self): + def fileno(self) -> int: return self._fd_holder.fd From 0da00689b7e0841fdb5de7f139c2cabeee43987e Mon Sep 17 00:00:00 2001 From: Ganden Schaffner Date: Mon, 19 Dec 2022 00:00:00 -0800 Subject: [PATCH 1272/1498] Fix crash due to runtime import of typing_extensions typing_extensions doesn't presently need to be a runtime dependency. --- trio/_unix_pipes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py index 50c6ad2aac..fa98e79521 100644 --- a/trio/_unix_pipes.py +++ b/trio/_unix_pipes.py @@ -2,13 +2,16 @@ import os import errno -from typing_extensions import Final as FinalType +from typing import TYPE_CHECKING from ._abc import Stream from ._util import ConflictDetector, Final import trio +if TYPE_CHECKING: + from typing_extensions import Final as FinalType + if os.name != "posix": # We raise an error here rather than gating the import in lowlevel.py # in order to keep jedi static analysis happy. From 5031655f4b923bad455d5834b198d951baa1b06c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 25 Dec 2022 18:08:37 +0000 Subject: [PATCH 1273/1498] Bump attrs from 22.1.0 to 22.2.0 Bumps [attrs](https://github.com/python-attrs/attrs) from 22.1.0 to 22.2.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.md) - [Commits](https://github.com/python-attrs/attrs/compare/22.1.0...22.2.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 64c59caba2..776211f358 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx async-generator==1.10 # via -r docs-requirements.in -attrs==22.1.0 +attrs==22.2.0 # via # -r docs-requirements.in # outcome diff --git a/test-requirements.txt b/test-requirements.txt index 687311ad66..c13817da00 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,7 @@ astroid==2.11.7 # via pylint async-generator==1.10 # via -r test-requirements.in -attrs==22.1.0 +attrs==22.2.0 # via # -r test-requirements.in # outcome From 7856f387069c45095225ac2becdeac6507fcfae5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 00:32:02 +0000 Subject: [PATCH 1274/1498] Bump ipython from 7.31.1 to 7.34.0 Bumps [ipython](https://github.com/ipython/ipython) from 7.31.1 to 7.34.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/7.31.1...7.34.0) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.in | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.in b/test-requirements.in index e8c73aed3f..85ca664925 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -3,7 +3,7 @@ pytest >= 5.0 # for faulthandler in core pytest-cov >= 2.6.0 async_generator >= 1.9 # ipython 7.x is the last major version supporting Python 3.7 -ipython < 7.32 # for the IPython traceback integration tests +ipython < 7.35 # for the IPython traceback integration tests pyOpenSSL >= 22.0.0 # for the ssl + DTLS tests trustme # for the ssl + DTLS tests pylint # for pylint finding all symbols tests diff --git a/test-requirements.txt b/test-requirements.txt index 687311ad66..9cf5d92968 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -42,7 +42,7 @@ idna==3.4 # trustme iniconfig==1.1.1 # via pytest -ipython==7.31.1 +ipython==7.34.0 # via -r test-requirements.in isort==5.10.1 # via pylint From 41c0d5f177582ba8b57bdbaed8673160413671b9 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Thu, 29 Dec 2022 00:47:25 +0000 Subject: [PATCH 1275/1498] Fix IPython tests --- trio/_core/tests/test_multierror.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/trio/_core/tests/test_multierror.py b/trio/_core/tests/test_multierror.py index a15ba3f2c9..650f9bf597 100644 --- a/trio/_core/tests/test_multierror.py +++ b/trio/_core/tests/test_multierror.py @@ -473,10 +473,12 @@ def run_script(name, use_ipython=False): return completed -def check_simple_excepthook(completed): +def check_simple_excepthook(completed, uses_ipython): assert_match_in_seq( [ - "in ", + "in = (3, 8) + else "in ", "MultiError", "--- 1 ---", "in exc1_fn", @@ -503,14 +505,14 @@ def check_simple_excepthook(completed): @need_ipython def test_ipython_exc_handler(): completed = run_script("simple_excepthook.py", use_ipython=True) - check_simple_excepthook(completed) + check_simple_excepthook(completed, True) @slow @need_ipython def test_ipython_imported_but_unused(): completed = run_script("simple_excepthook_IPython.py") - check_simple_excepthook(completed) + check_simple_excepthook(completed, False) @slow From cd1ec41d6e7ebb3c5eef8244b1af4dd8b7417642 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 01:12:44 +0000 Subject: [PATCH 1276/1498] Bump traitlets from 5.5.0 to 5.8.0 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.5.0 to 5.8.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Changelog](https://github.com/ipython/traitlets/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipython/traitlets/compare/5.5.0...v5.8.0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9cf5d92968..5aa7fc3864 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -113,7 +113,7 @@ sortedcontainers==2.4.0 # via -r test-requirements.in tomlkit==0.11.6 # via pylint -traitlets==5.5.0 +traitlets==5.8.0 # via # ipython # matplotlib-inline From 889143a6b70b45e1ee947bec0b7fcaaee2c12167 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 01:29:49 +0000 Subject: [PATCH 1277/1498] Bump pathspec from 0.10.2 to 0.10.3 Bumps [pathspec](https://github.com/cpburnz/python-pathspec) from 0.10.2 to 0.10.3. - [Release notes](https://github.com/cpburnz/python-pathspec/releases) - [Changelog](https://github.com/cpburnz/python-pathspec/blob/master/CHANGES.rst) - [Commits](https://github.com/cpburnz/python-pathspec/compare/v0.10.2...v0.10.3) --- updated-dependencies: - dependency-name: pathspec dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9cf5d92968..b0a4840fa4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -71,7 +71,7 @@ packaging==21.3 # via pytest parso==0.8.3 # via jedi -pathspec==0.10.2 +pathspec==0.10.3 # via black pexpect==4.8.0 # via ipython From 22461c9a2d5e37b252a91dea406a5cf431329ea9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 01:39:44 +0000 Subject: [PATCH 1278/1498] Bump packaging from 21.3 to 22.0 Bumps [packaging](https://github.com/pypa/packaging) from 21.3 to 22.0. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/21.3...22.0) --- updated-dependencies: - dependency-name: packaging dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 +--- test-requirements.txt | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 64c59caba2..c93501de21 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -49,12 +49,10 @@ markupsafe==2.1.1 # via jinja2 outcome==1.2.0 # via -r docs-requirements.in -packaging==21.3 +packaging==22.0 # via sphinx pygments==2.12.0 # via sphinx -pyparsing==3.0.9 - # via packaging pytz==2022.4 # via babel requests==2.28.1 diff --git a/test-requirements.txt b/test-requirements.txt index b0a4840fa4..d3489834a1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # mypy outcome==1.2.0 # via -r test-requirements.in -packaging==21.3 +packaging==22.0 # via pytest parso==0.8.3 # via jedi @@ -99,8 +99,6 @@ pylint==2.14.5 # via -r test-requirements.in pyopenssl==22.1.0 # via -r test-requirements.in -pyparsing==3.0.9 - # via packaging pytest==7.2.0 # via # -r test-requirements.in From 950cae60a913a29ad04bd4d64cf2ff47d3e958a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 02:29:46 +0000 Subject: [PATCH 1279/1498] Bump pytz from 2022.4 to 2022.7 Bumps [pytz](https://github.com/stub42/pytz) from 2022.4 to 2022.7. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2022.4...release_2022.7) --- updated-dependencies: - dependency-name: pytz dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index c93501de21..98b8cb6c44 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -53,7 +53,7 @@ packaging==22.0 # via sphinx pygments==2.12.0 # via sphinx -pytz==2022.4 +pytz==2022.7 # via babel requests==2.28.1 # via sphinx From 3d2635c5f4b4dcf45b094ec56163ac042307d439 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Dec 2022 10:12:25 +0000 Subject: [PATCH 1280/1498] Bump exceptiongroup from 1.0.4 to 1.1.0 Bumps [exceptiongroup](https://github.com/agronholm/exceptiongroup) from 1.0.4 to 1.1.0. - [Release notes](https://github.com/agronholm/exceptiongroup/releases) - [Changelog](https://github.com/agronholm/exceptiongroup/blob/main/CHANGES.rst) - [Commits](https://github.com/agronholm/exceptiongroup/compare/1.0.4...1.1.0) --- updated-dependencies: - dependency-name: exceptiongroup dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 98b8cb6c44..72ee05afad 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -28,7 +28,7 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -exceptiongroup==1.0.4 +exceptiongroup==1.1.0 # via -r docs-requirements.in idna==3.4 # via From b720eda670bbc516a20edcc8d23e3cf77b265e6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 10:10:49 +0000 Subject: [PATCH 1281/1498] Bump pygments from 2.12.0 to 2.14.0 Bumps [pygments](https://github.com/pygments/pygments) from 2.12.0 to 2.14.0. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.12.0...2.14.0) --- updated-dependencies: - dependency-name: pygments dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 72ee05afad..14ce4e7c6e 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -51,7 +51,7 @@ outcome==1.2.0 # via -r docs-requirements.in packaging==22.0 # via sphinx -pygments==2.12.0 +pygments==2.14.0 # via sphinx pytz==2022.7 # via babel diff --git a/test-requirements.txt b/test-requirements.txt index 0efc7172a3..467832b9fa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -93,7 +93,7 @@ pycparser==2.21 # via cffi pyflakes==2.4.0 # via flake8 -pygments==2.12.0 +pygments==2.14.0 # via ipython pylint==2.14.5 # via -r test-requirements.in From 2226e7f529d3ec2d065851ed5a34383d7517ce2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 10:18:42 +0000 Subject: [PATCH 1282/1498] Bump pyopenssl from 22.1.0 to 23.0.0 Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 22.1.0 to 23.0.0. - [Release notes](https://github.com/pyca/pyopenssl/releases) - [Changelog](https://github.com/pyca/pyopenssl/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/pyopenssl/compare/22.1.0...23.0.0) --- updated-dependencies: - dependency-name: pyopenssl dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 0efc7172a3..aecd542a3c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -97,7 +97,7 @@ pygments==2.12.0 # via ipython pylint==2.14.5 # via -r test-requirements.in -pyopenssl==22.1.0 +pyopenssl==23.0.0 # via -r test-requirements.in pytest==7.2.0 # via From 87631b7f239c209a573d9aec64901378f50b5c0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 10:12:40 +0000 Subject: [PATCH 1283/1498] Bump types-pyopenssl from 22.1.0.2 to 23.0.0.0 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 22.1.0.2 to 23.0.0.0. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d77dca5473..74f7c91bc6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -119,7 +119,7 @@ trustme==0.9.0 # via -r test-requirements.in types-cryptography==3.3.23.2 # via types-pyopenssl -types-pyopenssl==22.1.0.2 ; implementation_name == "cpython" +types-pyopenssl==23.0.0.0 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.4.0 ; implementation_name == "cpython" # via From fa7b3c99a9357a0b36f8f48864acba5cfdb20cd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jan 2023 10:09:24 +0000 Subject: [PATCH 1284/1498] Bump lazy-object-proxy from 1.8.0 to 1.9.0 Bumps [lazy-object-proxy](https://github.com/ionelmc/python-lazy-object-proxy) from 1.8.0 to 1.9.0. - [Release notes](https://github.com/ionelmc/python-lazy-object-proxy/releases) - [Changelog](https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-lazy-object-proxy/compare/v1.8.0...v1.9.0) --- updated-dependencies: - dependency-name: lazy-object-proxy dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d77dca5473..6e809f72fb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -50,7 +50,7 @@ jedi==0.18.2 # via # -r test-requirements.in # ipython -lazy-object-proxy==1.8.0 +lazy-object-proxy==1.9.0 # via astroid matplotlib-inline==0.1.6 # via ipython From a9e56bb3a0dc7c4304857fa69f0c6f183e7ddae5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 01:57:01 +0000 Subject: [PATCH 1285/1498] Bump cryptography from 38.0.4 to 39.0.0 Bumps [cryptography](https://github.com/pyca/cryptography) from 38.0.4 to 39.0.0. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/38.0.4...39.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e5a32a18d7..6d82e49f05 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.4.1 # via pytest-cov -cryptography==38.0.4 +cryptography==39.0.0 # via # -r test-requirements.in # pyopenssl From 77a87c4f1b7f8e83dfafa6b0cbb88c1daceccaf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:16:03 +0000 Subject: [PATCH 1286/1498] Bump iniconfig from 1.1.1 to 2.0.0 Bumps [iniconfig](https://github.com/pytest-dev/iniconfig) from 1.1.1 to 2.0.0. - [Release notes](https://github.com/pytest-dev/iniconfig/releases) - [Changelog](https://github.com/pytest-dev/iniconfig/blob/main/CHANGELOG) - [Commits](https://github.com/pytest-dev/iniconfig/compare/v1.1.1...v2.0.0) --- updated-dependencies: - dependency-name: iniconfig dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index e5a32a18d7..efe681e7db 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -40,7 +40,7 @@ idna==3.4 # via # -r test-requirements.in # trustme -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest ipython==7.34.0 # via -r test-requirements.in From bf22fbed547401cebb67d195515e7974d7cc78d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:25:33 +0000 Subject: [PATCH 1287/1498] Bump packaging from 22.0 to 23.0 Bumps [packaging](https://github.com/pypa/packaging) from 22.0 to 23.0. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/22.0...23.0) --- updated-dependencies: - dependency-name: packaging dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 14ce4e7c6e..b104045baf 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -49,7 +49,7 @@ markupsafe==2.1.1 # via jinja2 outcome==1.2.0 # via -r docs-requirements.in -packaging==22.0 +packaging==23.0 # via sphinx pygments==2.14.0 # via sphinx diff --git a/test-requirements.txt b/test-requirements.txt index e5a32a18d7..18befa1c7a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython" # mypy outcome==1.2.0 # via -r test-requirements.in -packaging==22.0 +packaging==23.0 # via pytest parso==0.8.3 # via jedi From 47b51d75cbc490a25b151d8998f8cf8c7e4b6827 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Mon, 9 Jan 2023 13:01:04 +0000 Subject: [PATCH 1288/1498] Try to ignore cryptography warning --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 944440661f..4b939510ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,4 +56,8 @@ xfail_strict = true faulthandler_timeout = 60 markers = ["redistributors_should_skip: tests that should be skipped by downstream redistributors"] junit_family = "xunit2" -filterwarnings = ["error"] +filterwarnings = [ + "error", + # https://gitter.im/python-trio/general?at=63bb8d0740557a3d5c688d67 + 'ignore:You are using cryptography on a 32-bit Python on a 64-bit Windows Operating System. Cryptography will be significantly faster if you switch to using a 64-bit Python.:UserWarning', +] From c6549919aff70ace5dc58ffabdf57c3e1d123772 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jan 2023 10:14:42 +0000 Subject: [PATCH 1289/1498] Bump types-pyopenssl from 23.0.0.0 to 23.0.0.1 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 23.0.0.0 to 23.0.0.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index c8ce8033cd..3c8aca1734 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -30,6 +30,7 @@ cryptography==39.0.0 # -r test-requirements.in # pyopenssl # trustme + # types-pyopenssl decorator==5.1.1 # via ipython dill==0.3.6 @@ -117,9 +118,7 @@ traitlets==5.8.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-cryptography==3.3.23.2 - # via types-pyopenssl -types-pyopenssl==23.0.0.0 ; implementation_name == "cpython" +types-pyopenssl==23.0.0.1 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.4.0 ; implementation_name == "cpython" # via From 078e1d83d5bc6ab5350334a5d8c53307a5baddeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:04:48 +0000 Subject: [PATCH 1290/1498] Bump traitlets from 5.8.0 to 5.8.1 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.8.0 to 5.8.1. - [Release notes](https://github.com/ipython/traitlets/releases) - [Changelog](https://github.com/ipython/traitlets/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipython/traitlets/compare/v5.8.0...v5.8.1) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3c8aca1734..ec931eab5f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -112,7 +112,7 @@ sortedcontainers==2.4.0 # via -r test-requirements.in tomlkit==0.11.6 # via pylint -traitlets==5.8.0 +traitlets==5.8.1 # via # ipython # matplotlib-inline From d0fecdd71e05932488d8508ce06b04ab50c27ac0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Jan 2023 10:14:40 +0000 Subject: [PATCH 1291/1498] Bump urllib3 from 1.26.13 to 1.26.14 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.13 to 1.26.14. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.14/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.13...1.26.14) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index b104045baf..61ebfe6e85 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -88,7 +88,7 @@ tomli==2.0.1 # via towncrier towncrier==22.8.0 # via -r docs-requirements.in -urllib3==1.26.13 +urllib3==1.26.14 # via requests # The following packages are considered to be unsafe in a requirements file: From 931a6f56be8537154b24d49b2bd8bb938aed070f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:06:02 +0000 Subject: [PATCH 1292/1498] Bump alabaster from 0.7.12 to 0.7.13 Bumps [alabaster](https://github.com/bitprophet/alabaster) from 0.7.12 to 0.7.13. - [Release notes](https://github.com/bitprophet/alabaster/releases) - [Changelog](https://github.com/bitprophet/alabaster/blob/main/docs/changelog.rst) - [Commits](https://github.com/bitprophet/alabaster/compare/0.7.12...0.7.13) --- updated-dependencies: - dependency-name: alabaster dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 61ebfe6e85..40c82b1b79 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile docs-requirements.in # -alabaster==0.7.12 +alabaster==0.7.13 # via sphinx async-generator==1.10 # via -r docs-requirements.in From 6744f3990ce2a060c4f3e1814d082b4400255143 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Jan 2023 10:18:17 +0000 Subject: [PATCH 1293/1498] Bump requests from 2.28.1 to 2.28.2 Bumps [requests](https://github.com/psf/requests) from 2.28.1 to 2.28.2. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.1...v2.28.2) --- updated-dependencies: - dependency-name: requests dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 61ebfe6e85..3048773850 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -55,7 +55,7 @@ pygments==2.14.0 # via sphinx pytz==2022.7 # via babel -requests==2.28.1 +requests==2.28.2 # via sphinx sniffio==1.3.0 # via -r docs-requirements.in From 3e0e6de898ad246acdb0ea543b13e5609365220f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 10:14:19 +0000 Subject: [PATCH 1294/1498] Bump charset-normalizer from 2.1.1 to 3.0.1 Bumps [charset-normalizer](https://github.com/Ousret/charset_normalizer) from 2.1.1 to 3.0.1. - [Release notes](https://github.com/Ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Upgrade guide](https://github.com/Ousret/charset_normalizer/blob/master/UPGRADE.md) - [Commits](https://github.com/Ousret/charset_normalizer/compare/2.1.1...3.0.1) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 52bb67e63c..545dae39de 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -16,7 +16,7 @@ babel==2.11.0 # via sphinx certifi==2022.12.7 # via requests -charset-normalizer==2.1.1 +charset-normalizer==3.0.1 # via requests click==8.1.3 # via From 5be78c92986408436780c89830799159501c8cd7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 10:16:28 +0000 Subject: [PATCH 1295/1498] Bump pytest from 7.2.0 to 7.2.1 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.2.0 to 7.2.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.2.0...7.2.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index ec931eab5f..6a8decd7f2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -100,7 +100,7 @@ pylint==2.14.5 # via -r test-requirements.in pyopenssl==23.0.0 # via -r test-requirements.in -pytest==7.2.0 +pytest==7.2.1 # via # -r test-requirements.in # pytest-cov From a29dce561f8ffdf96254edbfbaffac8bcf46cba7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 10:19:48 +0000 Subject: [PATCH 1296/1498] Bump pytz from 2022.7 to 2022.7.1 Bumps [pytz](https://github.com/stub42/pytz) from 2022.7 to 2022.7.1. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2022.7...release_2022.7.1) --- updated-dependencies: - dependency-name: pytz dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 52bb67e63c..2f4bcbc9bc 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -53,7 +53,7 @@ packaging==23.0 # via sphinx pygments==2.14.0 # via sphinx -pytz==2022.7 +pytz==2022.7.1 # via babel requests==2.28.2 # via sphinx From 4bb011e21c78dabeb79a6d72871903bff0b1588d Mon Sep 17 00:00:00 2001 From: Ganden Schaffner Date: Tue, 17 Jan 2023 00:00:00 -0800 Subject: [PATCH 1297/1498] Fix crash due to runtime import of typing_extensions typing_extensions doesn't presently need to be a runtime dependency. --- trio/_core/_run.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 31ff874a40..dc6ff21802 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -23,9 +23,6 @@ from sniffio import current_async_library_cvar from sortedcontainers import SortedDict -# An unfortunate name collision here with trio._util.Final -from typing_extensions import Final as FinalT - from .. import _core from .._util import Final, NoPublicConstructor, coroutine_or_error from ._asyncgens import AsyncGenerators @@ -47,6 +44,10 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup +if TYPE_CHECKING: + # An unfortunate name collision here with trio._util.Final + from typing_extensions import Final as FinalT + DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: FinalT = 1000 _NO_SEND: FinalT = object() From f1aca01181f70e9fd2261e7e7a47d9f09401acad Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 11:01:16 +0100 Subject: [PATCH 1298/1498] Address feedback --- trio/_core/_run.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 588886858c..ef24065327 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -13,8 +13,7 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, NoReturn, TypeVar, TYPE_CHECKING -from typing import Deque +from typing import Any, Deque, NoReturn, TypeVar, TYPE_CHECKING # An unfortunate name collision here with trio._util.Final from typing_extensions import Final as FinalT @@ -48,19 +47,23 @@ from .. import _core from .._util import Final, NoPublicConstructor, coroutine_or_error +if sys.version_info < (3, 9): + from typing import Callable +else: + from collections.abc import Callable + if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup -T = TypeVar("T") - DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: FinalT = 1000 _NO_SEND: FinalT = object() +FnT = TypeVar("FnT", bound=Callable[..., Any]) # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. -def _public(fn: T) -> T: +def _public(fn: FnT) -> FnT: return fn @@ -122,18 +125,18 @@ class SystemClock: # Add a large random offset to our clock to ensure that if people # accidentally call time.perf_counter() directly or start comparing clocks # between different runs, then they'll notice the bug quickly: - offset = attr.ib(factory=lambda: _r.uniform(10000, 200000)) + offset: float = attr.ib(factory=lambda: _r.uniform(10000, 200000)) - def start_clock(self): + def start_clock(self) -> None: pass # In cPython 3, on every platform except Windows, perf_counter is # exactly the same as time.monotonic; and on Windows, it uses # QueryPerformanceCounter instead of GetTickCount64. - def current_time(self): + def current_time(self) -> float: return self.offset + perf_counter() - def deadline_to_sleep_time(self, deadline): + def deadline_to_sleep_time(self, deadline: float) -> float: return deadline - self.current_time() From d3e58bf78383e29fa078ed92c7f57f597466cf79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jan 2023 10:17:34 +0000 Subject: [PATCH 1299/1498] Bump wcwidth from 0.2.5 to 0.2.6 Bumps [wcwidth](https://github.com/jquast/wcwidth) from 0.2.5 to 0.2.6. - [Release notes](https://github.com/jquast/wcwidth/releases) - [Commits](https://github.com/jquast/wcwidth/compare/0.2.5...0.2.6) --- updated-dependencies: - dependency-name: wcwidth dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6a8decd7f2..b3a8dc0ff3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -124,7 +124,7 @@ typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in # mypy -wcwidth==0.2.5 +wcwidth==0.2.6 # via prompt-toolkit wrapt==1.14.1 # via astroid From b3a20f8c8d885652b418dba3143a5751075c85e1 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 12:39:44 +0100 Subject: [PATCH 1300/1498] Delay annotation eval --- trio/_core/_run.py | 62 ++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index ef24065327..166232e2db 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1,56 +1,48 @@ +from __future__ import annotations + +import enum import functools +import gc import itertools import random import select import sys import threading -import gc +import warnings from collections import deque +from collections.abc import Callable from contextlib import contextmanager -import warnings -import enum - from contextvars import copy_context +from heapq import heapify, heappop, heappush from math import inf from time import perf_counter -from typing import Any, Deque, NoReturn, TypeVar, TYPE_CHECKING - -# An unfortunate name collision here with trio._util.Final -from typing_extensions import Final as FinalT - -from sniffio import current_async_library_cvar +from typing import TYPE_CHECKING, Any, NoReturn, TypeVar import attr -from heapq import heapify, heappop, heappush -from sortedcontainers import SortedDict from outcome import Error, Outcome, Value, capture +from sniffio import current_async_library_cvar +from sortedcontainers import SortedDict +# An unfortunate name collision here with trio._util.Final +from typing_extensions import Final as FinalT + +from .. import _core +from .._util import Final, NoPublicConstructor, coroutine_or_error +from ._asyncgens import AsyncGenerators from ._entry_queue import EntryQueue, TrioToken -from ._exceptions import TrioInternalError, RunFinishedError, Cancelled -from ._ki import ( - LOCALS_KEY_KI_PROTECTION_ENABLED, - KIManager, - enable_ki_protection, -) +from ._exceptions import Cancelled, RunFinishedError, TrioInternalError +from ._instrumentation import Instruments +from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED, KIManager, enable_ki_protection from ._multierror import MultiError, concat_tb +from ._thread_cache import start_thread_soon from ._traps import ( Abort, - wait_task_rescheduled, - cancel_shielded_checkpoint, CancelShieldedCheckpoint, PermanentlyDetachCoroutineObject, WaitTaskRescheduled, + cancel_shielded_checkpoint, + wait_task_rescheduled, ) -from ._asyncgens import AsyncGenerators -from ._thread_cache import start_thread_soon -from ._instrumentation import Instruments -from .. import _core -from .._util import Final, NoPublicConstructor, coroutine_or_error - -if sys.version_info < (3, 9): - from typing import Callable -else: - from collections.abc import Callable if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup @@ -1410,7 +1402,7 @@ class Runner: # Run-local values, see _local.py _locals = attr.ib(factory=dict) - runq: Deque[Task] = attr.ib(factory=deque) + runq: deque[Task] = attr.ib(factory=deque) tasks = attr.ib(factory=set) deadlines = attr.ib(factory=Deadlines) @@ -2530,16 +2522,16 @@ async def checkpoint_if_cancelled(): if sys.platform == "win32": - from ._io_windows import WindowsIOManager as TheIOManager from ._generated_io_windows import * + from ._io_windows import WindowsIOManager as TheIOManager elif sys.platform == "linux" or (not TYPE_CHECKING and hasattr(select, "epoll")): - from ._io_epoll import EpollIOManager as TheIOManager from ._generated_io_epoll import * + from ._io_epoll import EpollIOManager as TheIOManager elif TYPE_CHECKING or hasattr(select, "kqueue"): - from ._io_kqueue import KqueueIOManager as TheIOManager from ._generated_io_kqueue import * + from ._io_kqueue import KqueueIOManager as TheIOManager else: # pragma: no cover raise NotImplementedError("unsupported platform") -from ._generated_run import * from ._generated_instrumentation import * +from ._generated_run import * From c082f633d3b71eeb692c0297798e12742d34f5d5 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 12:43:29 +0100 Subject: [PATCH 1301/1498] Defer bound eval --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 166232e2db..31ff874a40 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -51,7 +51,7 @@ _NO_SEND: FinalT = object() -FnT = TypeVar("FnT", bound=Callable[..., Any]) +FnT = TypeVar("FnT", bound="Callable[..., Any]") # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. From fc4ed29fe20dbcd9619c7d2966465f0ee15f5e01 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 12:47:17 +0100 Subject: [PATCH 1302/1498] Revert removal of incomplete todo --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 6ccc92f5a4..31eeef1cd0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,7 @@ [mypy] +# TODO: run mypy against several OS/version combos in CI +# https://mypy.readthedocs.io/en/latest/command_line.html#platform-configuration + # Be flexible about dependencies that don't have stubs yet (like pytest) ignore_missing_imports = True From 82533e82aba33d6fdf299ebf4d842301876930c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jan 2023 10:06:15 +0000 Subject: [PATCH 1303/1498] Bump markupsafe from 2.1.1 to 2.1.2 Bumps [markupsafe](https://github.com/pallets/markupsafe) from 2.1.1 to 2.1.2. - [Release notes](https://github.com/pallets/markupsafe/releases) - [Changelog](https://github.com/pallets/markupsafe/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/markupsafe/compare/2.1.1...2.1.2) --- updated-dependencies: - dependency-name: markupsafe dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 62d4fbd44b..cdb4d4e9af 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -45,7 +45,7 @@ jinja2==3.0.3 # -r docs-requirements.in # sphinx # towncrier -markupsafe==2.1.1 +markupsafe==2.1.2 # via jinja2 outcome==1.2.0 # via -r docs-requirements.in From 40d745a21ecb4519e50e8b89ee7a5c1924eab162 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jan 2023 14:12:50 +0000 Subject: [PATCH 1304/1498] Bump towncrier from 22.8.0 to 22.12.0 Bumps [towncrier](https://github.com/twisted/towncrier) from 22.8.0 to 22.12.0. - [Release notes](https://github.com/twisted/towncrier/releases) - [Changelog](https://github.com/twisted/towncrier/blob/trunk/NEWS.rst) - [Commits](https://github.com/twisted/towncrier/compare/22.8.0...22.12.0) --- updated-dependencies: - dependency-name: towncrier dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index cdb4d4e9af..301426e830 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -84,9 +84,7 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -tomli==2.0.1 - # via towncrier -towncrier==22.8.0 +towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 # via requests From f087ec6dc3e070e897dfbd76977bc9abf7d06965 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jan 2023 10:05:34 +0000 Subject: [PATCH 1305/1498] Bump types-pyopenssl from 23.0.0.1 to 23.0.0.2 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 23.0.0.1 to 23.0.0.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a08f3b0b8e..bb80246d38 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -118,7 +118,7 @@ traitlets==5.8.1 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-pyopenssl==23.0.0.1 ; implementation_name == "cpython" +types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.4.0 ; implementation_name == "cpython" # via From 5a3144ede823c45c4fcb6328af31a85d3bd785b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:13:31 +0000 Subject: [PATCH 1306/1498] Bump pylint from 2.14.5 to 2.15.10 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.14.5 to 2.15.10. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.14.5...v2.15.10) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index bb80246d38..adbaa8e2fb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.11.7 +astroid==2.13.3 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -96,7 +96,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.14.0 # via ipython -pylint==2.14.5 +pylint==2.15.10 # via -r test-requirements.in pyopenssl==23.0.0 # via -r test-requirements.in From 90b688cc7cfc0ed13010a1731c62f0a0e3ed3587 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 10:11:51 +0000 Subject: [PATCH 1307/1498] Bump astroid from 2.13.3 to 2.13.5 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.13.3 to 2.13.5. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.13.3...v2.13.5) --- updated-dependencies: - dependency-name: astroid dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index adbaa8e2fb..6a479f7c7b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.13.3 +astroid==2.13.5 # via pylint async-generator==1.10 # via -r test-requirements.in From 8797e20bd3ed14a0cca08c450afc48d4f81f3411 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Feb 2023 10:04:53 +0000 Subject: [PATCH 1308/1498] Bump pylint from 2.15.10 to 2.16.0 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.15.10 to 2.16.0. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.15.10...v2.16.0) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 6a479f7c7b..9e73b0ac77 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.13.5 +astroid==2.14.1 # via pylint async-generator==1.10 # via -r test-requirements.in @@ -96,7 +96,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.14.0 # via ipython -pylint==2.15.10 +pylint==2.16.0 # via -r test-requirements.in pyopenssl==23.0.0 # via -r test-requirements.in From 798e87a126e391073c00c54dc2a8f466986cd7f3 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Sat, 28 Jan 2023 18:04:04 +0100 Subject: [PATCH 1309/1498] Add generic typing support for MemorySendChannel and MemoryReceiveChannel --- trio/_channel.py | 166 ++++++++++++++++++++++++++++------------- trio/_core/__init__.py | 1 + trio/_core/_traps.py | 7 +- trio/lowlevel.py | 1 + 4 files changed, 123 insertions(+), 52 deletions(-) diff --git a/trio/_channel.py b/trio/_channel.py index 1cecc55621..3fa225f2a3 100644 --- a/trio/_channel.py +++ b/trio/_channel.py @@ -1,6 +1,19 @@ +from __future__ import annotations + from collections import deque, OrderedDict +from collections.abc import Callable from math import inf +from types import TracebackType +from typing import ( + Any, + Generic, + NoReturn, + TypeVar, + TYPE_CHECKING, + Tuple, # only needed for typechecking on <3.9 +) + import attr from outcome import Error, Value @@ -8,11 +21,28 @@ from ._util import generic_function, NoPublicConstructor import trio -from ._core import enable_ki_protection +from ._core import enable_ki_protection, Task, Abort, RaiseCancelT + +# A regular invariant generic type +T = TypeVar("T") + +# The type of object produced by a ReceiveChannel (covariant because +# ReceiveChannel[Derived] can be passed to someone expecting +# ReceiveChannel[Base]) +ReceiveType = TypeVar("ReceiveType", covariant=True) + +# The type of object accepted by a SendChannel (contravariant because +# SendChannel[Base] can be passed to someone expecting +# SendChannel[Derived]) +SendType = TypeVar("SendType", contravariant=True) +# Temporary TypeVar needed until mypy release supports Self as a type +SelfT = TypeVar("SelfT") -@generic_function -def open_memory_channel(max_buffer_size): + +def _open_memory_channel( + max_buffer_size: int, +) -> tuple[MemorySendChannel[T], MemoryReceiveChannel[T]]: """Open a channel for passing objects between tasks within a process. Memory channels are lightweight, cheap to allocate, and entirely @@ -68,36 +98,57 @@ def open_memory_channel(max_buffer_size): raise TypeError("max_buffer_size must be an integer or math.inf") if max_buffer_size < 0: raise ValueError("max_buffer_size must be >= 0") - state = MemoryChannelState(max_buffer_size) + state: MemoryChannelState[T] = MemoryChannelState(max_buffer_size) return ( - MemorySendChannel._create(state), - MemoryReceiveChannel._create(state), + MemorySendChannel[T]._create(state), + MemoryReceiveChannel[T]._create(state), ) +# This workaround requires python3.9+, once older python versions are not supported +# or there's a better way of achieving type-checking on a generic factory function, +# it could replace the normal function header +if TYPE_CHECKING: + # written as a class so you can say open_memory_channel[int](5) + # Need to use Tuple instead of tuple due to CI check running on 3.8 + class open_memory_channel(Tuple[MemorySendChannel[T], MemoryReceiveChannel[T]]): + def __new__( # type: ignore[misc] # "must return a subtype" + cls, max_buffer_size: int + ) -> tuple[MemorySendChannel[T], MemoryReceiveChannel[T]]: + return _open_memory_channel(max_buffer_size) + + def __init__(self, max_buffer_size: int): + ... + +else: + # apply the generic_function decorator to make open_memory_channel indexable + # so it's valid to say e.g. ``open_memory_channel[bytes](5)`` at runtime + open_memory_channel = generic_function(_open_memory_channel) + + @attr.s(frozen=True, slots=True) class MemoryChannelStats: - current_buffer_used = attr.ib() - max_buffer_size = attr.ib() - open_send_channels = attr.ib() - open_receive_channels = attr.ib() - tasks_waiting_send = attr.ib() - tasks_waiting_receive = attr.ib() + current_buffer_used: int = attr.ib() + max_buffer_size: int = attr.ib() + open_send_channels: int = attr.ib() + open_receive_channels: int = attr.ib() + tasks_waiting_send: int = attr.ib() + tasks_waiting_receive: int = attr.ib() @attr.s(slots=True) -class MemoryChannelState: - max_buffer_size = attr.ib() - data = attr.ib(factory=deque) +class MemoryChannelState(Generic[T]): + max_buffer_size: int = attr.ib() + data: deque[T] = attr.ib(factory=deque) # Counts of open endpoints using this state - open_send_channels = attr.ib(default=0) - open_receive_channels = attr.ib(default=0) + open_send_channels: int = attr.ib(default=0) + open_receive_channels: int = attr.ib(default=0) # {task: value} - send_tasks = attr.ib(factory=OrderedDict) + send_tasks: OrderedDict[Task, T] = attr.ib(factory=OrderedDict) # {task: None} - receive_tasks = attr.ib(factory=OrderedDict) + receive_tasks: OrderedDict[Task, None] = attr.ib(factory=OrderedDict) - def statistics(self): + def statistics(self) -> MemoryChannelStats: return MemoryChannelStats( current_buffer_used=len(self.data), max_buffer_size=self.max_buffer_size, @@ -109,28 +160,28 @@ def statistics(self): @attr.s(eq=False, repr=False) -class MemorySendChannel(SendChannel, metaclass=NoPublicConstructor): - _state = attr.ib() - _closed = attr.ib(default=False) +class MemorySendChannel(SendChannel[SendType], metaclass=NoPublicConstructor): + _state: MemoryChannelState[SendType] = attr.ib() + _closed: bool = attr.ib(default=False) # This is just the tasks waiting on *this* object. As compared to # self._state.send_tasks, which includes tasks from this object and # all clones. - _tasks = attr.ib(factory=set) + _tasks: set[Task] = attr.ib(factory=set) - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: self._state.open_send_channels += 1 - def __repr__(self): + def __repr__(self) -> str: return "".format( id(self), id(self._state) ) - def statistics(self): + def statistics(self) -> MemoryChannelStats: # XX should we also report statistics specific to this object? return self._state.statistics() @enable_ki_protection - def send_nowait(self, value): + def send_nowait(self, value: SendType) -> None: """Like `~trio.abc.SendChannel.send`, but if the channel's buffer is full, raises `WouldBlock` instead of blocking. @@ -150,7 +201,7 @@ def send_nowait(self, value): raise trio.WouldBlock @enable_ki_protection - async def send(self, value): + async def send(self, value: SendType) -> None: """See `SendChannel.send `. Memory channels allow multiple tasks to call `send` at the same time. @@ -170,15 +221,16 @@ async def send(self, value): self._state.send_tasks[task] = value task.custom_sleep_data = self - def abort_fn(_): + def abort_fn(_: RaiseCancelT) -> Abort: self._tasks.remove(task) del self._state.send_tasks[task] return trio.lowlevel.Abort.SUCCEEDED await trio.lowlevel.wait_task_rescheduled(abort_fn) + # Return type must be stringified or use a TypeVar @enable_ki_protection - def clone(self): + def clone(self) -> "MemorySendChannel[SendType]": """Clone this send channel object. This returns a new `MemorySendChannel` object, which acts as a @@ -206,14 +258,19 @@ def clone(self): raise trio.ClosedResourceError return MemorySendChannel._create(self._state) - def __enter__(self): + def __enter__(self: SelfT) -> SelfT: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: self.close() @enable_ki_protection - def close(self): + def close(self) -> None: """Close this send channel object synchronously. All channel objects have an asynchronous `~.AsyncResource.aclose` method. @@ -241,30 +298,30 @@ def close(self): self._state.receive_tasks.clear() @enable_ki_protection - async def aclose(self): + async def aclose(self) -> None: self.close() await trio.lowlevel.checkpoint() @attr.s(eq=False, repr=False) -class MemoryReceiveChannel(ReceiveChannel, metaclass=NoPublicConstructor): - _state = attr.ib() - _closed = attr.ib(default=False) - _tasks = attr.ib(factory=set) +class MemoryReceiveChannel(ReceiveChannel[ReceiveType], metaclass=NoPublicConstructor): + _state: MemoryChannelState[ReceiveType] = attr.ib() + _closed: bool = attr.ib(default=False) + _tasks: set[trio._core._run.Task] = attr.ib(factory=set) - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: self._state.open_receive_channels += 1 - def statistics(self): + def statistics(self) -> MemoryChannelStats: return self._state.statistics() - def __repr__(self): + def __repr__(self) -> str: return "".format( id(self), id(self._state) ) @enable_ki_protection - def receive_nowait(self): + def receive_nowait(self) -> ReceiveType: """Like `~trio.abc.ReceiveChannel.receive`, but if there's nothing ready to receive, raises `WouldBlock` instead of blocking. @@ -284,7 +341,7 @@ def receive_nowait(self): raise trio.WouldBlock @enable_ki_protection - async def receive(self): + async def receive(self) -> ReceiveType: """See `ReceiveChannel.receive `. Memory channels allow multiple tasks to call `receive` at the same @@ -306,15 +363,17 @@ async def receive(self): self._state.receive_tasks[task] = None task.custom_sleep_data = self - def abort_fn(_): + def abort_fn(_: RaiseCancelT) -> Abort: self._tasks.remove(task) del self._state.receive_tasks[task] return trio.lowlevel.Abort.SUCCEEDED - return await trio.lowlevel.wait_task_rescheduled(abort_fn) + # Not strictly guaranteed to return ReceiveType, but will do so unless + # you intentionally reschedule with a bad value. + return await trio.lowlevel.wait_task_rescheduled(abort_fn) # type: ignore[no-any-return] @enable_ki_protection - def clone(self): + def clone(self) -> "MemoryReceiveChannel[ReceiveType]": """Clone this receive channel object. This returns a new `MemoryReceiveChannel` object, which acts as a @@ -345,14 +404,19 @@ def clone(self): raise trio.ClosedResourceError return MemoryReceiveChannel._create(self._state) - def __enter__(self): + def __enter__(self: SelfT) -> SelfT: return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: self.close() @enable_ki_protection - def close(self): + def close(self) -> None: """Close this receive channel object synchronously. All channel objects have an asynchronous `~.AsyncResource.aclose` method. @@ -381,6 +445,6 @@ def close(self): self._state.data.clear() @enable_ki_protection - async def aclose(self): + async def aclose(self) -> None: self.close() await trio.lowlevel.checkpoint() diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 8e3e526cfe..f9919b8323 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -55,6 +55,7 @@ from ._traps import ( cancel_shielded_checkpoint, Abort, + RaiseCancelT, wait_task_rescheduled, temporarily_detach_coroutine_object, permanently_detach_coroutine_object, diff --git a/trio/_core/_traps.py b/trio/_core/_traps.py index 95cf46de9b..39481c5903 100644 --- a/trio/_core/_traps.py +++ b/trio/_core/_traps.py @@ -8,6 +8,7 @@ from . import _run +from typing import Callable, NoReturn, Any # Helper for the bottommost 'yield'. You can't use 'yield' inside an async # function, but you can inside a generator, and if you decorate your generator @@ -64,7 +65,11 @@ class WaitTaskRescheduled: abort_func = attr.ib() -async def wait_task_rescheduled(abort_func): +RaiseCancelT = Callable[[], NoReturn] # TypeAlias + +# Should always return the type a Task "expects", unless you willfully reschedule it +# with a bad value. +async def wait_task_rescheduled(abort_func: Callable[[RaiseCancelT], Abort]) -> Any: """Put the current task to sleep, with cancellation support. This is the lowest-level API for blocking in Trio. Every time a diff --git a/trio/lowlevel.py b/trio/lowlevel.py index f30fdcc4bd..004692475f 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -16,6 +16,7 @@ from ._core import ( cancel_shielded_checkpoint, Abort, + RaiseCancelT, wait_task_rescheduled, enable_ki_protection, disable_ki_protection, From 323973f56fd2edf0f61fe895eb5a14f8a38508bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Feb 2023 10:17:31 +0000 Subject: [PATCH 1310/1498] Bump platformdirs from 2.6.0 to 3.0.0 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.6.0 to 3.0.0. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.6.0...3.0.0) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9e73b0ac77..d067abf15a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -78,7 +78,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.6.0 +platformdirs==3.0.0 # via # black # pylint From 54b4940f0deb9892d730c43dc3201fd8a18bf817 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:50:39 +0000 Subject: [PATCH 1311/1498] Bump sphinxcontrib-applehelp from 1.0.2 to 1.0.4 Bumps [sphinxcontrib-applehelp](https://github.com/sphinx-doc/sphinxcontrib-applehelp) from 1.0.2 to 1.0.4. - [Release notes](https://github.com/sphinx-doc/sphinxcontrib-applehelp/releases) - [Changelog](https://github.com/sphinx-doc/sphinxcontrib-applehelp/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinxcontrib-applehelp/compare/1.0.2...1.0.4) --- updated-dependencies: - dependency-name: sphinxcontrib-applehelp dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index cdb4d4e9af..7a93ab9f68 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -70,7 +70,7 @@ sphinx==3.3.1 # sphinxcontrib-trio sphinx-rtd-theme==1.1.1 # via -r docs-requirements.in -sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx From b1a18078ac5a4e3eff302388678f0b13318c6aad Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Tue, 7 Feb 2023 14:59:49 +0000 Subject: [PATCH 1312/1498] Bump Python version for RTD --- .readthedocs.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 2a32d6c9b5..9fde00ef8f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,8 +5,12 @@ formats: - htmlzip - epub +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + python: - version: 3.7 install: - requirements: docs-requirements.txt From d27e97c096ca35887e0aaf02e4cea45b48349e35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 02:11:45 +0000 Subject: [PATCH 1313/1498] Bump cryptography from 39.0.0 to 39.0.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 39.0.0 to 39.0.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/39.0.0...39.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index d067abf15a..166f05a546 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ click==8.1.3 # via black coverage[toml]==6.4.1 # via pytest-cov -cryptography==39.0.0 +cryptography==39.0.1 # via # -r test-requirements.in # pyopenssl From 838ae9bb87ce7edf55dea66595cfc90f7bda0d45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 02:12:15 +0000 Subject: [PATCH 1314/1498] Bump sphinxcontrib-htmlhelp from 2.0.0 to 2.0.1 Bumps [sphinxcontrib-htmlhelp](https://github.com/sphinx-doc/sphinxcontrib-htmlhelp) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/sphinx-doc/sphinxcontrib-htmlhelp/releases) - [Changelog](https://github.com/sphinx-doc/sphinxcontrib-htmlhelp/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinxcontrib-htmlhelp/commits) --- updated-dependencies: - dependency-name: sphinxcontrib-htmlhelp dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 7a93ab9f68..9f669af4a3 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -74,7 +74,7 @@ sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-htmlhelp==2.0.1 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx From 10ad6f60adaf466ce966ed02b45e2414bfc5ef98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 03:09:56 +0000 Subject: [PATCH 1315/1498] Bump prompt-toolkit from 3.0.33 to 3.0.36 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.33 to 3.0.36. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.33...3.0.36) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 0f989caca3..60981ad92c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -84,7 +84,7 @@ platformdirs==3.0.0 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.33 +prompt-toolkit==3.0.36 # via ipython ptyprocess==0.7.0 # via pexpect From 558f61c95945e9924a0ec3e438326c21e7dca7d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:08:24 +0000 Subject: [PATCH 1316/1498] Bump pylint from 2.16.0 to 2.16.1 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.16.0 to 2.16.1. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.16.0...v2.16.1) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 60981ad92c..5a7afe8045 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -96,7 +96,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.14.0 # via ipython -pylint==2.16.0 +pylint==2.16.1 # via -r test-requirements.in pyopenssl==23.0.0 # via -r test-requirements.in From f69c595eec90d8417f58352b490b502d32508790 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:12:03 +0000 Subject: [PATCH 1317/1498] Bump mypy from 0.991 to 1.0.0 Bumps [mypy](https://github.com/python/mypy) from 0.991 to 1.0.0. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.991...v1.0.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 60981ad92c..7408bedbdc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -59,7 +59,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==0.991 ; implementation_name == "cpython" +mypy==1.0.0 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==0.4.3 ; implementation_name == "cpython" # via From 5d904c465c40562b9e7b9bb95b64c712fccbcf1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:15:18 +0000 Subject: [PATCH 1318/1498] Bump mypy-extensions from 0.4.3 to 1.0.0 Bumps [mypy-extensions](https://github.com/python/mypy_extensions) from 0.4.3 to 1.0.0. - [Release notes](https://github.com/python/mypy_extensions/releases) - [Commits](https://github.com/python/mypy_extensions/compare/0.4.3...1.0.0) --- updated-dependencies: - dependency-name: mypy-extensions dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 60981ad92c..09f97847e1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -61,7 +61,7 @@ mccabe==0.6.1 # pylint mypy==0.991 ; implementation_name == "cpython" # via -r test-requirements.in -mypy-extensions==0.4.3 ; implementation_name == "cpython" +mypy-extensions==1.0.0 ; implementation_name == "cpython" # via # -r test-requirements.in # black From 7d9c83edeb7746d6522f89f8c1a02c0935bc15b8 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Wed, 8 Feb 2023 03:34:33 +0000 Subject: [PATCH 1319/1498] Run pip-compile in formatting check Additionally, commit autoformatter changes for Dependabot --- .github/workflows/ci.yml | 8 ++++++++ check.sh | 8 ++++++++ docs-requirements.txt | 6 ++++-- test-requirements.in | 1 + test-requirements.txt | 36 ++++++++++++++++++++++++++++++++---- 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92bc20539d..047083876f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,6 +120,14 @@ jobs: CHECK_FORMATTING: '${{ matrix.check_formatting }}' # Should match 'name:' up above JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' + - name: Commit autoformatter changes + continue-on-error: true + if: failure() && matrix.check_formatting == '1' && github.actor == 'dependabot[bot]' + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git commit -am "Autoformatter changes" + git push macOS: name: 'macOS (${{ matrix.python }})' diff --git a/check.sh b/check.sh index 57f1e2db40..fbd440521c 100755 --- a/check.sh +++ b/check.sh @@ -28,6 +28,14 @@ mypy -m trio -m trio.testing --platform linux || EXIT_STATUS=$? mypy -m trio -m trio.testing --platform darwin || EXIT_STATUS=$? # tests FreeBSD too mypy -m trio -m trio.testing --platform win32 || EXIT_STATUS=$? +# Check pip compile is consistent +pip-compile test-requirements.in +pip-compile docs-requirements.in + +if git status --porcelain | grep -q "requirements.txt"; then + EXIT_STATUS=1 +fi + # Finally, leave a really clear warning of any issues and exit if [ $EXIT_STATUS -ne 0 ]; then cat < Date: Fri, 10 Feb 2023 14:34:19 +0000 Subject: [PATCH 1320/1498] Use the correct user for GitHub Actions --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 047083876f..1584e356f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,8 +124,8 @@ jobs: continue-on-error: true if: failure() && matrix.check_formatting == '1' && github.actor == 'dependabot[bot]' run: | - git config user.name github-actions - git config user.email github-actions@github.com + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git commit -am "Autoformatter changes" git push From b1a8ef1dd37b16933ec829734d08e37ab7aef98d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Feb 2023 01:52:46 +0000 Subject: [PATCH 1321/1498] Bump sphinx-rtd-theme from 1.1.1 to 1.2.0 Bumps [sphinx-rtd-theme](https://github.com/readthedocs/sphinx_rtd_theme) from 1.1.1 to 1.2.0. - [Release notes](https://github.com/readthedocs/sphinx_rtd_theme/releases) - [Changelog](https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/changelog.rst) - [Commits](https://github.com/readthedocs/sphinx_rtd_theme/compare/1.1.1...1.2.0) --- updated-dependencies: - dependency-name: sphinx-rtd-theme dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 537e0f908b..46c19380e3 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -68,7 +68,7 @@ sphinx==3.3.1 # -r docs-requirements.in # sphinx-rtd-theme # sphinxcontrib-trio -sphinx-rtd-theme==1.1.1 +sphinx-rtd-theme==1.2.0 # via -r docs-requirements.in sphinxcontrib-applehelp==1.0.4 # via sphinx @@ -76,6 +76,8 @@ sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==2.0.1 # via sphinx +sphinxcontrib-jquery==2.0.0 + # via sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 @@ -84,8 +86,6 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -tomli==2.0.1 - # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 From 104ed0beea71b3a689e7ad232dedf2abf2b89340 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Sat, 11 Feb 2023 01:57:00 +0000 Subject: [PATCH 1322/1498] Fix autocommit step --- .github/workflows/ci.yml | 2 +- check.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1584e356f2..d9facabbe1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,7 +127,7 @@ jobs: git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git commit -am "Autoformatter changes" - git push + git push origin "HEAD:${GITHUB_HEAD_REF}" macOS: name: 'macOS (${{ matrix.python }})' diff --git a/check.sh b/check.sh index fbd440521c..8416a9c5d1 100755 --- a/check.sh +++ b/check.sh @@ -33,6 +33,7 @@ pip-compile test-requirements.in pip-compile docs-requirements.in if git status --porcelain | grep -q "requirements.txt"; then + git status --porcelain EXIT_STATUS=1 fi From a211f635def378353b246d514d5d92c95c19a8cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Feb 2023 02:19:23 +0000 Subject: [PATCH 1323/1498] Bump black from 22.12.0 to 23.1.0 Bumps [black](https://github.com/psf/black) from 22.12.0 to 23.1.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.12.0...23.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index e6d832f9d0..1d45dd93ec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -17,7 +17,7 @@ attrs==22.2.0 # pytest backcall==0.2.0 # via ipython -black==22.12.0 ; implementation_name == "cpython" +black==23.1.0 ; implementation_name == "cpython" # via -r test-requirements.in build==0.10.0 # via pip-tools @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -78,6 +74,7 @@ outcome==1.2.0 # via -r test-requirements.in packaging==23.0 # via + # black # build # pytest parso==0.8.3 @@ -124,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.8.1 @@ -145,10 +134,7 @@ types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From b196eb6e6c92fa7cffca14f607c552e5129b9da3 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Sat, 11 Feb 2023 02:24:31 +0000 Subject: [PATCH 1324/1498] Try to fix committing one more time --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9facabbe1..3c5a8645c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.ref }} - name: Setup python uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" @@ -122,12 +124,12 @@ jobs: JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' - name: Commit autoformatter changes continue-on-error: true - if: failure() && matrix.check_formatting == '1' && github.actor == 'dependabot[bot]' + if: failure() && matrix.check_formatting == '1' #&& github.actor == 'dependabot[bot]' run: | git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git commit -am "Autoformatter changes" - git push origin "HEAD:${GITHUB_HEAD_REF}" + git push macOS: name: 'macOS (${{ matrix.python }})' From 93205f1957d35d8660907c3999192165f8340500 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 11 Feb 2023 02:47:54 +0000 Subject: [PATCH 1325/1498] Autoformatter changes --- docs-requirements.txt | 2 ++ test-requirements.txt | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/docs-requirements.txt b/docs-requirements.txt index 46c19380e3..ae4527f7c7 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -86,6 +86,8 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in +tomli==2.0.1 + # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 diff --git a/test-requirements.txt b/test-requirements.txt index 1d45dd93ec..8676cb210a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.8.1 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From bfe6e9b47cc19f392b508d4cdcc082dbb2c0ea64 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Sat, 11 Feb 2023 02:50:10 +0000 Subject: [PATCH 1326/1498] Remove the testing configuration for autoformatting --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c5a8645c8..fcc10c8104 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,7 +124,7 @@ jobs: JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' - name: Commit autoformatter changes continue-on-error: true - if: failure() && matrix.check_formatting == '1' #&& github.actor == 'dependabot[bot]' + if: failure() && matrix.check_formatting == '1' && github.actor == 'dependabot[bot]' run: | git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' From 0381fef5a40d5206c018f9fd3f99e8770e798855 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Sat, 11 Feb 2023 02:54:53 +0000 Subject: [PATCH 1327/1498] Make sure to actually commit black's changes --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcc10c8104..87c6fb93d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,7 @@ jobs: continue-on-error: true if: failure() && matrix.check_formatting == '1' && github.actor == 'dependabot[bot]' run: | + black setup.py trio git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git commit -am "Autoformatter changes" From 699efc898caeb87b208194b948b0380ae91d53d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Feb 2023 03:05:25 +0000 Subject: [PATCH 1328/1498] Bump pathspec from 0.10.3 to 0.11.0 Bumps [pathspec](https://github.com/cpburnz/python-pathspec) from 0.10.3 to 0.11.0. - [Release notes](https://github.com/cpburnz/python-pathspec/releases) - [Changelog](https://github.com/cpburnz/python-pathspec/blob/master/CHANGES.rst) - [Commits](https://github.com/cpburnz/python-pathspec/compare/v0.10.3...v0.11.0) --- updated-dependencies: - dependency-name: pathspec dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 8676cb210a..a65044dd89 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -83,7 +79,7 @@ packaging==23.0 # pytest parso==0.8.3 # via jedi -pathspec==0.10.3 +pathspec==0.11.0 # via black pexpect==4.8.0 # via ipython @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.8.1 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From b1fbbd3b870a4acbc8ea14bb76ac6a4a2920ed22 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Sat, 11 Feb 2023 03:12:21 +0000 Subject: [PATCH 1329/1498] Give the committing job enough permissions --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87c6fb93d8..0b650fc520 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,12 @@ on: - "dependabot/**" pull_request: +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#changing-github_token-permissions +permissions: + pull-requests: write + issues: write + repository-projects: write + jobs: Windows: name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' From 04b4b8818fdf305947f4bde6c0fed64430a13cad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Feb 2023 03:21:37 +0000 Subject: [PATCH 1330/1498] Bump traitlets from 5.8.1 to 5.9.0 Bumps [traitlets](https://github.com/ipython/traitlets) from 5.8.1 to 5.9.0. - [Release notes](https://github.com/ipython/traitlets/releases) - [Changelog](https://github.com/ipython/traitlets/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipython/traitlets/compare/v5.8.1...v5.9.0) --- updated-dependencies: - dependency-name: traitlets dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index a65044dd89..2ea1640ff0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -123,7 +123,7 @@ sortedcontainers==2.4.0 # via -r test-requirements.in tomlkit==0.11.6 # via pylint -traitlets==5.8.1 +traitlets==5.9.0 # via # ipython # matplotlib-inline From 4ed59c327baa646662e1662295c8184e4a4c341a Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Sat, 11 Feb 2023 03:33:09 +0000 Subject: [PATCH 1331/1498] GITHUB_TOKEN needs `contents` for `git push` --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b650fc520..55cd0bbf4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,6 @@ on: - "dependabot/**" pull_request: -# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#changing-github_token-permissions -permissions: - pull-requests: write - issues: write - repository-projects: write - jobs: Windows: name: 'Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})' @@ -77,6 +71,12 @@ jobs: name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' timeout-minutes: 10 runs-on: 'ubuntu-latest' + # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#changing-github_token-permissions + permissions: + pull-requests: write + issues: write + repository-projects: write + contents: write strategy: fail-fast: false matrix: From 4d02fa4028391b75d78483b3ba89ac0ff7736522 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Feb 2023 11:12:28 +0000 Subject: [PATCH 1332/1498] Bump astroid from 2.14.1 to 2.14.2 Bumps [astroid](https://github.com/PyCQA/astroid) from 2.14.1 to 2.14.2. - [Release notes](https://github.com/PyCQA/astroid/releases) - [Changelog](https://github.com/PyCQA/astroid/blob/main/ChangeLog) - [Commits](https://github.com/PyCQA/astroid/compare/v2.14.1...v2.14.2) --- updated-dependencies: - dependency-name: astroid dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2ea1640ff0..c097fed8f9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # astor==0.8.1 # via -r test-requirements.in -astroid==2.14.1 +astroid==2.14.2 # via pylint async-generator==1.10 # via -r test-requirements.in From 1b4ed1c1b56287f2b6c03ba9841e996f5f46de3f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Feb 2023 11:14:01 +0000 Subject: [PATCH 1333/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ trio/_core/_instrumentation.py | 1 + trio/_core/_run.py | 1 + trio/_core/_traps.py | 2 ++ trio/_core/tests/test_guest_mode.py | 2 ++ trio/_socket.py | 1 + 6 files changed, 22 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index c097fed8f9..adab2e00bb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 diff --git a/trio/_core/_instrumentation.py b/trio/_core/_instrumentation.py index e14c1ef1e0..b133d47406 100644 --- a/trio/_core/_instrumentation.py +++ b/trio/_core/_instrumentation.py @@ -11,6 +11,7 @@ F = TypeVar("F", bound=Callable[..., Any]) + # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. def _public(fn: F) -> F: diff --git a/trio/_core/_run.py b/trio/_core/_run.py index dc6ff21802..7811af172c 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -54,6 +54,7 @@ FnT = TypeVar("FnT", bound="Callable[..., Any]") + # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. def _public(fn: FnT) -> FnT: diff --git a/trio/_core/_traps.py b/trio/_core/_traps.py index 39481c5903..aedf839a8d 100644 --- a/trio/_core/_traps.py +++ b/trio/_core/_traps.py @@ -10,6 +10,7 @@ from typing import Callable, NoReturn, Any + # Helper for the bottommost 'yield'. You can't use 'yield' inside an async # function, but you can inside a generator, and if you decorate your generator # with @types.coroutine, then it's even awaitable. However, it's still not a @@ -67,6 +68,7 @@ class WaitTaskRescheduled: RaiseCancelT = Callable[[], NoReturn] # TypeAlias + # Should always return the type a Task "expects", unless you willfully reschedule it # with a bad value. async def wait_task_rescheduled(abort_func: Callable[[RaiseCancelT], Abort]) -> Any: diff --git a/trio/_core/tests/test_guest_mode.py b/trio/_core/tests/test_guest_mode.py index a5d6d78e18..9fed232214 100644 --- a/trio/_core/tests/test_guest_mode.py +++ b/trio/_core/tests/test_guest_mode.py @@ -17,6 +17,7 @@ from .tutil import gc_collect_harder, buggy_pypy_asyncgens, restore_unraisablehook from ..._util import signal_raise + # The simplest possible "host" loop. # Nice features: # - we can run code "outside" of trio using the schedule function passed to @@ -227,6 +228,7 @@ async def sit_in_wait_all_tasks_blocked(watb_cscope): async def get_woken_by_host_deadline(watb_cscope): with trio.CancelScope() as cscope: print("scheduling stuff to happen") + # Altering the deadline from the host, to something in the # future, will cause the run loop to wake up, but then # discover that there is nothing to do and go back to sleep. diff --git a/trio/_socket.py b/trio/_socket.py index 8bfa6d265a..b12126f7e1 100644 --- a/trio/_socket.py +++ b/trio/_socket.py @@ -353,6 +353,7 @@ async def wrapper(self, *args, **kwargs): # addresses everywhere. Split out into a standalone function so it can be reused by # FakeNet. + # Take an address in Python's representation, and returns a new address in # the same representation, but with names resolved to numbers, # etc. From 1ae9e480bd38339fa588a964f6a20d9bc3b16a16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:03:30 +0000 Subject: [PATCH 1334/1498] Bump pylint from 2.16.1 to 2.16.2 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.16.1 to 2.16.2. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.16.1...v2.16.2) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index adab2e00bb..14c1945362 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -109,7 +105,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.14.0 # via ipython -pylint==2.16.1 +pylint==2.16.2 # via -r test-requirements.in pyopenssl==23.0.0 # via -r test-requirements.in @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 6d95ae98120a061750a80c3cda74d3d0dedc2c8e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:05:18 +0000 Subject: [PATCH 1335/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 14c1945362..307428a3a6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 64e9b2094f92f800dcc056d413d1f38684435387 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:08:26 +0000 Subject: [PATCH 1336/1498] Bump types-pyopenssl from 23.0.0.2 to 23.0.0.3 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 23.0.0.2 to 23.0.0.3. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index adab2e00bb..e30981eb80 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -141,15 +129,12 @@ traitlets==5.9.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" +types-pyopenssl==23.0.0.3 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From c9adb4960be822e3885629db3bf57a60c4f783fc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 11:09:35 +0000 Subject: [PATCH 1337/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index e30981eb80..7fdf78d3ef 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.3 ; implementation_name == "cpython" typing-extensions==4.4.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 6a49dce2acd50d3587db061adfa19e1d1f8387db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:46:40 +0000 Subject: [PATCH 1338/1498] Bump sphinx from 3.3.1 to 6.1.3 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.3.1 to 6.1.3. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.3.1...v6.1.3) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs-requirements.in | 2 +- docs-requirements.txt | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs-requirements.in b/docs-requirements.in index 7d6abc0033..fab339e1f9 100644 --- a/docs-requirements.in +++ b/docs-requirements.in @@ -1,6 +1,6 @@ # RTD is currently installing 1.5.3, which has a bug in :lineno-match: # sphinx-3.4 causes warnings about some trio._abc classes: GH#2338 -sphinx >= 1.7.0, < 3.4 +sphinx >= 1.7.0, < 6.2 # jinja2-3.1 causes importerror with sphinx<4.0 jinja2 < 3.1 sphinx_rtd_theme diff --git a/docs-requirements.txt b/docs-requirements.txt index ae4527f7c7..004b0f0fe0 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -24,7 +24,7 @@ click==8.1.3 # towncrier click-default-group==1.2.2 # via towncrier -docutils==0.17.1 +docutils==0.18.1 # via # sphinx # sphinx-rtd-theme @@ -63,7 +63,7 @@ snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via -r docs-requirements.in -sphinx==3.3.1 +sphinx==6.1.3 # via # -r docs-requirements.in # sphinx-rtd-theme @@ -86,8 +86,6 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -tomli==2.0.1 - # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 From 2b43f76367e8a8650bf3db20b3a556239894a83f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 12:47:51 +0000 Subject: [PATCH 1339/1498] Autoformatter changes --- docs-requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs-requirements.txt b/docs-requirements.txt index 004b0f0fe0..c83fa2b285 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -38,6 +38,8 @@ imagesize==1.4.1 # via sphinx immutables==0.19 # via -r docs-requirements.in +importlib-metadata==6.0.0 + # via sphinx incremental==22.10.0 # via towncrier jinja2==3.0.3 @@ -86,10 +88,14 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in +tomli==2.0.1 + # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 # via requests +zipp==3.13.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From c6d2c071e1b1c37ba128927ffbcaa0644cbb0065 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 11:01:18 +0000 Subject: [PATCH 1340/1498] Bump typing-extensions from 4.4.0 to 4.5.0 Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.4.0 to 4.5.0. - [Release notes](https://github.com/python/typing_extensions/releases) - [Changelog](https://github.com/python/typing_extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/python/typing_extensions/compare/4.4.0...4.5.0) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 307428a3a6..f2bca755f8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -143,13 +131,10 @@ trustme==0.9.0 # via -r test-requirements.in types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" # via -r test-requirements.in -typing-extensions==4.4.0 ; implementation_name == "cpython" +typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From d07a051a57dc334693c3e77a7cf29089dc6bee42 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 11:02:56 +0000 Subject: [PATCH 1341/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index f2bca755f8..9a2dd1651e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.2 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 17d76da1a9403647b5c7113b615eba4ee6f3f82f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Feb 2023 07:24:42 +0000 Subject: [PATCH 1342/1498] Bump mypy from 1.0.0 to 1.0.1 Bumps [mypy](https://github.com/python/mypy) from 1.0.0 to 1.0.1. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v1.0.0...v1.0.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5e6e39b6df..e4ec917b7b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -67,7 +63,7 @@ mccabe==0.6.1 # via # flake8 # pylint -mypy==1.0.0 ; implementation_name == "cpython" +mypy==1.0.1 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==1.0.0 ; implementation_name == "cpython" # via @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.3 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 5923c0774903f4777dc8cae119c724bbd6dc8653 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Feb 2023 07:27:20 +0000 Subject: [PATCH 1343/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index e4ec917b7b..d6d2937bc8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.3 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 0ec92a50fbf8d2c838dd1e7e140a334c5b166b36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:00:30 +0000 Subject: [PATCH 1344/1498] Bump types-pyopenssl from 23.0.0.3 to 23.0.0.4 Bumps [types-pyopenssl](https://github.com/python/typeshed) from 23.0.0.3 to 23.0.0.4. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyopenssl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5e6e39b6df..6c34093e8d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -141,15 +129,12 @@ traitlets==5.9.0 # matplotlib-inline trustme==0.9.0 # via -r test-requirements.in -types-pyopenssl==23.0.0.3 ; implementation_name == "cpython" +types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" # via -r test-requirements.in typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 28d080a012d94af8d94ea11c5c05bf56f1b36200 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:02:00 +0000 Subject: [PATCH 1345/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 6c34093e8d..7082dc4212 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From de8ca19fa2849e07a709603bd90c0adb9b498c42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 10:22:59 +0000 Subject: [PATCH 1346/1498] Bump prompt-toolkit from 3.0.36 to 3.0.37 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.36 to 3.0.37. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.36...3.0.37) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7082dc4212..cd79a98cbc 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -97,7 +93,7 @@ platformdirs==3.0.0 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.36 +prompt-toolkit==3.0.37 # via ipython ptyprocess==0.7.0 # via pexpect @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 14071f49895456085b54308640e59dd7bd27f101 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Feb 2023 10:24:39 +0000 Subject: [PATCH 1347/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index cd79a98cbc..50026195e7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 3c048625f7dff7f9e97b97abcc83cc8a5bb6023a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Feb 2023 11:05:46 +0000 Subject: [PATCH 1348/1498] Bump wrapt from 1.14.1 to 1.15.0 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.14.1 to 1.15.0. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.14.1...1.15.0) --- updated-dependencies: - dependency-name: wrapt dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 50026195e7..de2fcea2f8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,15 +134,12 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 # via pip-tools -wrapt==1.14.1 +wrapt==1.15.0 # via astroid # The following packages are considered to be unsafe in a requirements file: From 595362c04051795a48940996603ee5e2041790ac Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 27 Feb 2023 11:07:34 +0000 Subject: [PATCH 1349/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index de2fcea2f8..aa6ae164db 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 06372750829653c43796d61fabfa8439b6e51ba7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 11:06:17 +0000 Subject: [PATCH 1350/1498] Bump babel from 2.11.0 to 2.12.0 Bumps [babel](https://github.com/python-babel/babel) from 2.11.0 to 2.12.0. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES.rst) - [Commits](https://github.com/python-babel/babel/compare/v2.11.0...v2.12.0) --- updated-dependencies: - dependency-name: babel dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index ae4527f7c7..caf809993c 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -12,7 +12,7 @@ attrs==22.2.0 # via # -r docs-requirements.in # outcome -babel==2.11.0 +babel==2.12.0 # via sphinx certifi==2022.12.7 # via requests @@ -53,8 +53,6 @@ packaging==23.0 # via sphinx pygments==2.14.0 # via sphinx -pytz==2022.7.1 - # via babel requests==2.28.2 # via sphinx sniffio==1.3.0 @@ -86,8 +84,6 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -tomli==2.0.1 - # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 From 9c6c2b07984ee530570a01b3710e729d3f5546b9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 11:07:35 +0000 Subject: [PATCH 1351/1498] Autoformatter changes --- docs-requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs-requirements.txt b/docs-requirements.txt index caf809993c..bfd8b8ed44 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -53,6 +53,8 @@ packaging==23.0 # via sphinx pygments==2.14.0 # via sphinx +pytz==2022.7.1 + # via babel requests==2.28.2 # via sphinx sniffio==1.3.0 @@ -84,6 +86,8 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in +tomli==2.0.1 + # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 From 8139a56c7cd6ed9d54f3b0e375ecef3a79595c48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 11:12:22 +0000 Subject: [PATCH 1352/1498] Bump prompt-toolkit from 3.0.37 to 3.0.38 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.37 to 3.0.38. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.37...3.0.38) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index aa6ae164db..1901a0fe1e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -97,7 +93,7 @@ platformdirs==3.0.0 # pylint pluggy==1.0.0 # via pytest -prompt-toolkit==3.0.37 +prompt-toolkit==3.0.38 # via ipython ptyprocess==0.7.0 # via pexpect @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 36290994c4890d0f42f2c8352c3fc265b9bbcb64 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 11:13:36 +0000 Subject: [PATCH 1353/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 1901a0fe1e..741c68bc81 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 4eac9fc614b67ace845803649431144a9843c196 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 11:05:47 +0000 Subject: [PATCH 1354/1498] Bump babel from 2.12.0 to 2.12.1 Bumps [babel](https://github.com/python-babel/babel) from 2.12.0 to 2.12.1. - [Release notes](https://github.com/python-babel/babel/releases) - [Changelog](https://github.com/python-babel/babel/blob/master/CHANGES.rst) - [Commits](https://github.com/python-babel/babel/compare/v2.12.0...v2.12.1) --- updated-dependencies: - dependency-name: babel dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs-requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index bfd8b8ed44..f42f64df55 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -12,7 +12,7 @@ attrs==22.2.0 # via # -r docs-requirements.in # outcome -babel==2.12.0 +babel==2.12.1 # via sphinx certifi==2022.12.7 # via requests @@ -53,8 +53,6 @@ packaging==23.0 # via sphinx pygments==2.14.0 # via sphinx -pytz==2022.7.1 - # via babel requests==2.28.2 # via sphinx sniffio==1.3.0 @@ -86,8 +84,6 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in -tomli==2.0.1 - # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 From 7071814bdbf990fadc7701dc33fca8a5b91d8651 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 11:07:16 +0000 Subject: [PATCH 1355/1498] Autoformatter changes --- docs-requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs-requirements.txt b/docs-requirements.txt index f42f64df55..660a2d94b4 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -53,6 +53,8 @@ packaging==23.0 # via sphinx pygments==2.14.0 # via sphinx +pytz==2022.7.1 + # via babel requests==2.28.2 # via sphinx sniffio==1.3.0 @@ -84,6 +86,8 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-trio==1.1.2 # via -r docs-requirements.in +tomli==2.0.1 + # via towncrier towncrier==22.12.0 # via -r docs-requirements.in urllib3==1.26.14 From 428a257f9f84c0667fee4c8852dee0e56768e8f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 12:13:46 +0000 Subject: [PATCH 1356/1498] Bump pip-tools from 6.12.2 to 6.12.3 Bumps [pip-tools](https://github.com/jazzband/pip-tools) from 6.12.2 to 6.12.3. - [Release notes](https://github.com/jazzband/pip-tools/releases) - [Changelog](https://github.com/jazzband/pip-tools/blob/main/CHANGELOG.md) - [Commits](https://github.com/jazzband/pip-tools/compare/6.12.2...6.12.3) --- updated-dependencies: - dependency-name: pip-tools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 741c68bc81..667762f71a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -89,7 +85,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.12.2 +pip-tools==6.12.3 # via -r test-requirements.in platformdirs==3.0.0 # via @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 74061550b940e202a9c8ec554a2e5b99930fd3c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 12:18:24 +0000 Subject: [PATCH 1357/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 667762f71a..d76447cb39 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From dac2f8cbdc2fef164438c0a1810fe47fab195e77 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 10:21:19 +0000 Subject: [PATCH 1358/1498] Bump cryptography from 39.0.1 to 39.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 39.0.1 to 39.0.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/39.0.1...39.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index d76447cb39..e9577238c2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -29,7 +29,7 @@ click==8.1.3 # pip-tools coverage[toml]==6.4.1 # via pytest-cov -cryptography==39.0.1 +cryptography==39.0.2 # via # -r test-requirements.in # pyopenssl @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From cb237764897c160e5b09f5bd07d7ccc4a3e4d19b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 10:22:43 +0000 Subject: [PATCH 1359/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index e9577238c2..ce448d39a1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 1862393c1f75840c5e9376a7cbe4aa9642f4c4de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:10:53 +0000 Subject: [PATCH 1360/1498] Bump pylint from 2.16.2 to 2.16.3 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.16.2 to 2.16.3. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Commits](https://github.com/PyCQA/pylint/compare/v2.16.2...v2.16.3) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index d76447cb39..71a92ebdac 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -109,7 +105,7 @@ pyflakes==2.4.0 # via flake8 pygments==2.14.0 # via ipython -pylint==2.16.2 +pylint==2.16.3 # via -r test-requirements.in pyopenssl==23.0.0 # via -r test-requirements.in @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From abb7c73ccd47a931dd0fd6b244d8bbf1c940d130 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:12:21 +0000 Subject: [PATCH 1361/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 71a92ebdac..017f322ddf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 21f08cf0754207a133749acf768719c9b4288f3a Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 4 Mar 2023 20:22:09 -0500 Subject: [PATCH 1362/1498] bump checkout version to revive CI --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55cd0bbf4a..8ad5c623c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup python uses: actions/setup-python@v2 with: @@ -106,7 +106,7 @@ jobs: }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.ref }} - name: Setup python @@ -153,7 +153,7 @@ jobs: extra_name: ', pypy 3.8 nightly' steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup python uses: actions/setup-python@v2 with: From bf5f717b4722bbe8c25770dfff33994955cb55cc Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 4 Mar 2023 20:22:59 -0500 Subject: [PATCH 1363/1498] remove pr head ref injection to revive CI --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ad5c623c7..eb28b14066 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,8 +107,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.ref }} - name: Setup python uses: actions/setup-python@v2 if: "!endsWith(matrix.python, '-dev')" From 8f90c60261526bc58cdd08a4fd1bd9484ad5fac4 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 4 Mar 2023 22:00:31 -0500 Subject: [PATCH 1364/1498] split highly permissioned dependabot autoformatter into separate job --- .github/workflows/ci.yml | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb28b14066..d875aadf35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,12 +71,6 @@ jobs: name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' timeout-minutes: 10 runs-on: 'ubuntu-latest' - # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#changing-github_token-permissions - permissions: - pull-requests: write - issues: write - repository-projects: write - contents: write strategy: fail-fast: false matrix: @@ -126,9 +120,33 @@ jobs: CHECK_FORMATTING: '${{ matrix.check_formatting }}' # Should match 'name:' up above JOB_NAME: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})' + + autofmt: + name: Autoformat dependabot PR + timeout-minutes: 10 + if: github.actor == 'dependabot[bot]' + runs-on: 'ubuntu-latest' + # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#changing-github_token-permissions + permissions: + pull-requests: write + issues: write + repository-projects: write + contents: write + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Check formatting + run: | + python -m pip install -r test-requirements.txt + source check.sh - name: Commit autoformatter changes - continue-on-error: true - if: failure() && matrix.check_formatting == '1' && github.actor == 'dependabot[bot]' + if: failure() run: | black setup.py trio git config user.name 'github-actions[bot]' From 06db0cb92490efb82046b03118db5e65a82c2c15 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sat, 4 Mar 2023 22:16:13 -0500 Subject: [PATCH 1365/1498] run instead of source Co-authored-by: EXPLOSION --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d875aadf35..145ee55b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,7 +144,7 @@ jobs: - name: Check formatting run: | python -m pip install -r test-requirements.txt - source check.sh + ./check.sh - name: Commit autoformatter changes if: failure() run: | From b65b44ccc7318a898eb22853f982b40bd66c3e44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 03:27:56 +0000 Subject: [PATCH 1366/1498] Bump platformdirs from 3.0.0 to 3.1.0 Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/3.0.0...3.1.0) --- updated-dependencies: - dependency-name: platformdirs dependency-type: indirect update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index f49e777148..88d0b490ff 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -91,7 +87,7 @@ pickleshare==0.7.5 # via ipython pip-tools==6.12.3 # via -r test-requirements.in -platformdirs==3.0.0 +platformdirs==3.1.0 # via # black # pylint @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From f0f72adef12232a5b866bc54c7c86c169c5fa44e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 03:28:56 +0000 Subject: [PATCH 1367/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 88d0b490ff..f2f7e37778 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 9eae00d1f0b16244ea10f052f2361a61b76e331b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 03:44:13 +0000 Subject: [PATCH 1368/1498] Bump pytest from 7.2.1 to 7.2.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.2.1 to 7.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.2.1...7.2.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test-requirements.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index f2f7e37778..1f14f8bc35 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,10 +39,6 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint -exceptiongroup==1.1.0 ; python_version < "3.11" - # via - # -r test-requirements.in - # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -115,7 +111,7 @@ pyopenssl==23.0.0 # via -r test-requirements.in pyproject-hooks==1.0.0 # via build -pytest==7.2.1 +pytest==7.2.2 # via # -r test-requirements.in # pytest-cov @@ -125,14 +121,6 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pylint - # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -146,10 +134,7 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in - # astroid - # black # mypy - # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From 5ad1b8bf61f60eafac93422574f0fa5d471cc7d8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 03:48:05 +0000 Subject: [PATCH 1369/1498] Autoformatter changes --- test-requirements.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 1f14f8bc35..b19d24334d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -39,6 +39,10 @@ decorator==5.1.1 # via ipython dill==0.3.6 # via pylint +exceptiongroup==1.1.0 ; python_version < "3.11" + # via + # -r test-requirements.in + # pytest flake8==4.0.1 # via -r test-requirements.in idna==3.4 @@ -121,6 +125,14 @@ sniffio==1.3.0 # via -r test-requirements.in sortedcontainers==2.4.0 # via -r test-requirements.in +tomli==2.0.1 + # via + # black + # build + # coverage + # mypy + # pylint + # pytest tomlkit==0.11.6 # via pylint traitlets==5.9.0 @@ -134,7 +146,10 @@ types-pyopenssl==23.0.0.4 ; implementation_name == "cpython" typing-extensions==4.5.0 ; implementation_name == "cpython" # via # -r test-requirements.in + # astroid + # black # mypy + # pylint wcwidth==0.2.6 # via prompt-toolkit wheel==0.38.4 From f860ff77596fe5633196c7f238d51a6518182ec5 Mon Sep 17 00:00:00 2001 From: richardsheridan Date: Sun, 5 Mar 2023 00:02:05 -0500 Subject: [PATCH 1370/1498] enable sphinx 6 --- docs/source/_templates/layout.html | 9 +++++++-- docs/source/conf.py | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html index d3b0ca89fd..dbebf5c2ae 100644 --- a/docs/source/_templates/layout.html +++ b/docs/source/_templates/layout.html @@ -4,8 +4,13 @@ {% extends "!layout.html" %} {% block sidebartitle %} -