From 4f083224b54db75b4afd33ee7647bd81d17e1db6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 11 Jun 2024 02:28:14 +0800 Subject: [PATCH 1/5] Lock gen_send --- Objects/genobject.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 92cd8c61e7e9ca..8889526b837e34 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -14,6 +14,7 @@ #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_critical_section.h" #include "pystats.h" @@ -159,7 +160,7 @@ gen_dealloc(PyGenObject *gen) } static PySendResult -gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, +gen_send_ex2_unlocked(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc, int closing) { PyThreadState *tstate = _PyThreadState_GET(); @@ -257,6 +258,17 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, return result ? PYGEN_RETURN : PYGEN_ERROR; } +static PySendResult +gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, + int exc, int closing) +{ + PySendResult res; + Py_BEGIN_CRITICAL_SECTION(gen); + res = gen_send_ex2_unlocked(gen, arg, presult, exc, closing); + Py_END_CRITICAL_SECTION(); + return res; +} + static PySendResult PyGen_am_send(PyGenObject *gen, PyObject *arg, PyObject **result) { From ab453ea60e7905b6f9f85ae019ca7b535558b450 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 11 Jun 2024 02:29:11 +0800 Subject: [PATCH 2/5] Add comment --- Objects/genobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 8889526b837e34..545b18b317c1aa 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -14,7 +14,7 @@ #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_critical_section.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION #include "pystats.h" From 3d23adaff6e325df53cefbd870c8b03103814890 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 11 Jun 2024 02:38:22 +0800 Subject: [PATCH 3/5] Create test_generators.py --- .../test_free_threading/test_generators.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Lib/test/test_free_threading/test_generators.py diff --git a/Lib/test/test_free_threading/test_generators.py b/Lib/test/test_free_threading/test_generators.py new file mode 100644 index 00000000000000..999cb189be15b4 --- /dev/null +++ b/Lib/test/test_free_threading/test_generators.py @@ -0,0 +1,23 @@ +import unittest +import concurrent.futures + +from unittest import TestCase + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestGen(TestCase): + def test_generators_basic(self): + def gen(): + for _ in range(5000): + yield + + + it = gen() + with concurrent.futures.ThreadPoolExecutor() as executor: + for _ in range(5000): + executor.submit(lambda: next(it)) + + +if __name__ == "__main__": + unittest.main() From bfe9eade1946694128e0115a8dec2c6ac9db4dbe Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:40:08 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst new file mode 100644 index 00000000000000..c2107256f9883c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst @@ -0,0 +1 @@ +Fix generator ``send`` crashing on free-threaded builds. Thanks to ``rostan-t`` for the reproducer. From 24e34e31c5fa1f9de5fd6a56edefd2e5a88f8e53 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 13 Jul 2024 23:31:00 +0800 Subject: [PATCH 5/5] lock SEND as well --- Python/bytecodes.c | 5 ++++- Python/generated_cases.c.h | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 84241c64ffae88..a12c897126b470 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1117,7 +1117,9 @@ dummy_func( ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyInterpreterFrame *gen_frame; + Py_BEGIN_CRITICAL_SECTION(gen); + gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; @@ -1125,6 +1127,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + Py_END_CRITICAL_SECTION(); DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 61057221291c0a..f480d334bafd9e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5768,7 +5768,9 @@ ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyInterpreterFrame *gen_frame; + Py_BEGIN_CRITICAL_SECTION(gen); + gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; @@ -5776,6 +5778,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + Py_END_CRITICAL_SECTION(); DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {