From cc387f8a39dc6dae3cb9c203c0c808c768eef817 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 9 May 2025 18:55:26 -0600 Subject: [PATCH 01/11] Add _PyXI_session_result. --- Include/internal/pycore_crossinterp.h | 12 +- Modules/_interpretersmodule.c | 85 ++++++-------- Python/crossinterp.c | 152 +++++++++++++++++--------- 3 files changed, 140 insertions(+), 109 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 1272927413868b..874cbbe6e0272e 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -350,17 +350,19 @@ typedef struct xi_session _PyXI_session; PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void); PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); +typedef struct { + PyObject *excinfo; +} _PyXI_session_result; + PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, PyInterpreterState *interp, - PyObject *nsupdates); -PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); + PyObject *nsupdates, + _PyXI_session_result *); +PyAPI_FUNC(int) _PyXI_Exit(_PyXI_session *, _PyXI_session_result *); PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *); -PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); -PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); - /*************/ /* other API */ diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 376517ab92360f..66784c78c6fcf6 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -441,7 +441,7 @@ _run_script(_PyXIData_t *script, PyObject *ns) static int _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, _PyXIData_t *script, PyObject *shareables, - PyObject **p_excinfo) + _PyXI_session_result *result) { assert(!_PyErr_Occurred(tstate)); _PyXI_session *session = _PyXI_NewSession(); @@ -450,19 +450,9 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, } // Prep and switch interpreters. - if (_PyXI_Enter(session, interp, shareables) < 0) { - if (_PyErr_Occurred(tstate)) { - // If an error occured at this step, it means that interp - // was not prepared and switched. - _PyXI_FreeSession(session); - return -1; - } - // Now, apply the error from another interpreter: - PyObject *excinfo = _PyXI_ApplyCapturedException(session); - if (excinfo != NULL) { - *p_excinfo = excinfo; - } - assert(PyErr_Occurred()); + if (_PyXI_Enter(session, interp, shareables, result) < 0) { + // If an error occured at this step, it means that interp + // was not prepared and switched. _PyXI_FreeSession(session); return -1; } @@ -477,20 +467,7 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, finally: // Clean up and switch back. - _PyXI_Exit(session); - - // Propagate any exception out to the caller. - assert(!PyErr_Occurred()); - if (res < 0) { - PyObject *excinfo = _PyXI_ApplyCapturedException(session); - if (excinfo != NULL) { - *p_excinfo = excinfo; - } - } - else { - assert(!_PyXI_HasCapturedException(session)); - } - + (void)_PyXI_Exit(session, result); _PyXI_FreeSession(session); return res; } @@ -842,21 +819,23 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) } // Prep and switch interpreters, including apply the updates. - if (_PyXI_Enter(session, interp, updates) < 0) { - if (!PyErr_Occurred()) { - _PyXI_ApplyCapturedException(session); - assert(PyErr_Occurred()); - } - else { - assert(!_PyXI_HasCapturedException(session)); - } + if (_PyXI_Enter(session, interp, updates, NULL) < 0) { _PyXI_FreeSession(session); return NULL; } // Clean up and switch back. - _PyXI_Exit(session); + assert(!PyErr_Occurred()); + int res = _PyXI_Exit(session, NULL); _PyXI_FreeSession(session); + assert(res == 0); + if (res < 0) { + // unreachable + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "unresolved error"); + } + return NULL; + } Py_RETURN_NONE; } @@ -918,12 +897,12 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + _PyXI_session_result result = {0}; + int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &result); _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); + return result.excinfo; } Py_RETURN_NONE; #undef FUNCNAME @@ -981,12 +960,12 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + _PyXI_session_result result = {0}; + int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &result); _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); + return result.excinfo; } Py_RETURN_NONE; #undef FUNCNAME @@ -1043,12 +1022,12 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + _PyXI_session_result result = {0}; + int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &result); _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); + return result.excinfo; } Py_RETURN_NONE; #undef FUNCNAME @@ -1104,12 +1083,12 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &excinfo); + _PyXI_session_result result = {0}; + int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &result); _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); + return result.excinfo; } Py_RETURN_NONE; #undef FUNCNAME diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 65ccab32daf730..534e8eaeb79aa5 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -2146,7 +2146,9 @@ _fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, _PyXI_session *session) assert(ns->numvalues == 0); for (Py_ssize_t i=0; i < ns->maxitems; i++) { if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj) < 0) { - _propagate_not_shareable_error(session); + if (session != NULL) { + _propagate_not_shareable_error(session); + } // Clear out the ones we set so far. for (Py_ssize_t j=0; j < i; j++) { _sharednsitem_clear_value(&ns->items[j]); @@ -2212,6 +2214,18 @@ _apply_sharedns(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) /* switched-interpreter sessions */ /*********************************/ +struct xi_session_error { + // This is set if the interpreter is entered and raised an exception + // that needs to be handled in some special way during exit. + _PyXI_errcode *override; + // This is set if exit captured an exception to propagate. + _PyXI_error *info; + + // -- pre-allocated memory -- + _PyXI_error _info; + _PyXI_errcode _override; +}; + struct xi_session { #define SESSION_UNUSED 0 #define SESSION_ACTIVE 1 @@ -2240,15 +2254,7 @@ struct xi_session { // beginning of the session as a convenience. PyObject *main_ns; - // This is set if the interpreter is entered and raised an exception - // that needs to be handled in some special way during exit. - _PyXI_errcode *error_override; - // This is set if exit captured an exception to propagate. - _PyXI_error *error; - - // -- pre-allocated memory -- - _PyXI_error _error; - _PyXI_errcode _error_override; + struct xi_session_error error; }; @@ -2277,6 +2283,23 @@ _session_is_active(_PyXI_session *session) return session->status == SESSION_ACTIVE; } +static int +_session_pop_error(_PyXI_session *session, struct xi_session_error *err) +{ + if (session->error.info == NULL) { + assert(session->error.override == NULL); + *err = (struct xi_session_error){0}; + return 0; + } + *err = session->error; + err->info = &err->_info; + if (err->override != NULL) { + err->override = &err->_override; + } + session->error = (struct xi_session_error){0}; + return 1; +} + static int _ensure_main_ns(_PyXI_session *); static inline void _session_set_error(_PyXI_session *, _PyXI_errcode); static void _capture_current_exception(_PyXI_session *); @@ -2296,9 +2319,9 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp) assert(!session->running); assert(session->main_ns == NULL); // Set elsewhere and cleared in _capture_current_exception(). - assert(session->error_override == NULL); - // Set elsewhere and cleared in _PyXI_ApplyCapturedException(). - assert(session->error == NULL); + assert(session->error.override == NULL); + // Set elsewhere and cleared in _PyXI_Exit(). + assert(session->error.info == NULL); // Switch to interpreter. PyThreadState *tstate = PyThreadState_Get(); @@ -2351,21 +2374,17 @@ _exit_session(_PyXI_session *session) assert(!session->own_init_tstate); } + assert(session->error.info == NULL); + assert(session->error.override == _PyXI_ERR_NO_ERROR); + // For now the error data persists past the exit. - *session = (_PyXI_session){ - .error_override = session->error_override, - .error = session->error, - ._error = session->_error, - ._error_override = session->_error_override, - }; + *session = (_PyXI_session){0}; } static void _propagate_not_shareable_error(_PyXI_session *session) { - if (session == NULL) { - return; - } + assert(session != NULL); PyThreadState *tstate = PyThreadState_Get(); PyObject *exctype = get_notshareableerror_type(tstate); if (exctype == NULL) { @@ -2379,26 +2398,10 @@ _propagate_not_shareable_error(_PyXI_session *session) } } -PyObject * -_PyXI_ApplyCapturedException(_PyXI_session *session) -{ - assert(!PyErr_Occurred()); - assert(session->error != NULL); - PyObject *res = _PyXI_ApplyError(session->error); - assert((res == NULL) != (PyErr_Occurred() == NULL)); - session->error = NULL; - return res; -} - -int -_PyXI_HasCapturedException(_PyXI_session *session) -{ - return session->error != NULL; -} - int _PyXI_Enter(_PyXI_session *session, - PyInterpreterState *interp, PyObject *nsupdates) + PyInterpreterState *interp, PyObject *nsupdates, + _PyXI_session_result *result) { // Convert the attrs for cross-interpreter use. _PyXI_namespace *sharedns = NULL; @@ -2413,7 +2416,7 @@ _PyXI_Enter(_PyXI_session *session, return -1; } if (_fill_sharedns(sharedns, nsupdates, NULL) < 0) { - assert(session->error == NULL); + assert(session->error.info == NULL); _destroy_sharedns(sharedns); return -1; } @@ -2453,19 +2456,66 @@ _PyXI_Enter(_PyXI_session *session, error: // We want to propagate all exceptions here directly (best effort). + assert(errcode != _PyXI_ERR_NO_ERROR); _session_set_error(session, errcode); + assert(!PyErr_Occurred()); + + struct xi_session_error err; + (void)_session_pop_error(session, &err); _exit_session(session); + if (sharedns != NULL) { _destroy_sharedns(sharedns); } + + // Apply the error from the other interpreter. + PyObject *excinfo = _PyXI_ApplyError(err.info); + if (excinfo != NULL) { + if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Enter(): uncaught exception discarded"); +#endif + } + } + assert(PyErr_Occurred()); + return -1; } -void -_PyXI_Exit(_PyXI_session *session) +int +_PyXI_Exit(_PyXI_session *session, _PyXI_session_result *result) { _capture_current_exception(session); + assert(!PyErr_Occurred()); + + struct xi_session_error err; + (void)_session_pop_error(session, &err); _exit_session(session); + + if (err.info == NULL) { + return 0; + } + + // Apply the error from the other interpreter. + PyObject *excinfo = _PyXI_ApplyError(err.info); + if (excinfo == NULL) { + assert(PyErr_Occurred()); + if (result != NULL) { + *result = (_PyXI_session_result){0}; + } + } + else if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded"); +#endif + } + return -1; } @@ -2474,15 +2524,15 @@ _PyXI_Exit(_PyXI_session *session) static void _capture_current_exception(_PyXI_session *session) { - assert(session->error == NULL); + assert(session->error.info == NULL); if (!PyErr_Occurred()) { - assert(session->error_override == NULL); + assert(session->error.override == NULL); return; } // Handle the exception override. - _PyXI_errcode *override = session->error_override; - session->error_override = NULL; + _PyXI_errcode *override = session->error.override; + session->error.override = NULL; _PyXI_errcode errcode = override != NULL ? *override : _PyXI_ERR_UNCAUGHT_EXCEPTION; @@ -2505,7 +2555,7 @@ _capture_current_exception(_PyXI_session *session) } // Capture the exception. - _PyXI_error *err = &session->_error; + _PyXI_error *err = &session->error._info; *err = (_PyXI_error){ .interp = session->init_tstate->interp, }; @@ -2532,7 +2582,7 @@ _capture_current_exception(_PyXI_session *session) // Finished! assert(!PyErr_Occurred()); - session->error = err; + session->error.info = err; } static inline void @@ -2541,8 +2591,8 @@ _session_set_error(_PyXI_session *session, _PyXI_errcode errcode) assert(_session_is_active(session)); assert(PyErr_Occurred()); if (errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION) { - session->_error_override = errcode; - session->error_override = &session->_error_override; + session->error._override = errcode; + session->error.override = &session->error._override; } _capture_current_exception(session); } From b906d4acf49b1a44e40d50b7e008b792708e4cea Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 8 May 2025 16:16:51 -0600 Subject: [PATCH 02/11] Add _PyXI_Preserve(). --- Include/internal/pycore_crossinterp.h | 4 + Python/crossinterp.c | 143 ++++++++++++++++++++++++-- 2 files changed, 137 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 874cbbe6e0272e..d8e87174988b6d 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -351,6 +351,7 @@ PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void); PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); typedef struct { + PyObject *preserved; PyObject *excinfo; } _PyXI_session_result; @@ -363,6 +364,9 @@ PyAPI_FUNC(int) _PyXI_Exit(_PyXI_session *, _PyXI_session_result *); PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *); +PyAPI_FUNC(int) _PyXI_Preserve(_PyXI_session *, const char *, PyObject *); +PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *); + /*************/ /* other API */ diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 534e8eaeb79aa5..360d9bd08601c9 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1830,7 +1830,8 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) } static int -_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) +_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value, + xidata_fallback_t fallback) { assert(_sharednsitem_is_initialized(item)); assert(item->xidata == NULL); @@ -1839,8 +1840,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) return -1; } PyThreadState *tstate = PyThreadState_Get(); - // XXX Use _PyObject_GetXIDataWithFallback()? - if (_PyObject_GetXIDataNoFallback(tstate, value, item->xidata) != 0) { + if (_PyObject_GetXIData(tstate, value, fallback, item->xidata) < 0) { PyMem_RawFree(item->xidata); item->xidata = NULL; // The caller may want to propagate PyExc_NotShareableError @@ -1872,7 +1872,8 @@ _sharednsitem_clear(_PyXI_namespace_item *item) } static int -_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) +_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns, + xidata_fallback_t fallback) { assert(item->name != NULL); assert(item->xidata == NULL); @@ -1884,7 +1885,7 @@ _sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) // When applied, this item will be set to the default (or fail). return 0; } - if (_sharednsitem_set_value(item, value) < 0) { + if (_sharednsitem_set_value(item, value, fallback) < 0) { return -1; } return 0; @@ -2138,14 +2139,15 @@ _create_sharedns(PyObject *names) static void _propagate_not_shareable_error(_PyXI_session *); static int -_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, _PyXI_session *session) +_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, + xidata_fallback_t fallback, _PyXI_session *session) { // All items are expected to be shareable. assert(_sharedns_check_counts(ns)); assert(ns->numnames == ns->maxitems); assert(ns->numvalues == 0); for (Py_ssize_t i=0; i < ns->maxitems; i++) { - if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj) < 0) { + if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj, fallback) < 0) { if (session != NULL) { _propagate_not_shareable_error(session); } @@ -2254,10 +2256,14 @@ struct xi_session { // beginning of the session as a convenience. PyObject *main_ns; + // This is a dict of objects that will be available (via sharing) + // once the session exits. Do not access this directly; use + // _PyXI_Preserve() and _PyXI_GetPreserved() instead; + PyObject *_preserved; + struct xi_session_error error; }; - _PyXI_session * _PyXI_NewSession(void) { @@ -2350,14 +2356,16 @@ _exit_session(_PyXI_session *session) PyThreadState *tstate = session->init_tstate; assert(tstate != NULL); assert(PyThreadState_Get() == tstate); + assert(!_PyErr_Occurred(tstate)); // Release any of the entered interpreters resources. Py_CLEAR(session->main_ns); + Py_CLEAR(session->_preserved); // Ensure this thread no longer owns __main__. if (session->running) { _PyInterpreterState_SetNotRunningMain(tstate->interp); - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); session->running = 0; } @@ -2415,7 +2423,9 @@ _PyXI_Enter(_PyXI_session *session, if (sharedns == NULL) { return -1; } - if (_fill_sharedns(sharedns, nsupdates, NULL) < 0) { + // For now we limit it to shareable objects. + xidata_fallback_t fallback = _PyXIDATA_XIDATA_ONLY; + if (_fill_sharedns(sharedns, nsupdates, fallback, NULL) < 0) { assert(session->error.info == NULL); _destroy_sharedns(sharedns); return -1; @@ -2485,12 +2495,27 @@ _PyXI_Enter(_PyXI_session *session, return -1; } +static PyObject * _capture_preserved(_PyXI_session *); + int _PyXI_Exit(_PyXI_session *session, _PyXI_session_result *result) { _capture_current_exception(session); assert(!PyErr_Occurred()); + if (result != NULL) { + result->preserved = _capture_preserved(session); + if (result->preserved == NULL && PyErr_Occurred()) { + if (session->error.info != NULL) { + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + } + else { + _capture_current_exception(session); + } + } + } + struct xi_session_error err; (void)_session_pop_error(session, &err); _exit_session(session); @@ -2634,6 +2659,104 @@ _PyXI_GetMainNamespace(_PyXI_session *session) } +static PyObject * +_capture_preserved(_PyXI_session *session) +{ + assert(_PyThreadState_GET() == session->init_tstate); // active session + if (session->init_tstate == session->prev_tstate) { + // didn't switch + return Py_XNewRef(session->_preserved); + } + + _PyXI_namespace *preserved = NULL; + if (session->_preserved != NULL) { + Py_ssize_t len = PyDict_Size(session->_preserved); + if (len < 0) { + return NULL; + } + if (len > 0) { + preserved = _create_sharedns(session->_preserved); + if (preserved == NULL) { + return NULL; + } + if (_fill_sharedns(preserved, session->_preserved, + _PyXIDATA_FULL_FALLBACK, NULL) < 0) + { + assert(session->error.info == NULL); + _destroy_sharedns(preserved); + return NULL; + } + } + Py_CLEAR(session->_preserved); + } + if (preserved == NULL) { + return NULL; + } + + // We need to switch back to the original interpreter long enough + // to restore the preservd objects. + (void)PyThreadState_Swap(session->prev_tstate); + + PyObject *ns = PyDict_New(); + if (ns == NULL) { + goto finally; + } + if (_apply_sharedns(preserved, ns, NULL) < 0) { + Py_CLEAR(ns); + goto finally; + } + +finally: + // Swap back into the session. + (void)PyThreadState_Swap(session->init_tstate); + assert(preserved != NULL); + _destroy_sharedns(preserved); + return ns; +} + +static void +set_exc_with_cause(PyObject *exctype, const char *msg) +{ + PyObject *cause = PyErr_GetRaisedException(); + PyErr_SetString(exctype, msg); + PyObject *exc = PyErr_GetRaisedException(); + PyException_SetCause(exc, cause); + PyErr_SetRaisedException(exc); +} + +int +_PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value) +{ + if (session->init_tstate == NULL) { + PyErr_SetString(PyExc_RuntimeError, "session not active"); + return -1; + } + if (session->_preserved == NULL) { + session->_preserved = PyDict_New(); + if (session->_preserved == NULL) { + set_exc_with_cause(PyExc_RuntimeError, + "failed to initialize preserved objects"); + return -1; + } + } + if (PyDict_SetItemString(session->_preserved, name, value) < 0) { + set_exc_with_cause(PyExc_RuntimeError, "failed to preserve object"); + return -1; + } + return 0; +} + +PyObject * +_PyXI_GetPreserved(_PyXI_session_result *result, const char *name) +{ + PyObject *value = NULL; + if (result->preserved != NULL) { + (void)PyDict_GetItemStringRef(result->preserved, name, &value); + } + return value; +} + + /*********************/ /* runtime lifecycle */ /*********************/ From 08307a0c29d4f8ed71c2c71afc1d5c3bb9a75c3b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 24 Apr 2025 22:12:17 -0600 Subject: [PATCH 03/11] Support full calls in interp_call(). --- Lib/test/_code_definitions.py | 20 + Lib/test/support/interpreters/__init__.py | 21 +- Lib/test/test_code.py | 48 ++- Lib/test/test_interpreters/test_api.py | 485 +++++++++++++++++++--- Modules/_interpretersmodule.c | 336 +++++++++++++-- 5 files changed, 787 insertions(+), 123 deletions(-) diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index 733a15b25f6894..274beb65a6d0f4 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -57,6 +57,15 @@ def spam_with_globals_and_builtins(): print(res) +def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + +def spam_full_args_with_defaults(a=-1, b=-2, /, c=-3, d=-4, *args, + e=-5, f=-6, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + def spam_args_attrs_and_builtins(a, b, /, c, d, *args, e, f, **kwargs): if args.__len__() > 2: return None @@ -67,6 +76,10 @@ def spam_returns_arg(x): return x +def spam_raises(): + raise Exception('spam!') + + def spam_with_inner_not_closure(): def eggs(): pass @@ -177,8 +190,11 @@ def ham_C_closure(z): spam_minimal, spam_with_builtins, spam_with_globals_and_builtins, + spam_full_args, + spam_full_args_with_defaults, spam_args_attrs_and_builtins, spam_returns_arg, + spam_raises, spam_with_inner_not_closure, spam_with_inner_closure, spam_annotated, @@ -219,8 +235,10 @@ def ham_C_closure(z): spam, spam_minimal, spam_with_builtins, + spam_full_args, spam_args_attrs_and_builtins, spam_returns_arg, + spam_raises, spam_annotated, spam_with_inner_not_closure, spam_with_inner_closure, @@ -238,6 +256,7 @@ def ham_C_closure(z): STATELESS_CODE = [ *STATELESS_FUNCTIONS, script_with_globals, + spam_full_args_with_defaults, spam_with_globals_and_builtins, spam_full, ] @@ -248,6 +267,7 @@ def ham_C_closure(z): script_with_explicit_empty_return, spam_minimal, spam_with_builtins, + spam_raises, spam_with_inner_not_closure, spam_with_inner_closure, ] diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index e067f259364d2a..93b367452a4f9f 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -226,7 +226,13 @@ def exec(self, code, /): if excinfo is not None: raise ExecutionFailed(excinfo) - def call(self, callable, /): + def _call(self, callable, args, kwargs): + res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + return res + + def call(self, callable, /, *args, **kwargs): """Call the object in the interpreter with given args/kwargs. Only functions that take no arguments and have no closure @@ -239,20 +245,13 @@ def call(self, callable, /): and an ExecutionFailed exception is raised, much like what happens with Interpreter.exec(). """ - # XXX Support args and kwargs. - # XXX Support arbitrary callables. - # XXX Support returning the return value (e.g. via pickle). - excinfo = _interpreters.call(self._id, callable, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) + return self._call(callable, args, kwargs) - def call_in_thread(self, callable, /): + def call_in_thread(self, callable, /, *args, **kwargs): """Return a new thread that calls the object in the interpreter. The return value and any raised exception are discarded. """ - def task(): - self.call(callable) - t = threading.Thread(target=task) + t = threading.Thread(target=self._call, args=(callable, args, kwargs)) t.start() return t diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 32cf8aacaf6b72..9fc2b047bef719 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -701,6 +701,26 @@ def test_local_kinds(self): 'checks': CO_FAST_LOCAL, 'res': CO_FAST_LOCAL, }, + defs.spam_full_args: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, + defs.spam_full_args_with_defaults: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, defs.spam_args_attrs_and_builtins: { 'a': POSONLY, 'b': POSONLY, @@ -714,6 +734,7 @@ def test_local_kinds(self): defs.spam_returns_arg: { 'x': POSORKW, }, + defs.spam_raises: {}, defs.spam_with_inner_not_closure: { 'eggs': CO_FAST_LOCAL, }, @@ -934,6 +955,20 @@ def new_var_counts(*, purelocals=5, globalvars=6, ), + defs.spam_full_args: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), + defs.spam_full_args_with_defaults: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), defs.spam_args_attrs_and_builtins: new_var_counts( posonly=2, posorkw=2, @@ -945,6 +980,9 @@ def new_var_counts(*, defs.spam_returns_arg: new_var_counts( posorkw=1, ), + defs.spam_raises: new_var_counts( + globalvars=1, + ), defs.spam_with_inner_not_closure: new_var_counts( purelocals=1, ), @@ -1097,10 +1135,16 @@ def new_var_counts(*, def test_stateless(self): self.maxDiff = None + STATELESS_FUNCTIONS = [ + *defs.STATELESS_FUNCTIONS, + # stateless with defaults + defs.spam_full_args_with_defaults, + ] + for func in defs.STATELESS_CODE: with self.subTest((func, '(code)')): _testinternalcapi.verify_stateless_code(func.__code__) - for func in defs.STATELESS_FUNCTIONS: + for func in STATELESS_FUNCTIONS: with self.subTest((func, '(func)')): _testinternalcapi.verify_stateless_code(func) @@ -1110,7 +1154,7 @@ def test_stateless(self): with self.assertRaises(Exception): _testinternalcapi.verify_stateless_code(func.__code__) - if func not in defs.STATELESS_FUNCTIONS: + if func not in STATELESS_FUNCTIONS: with self.subTest((func, '(func)')): with self.assertRaises(Exception): _testinternalcapi.verify_stateless_code(func) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 165949167ceba8..7133f4605ec753 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,17 +1,22 @@ +import contextlib import os import pickle +import sys from textwrap import dedent import threading import types import unittest from test import support +from test.support import os_helper +from test.support import script_helper from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') from test.support import Py_GIL_DISABLED from test.support import interpreters from test.support import force_not_colorized +import test._crossinterp_definitions as defs from test.support.interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) @@ -29,6 +34,29 @@ WHENCE_STR_STDLIB = '_interpreters module' +def is_pickleable(obj): + try: + pickle.dumps(obj) + except Exception: + return False + return True + + +@contextlib.contextmanager +def defined_in___main__(name, script, *, remove=False): + import __main__ as mainmod + mainns = vars(mainmod) + assert name not in mainns + exec(script, mainns, mainns) + if remove: + yield mainns.pop(name) + else: + try: + yield mainns[name] + finally: + mainns.pop(name, None) + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -890,24 +918,26 @@ def test_created_with_capi(self): # Interpreter.exec() behavior. -def call_func_noop(): - pass +call_func_noop = defs.spam_minimal +call_func_ident = defs.spam_returns_arg +call_func_failure = defs.spam_raises def call_func_return_shareable(): return (1, None) -def call_func_return_not_shareable(): - return [1, 2, 3] +def call_func_return_stateless_func(): + return (lambda x: x) -def call_func_failure(): - raise Exception('spam!') +def call_func_return_pickleable(): + return [1, 2, 3] -def call_func_ident(value): - return value +def call_func_return_unpickleable(): + x = 42 + return (lambda: x) def get_call_func_closure(value): @@ -916,6 +946,11 @@ def call_func_closure(): return call_func_closure +def call_func_exec_wrapper(script, ns): + res = exec(script, ns, ns) + return res, ns, id(ns) + + class Spam: @staticmethod @@ -1012,86 +1047,375 @@ class TestInterpreterCall(TestBase): # - preserves info (e.g. SyntaxError) # - matching error display - def test_call(self): + @contextlib.contextmanager + def assert_fails(self, expected): + with self.assertRaises(ExecutionFailed) as cm: + yield cm + uncaught = cm.exception.excinfo + self.assertEqual(uncaught.type.__name__, expected.__name__) + + def assert_fails_not_shareable(self): + return self.assert_fails(interpreters.NotShareableError) + + def assert_code_equal(self, code1, code2): + if code1 == code2: + return + self.assertEqual(code1.co_name, code2.co_name) + self.assertEqual(code1.co_flags, code2.co_flags) + self.assertEqual(code1.co_consts, code2.co_consts) + self.assertEqual(code1.co_varnames, code2.co_varnames) + self.assertEqual(code1.co_cellvars, code2.co_cellvars) + self.assertEqual(code1.co_freevars, code2.co_freevars) + self.assertEqual(code1.co_names, code2.co_names) + self.assertEqual( + _testinternalcapi.get_code_var_counts(code1), + _testinternalcapi.get_code_var_counts(code2), + ) + self.assertEqual(code1.co_code, code2.co_code) + + def assert_funcs_equal(self, func1, func2): + if func1 == func2: + return + self.assertIs(type(func1), type(func2)) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + self.assertEqual(func1.__closure__, func2.__closure__) + self.assert_code_equal(func1.__code__, func2.__code__) + self.assertEqual( + _testinternalcapi.get_code_var_counts(func1), + _testinternalcapi.get_code_var_counts(func2), + ) + + def assert_exceptions_equal(self, exc1, exc2): + assert isinstance(exc1, Exception) + assert isinstance(exc2, Exception) + if exc1 == exc2: + return + self.assertIs(type(exc1), type(exc2)) + self.assertEqual(exc1.args, exc2.args) + + def test_stateless_funcs(self): interp = interpreters.create() - for i, (callable, args, kwargs) in enumerate([ - (call_func_noop, (), {}), - (Spam.noop, (), {}), + func = call_func_noop + with self.subTest('no args, no return'): + res = interp.call(func) + self.assertIsNone(res) + + func = call_func_return_shareable + with self.subTest('no args, returns shareable'): + res = interp.call(func) + self.assertEqual(res, (1, None)) + + func = call_func_return_stateless_func + expected = (lambda x: x) + with self.subTest('no args, returns stateless func'): + res = interp.call(func) + self.assert_funcs_equal(res, expected) + + func = call_func_return_pickleable + with self.subTest('no args, returns pickleable'): + res = interp.call(func) + self.assertEqual(res, [1, 2, 3]) + + func = call_func_return_unpickleable + with self.subTest('no args, returns unpickleable'): + with self.assert_fails_not_shareable(): + interp.call(func) + + def test_stateless_func_returns_arg(self): + interp = interpreters.create() + + for arg in [ + None, + 10, + 'spam!', + b'spam!', + (1, 2, 'spam!'), + memoryview(b'spam!'), + ]: + with self.subTest(f'shareable {arg!r}'): + assert _interpreters.is_shareable(arg) + res = interp.call(defs.spam_returns_arg, arg) + self.assertEqual(res, arg) + + for arg in defs.STATELESS_FUNCTIONS: + with self.subTest(f'stateless func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + + for arg in defs.TOP_FUNCTIONS: + if arg in defs.STATELESS_FUNCTIONS: + continue + with self.subTest(f'stateful func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + assert is_pickleable(arg) + + for arg in [ + Ellipsis, + NotImplemented, + object(), + 2**1000, + [1, 2, 3], + {'a': 1, 'b': 2}, + types.SimpleNamespace(x=42), + # builtin types + object, + type, + Exception, + ModuleNotFoundError, + # builtin exceptions + Exception('uh-oh!'), + ModuleNotFoundError('mymodule'), + # builtin fnctions + len, + sys.exit, + # user classes + *defs.TOP_CLASSES, + *(c(*a) for c, a in defs.TOP_CLASSES.items() + if c not in defs.CLASSES_WITHOUT_EQUALITY), + ]: + with self.subTest(f'pickleable {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + if type(arg) is object: + self.assertIs(type(res), object) + elif isinstance(arg, BaseException): + self.assert_exceptions_equal(res, arg) + else: + self.assertEqual(res, arg) + assert is_pickleable(arg) + + for arg in [ + types.MappingProxyType({}), + *(f for f in defs.NESTED_FUNCTIONS + if f not in defs.STATELESS_FUNCTIONS), + ]: + with self.subTest(f'unpickleable {arg!r}'): + assert not _interpreters.is_shareable(arg) + assert not is_pickleable(arg) + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_full_args(self): + interp = interpreters.create() + expected = (1, 2, 3, 4, 5, 6, ('?',), {'g': 7, 'h': 8}) + func = defs.spam_full_args + res = interp.call(func, 1, 2, 3, 4, '?', e=5, f=6, g=7, h=8) + self.assertEqual(res, expected) + + def test_full_defaults(self): + # pickleable, but not stateless + interp = interpreters.create() + expected = (-1, -2, -3, -4, -5, -6, (), {'g': 8, 'h': 9}) + res = interp.call(defs.spam_full_args_with_defaults, g=8, h=9) + self.assertEqual(res, expected) + + def test_modified_arg(self): + interp = interpreters.create() + script = dedent(""" + a = 7 + b = 2 + c = a ** b + """) + ns = {} + expected = {'a': 7, 'b': 2, 'c': 49} + res = interp.call(call_func_exec_wrapper, script, ns) + obj, resns, resid = res + del resns['__builtins__'] + self.assertIsNone(obj) + self.assertEqual(ns, {}) + self.assertEqual(resns, expected) + self.assertNotEqual(resid, id(ns)) + self.assertNotEqual(resid, id(resns)) + + def test_func_in___main___valid(self): + # pickleable, already there' + + with os_helper.temp_dir() as tempdir: + def new_mod(name, text): + script_helper.make_script(tempdir, name, dedent(text)) + + def run(text): + name = 'myscript' + text = dedent(f""" + import sys + sys.path.insert(0, {tempdir!r}) + + """) + dedent(text) + filename = script_helper.make_script(tempdir, name, text) + res = script_helper.assert_python_ok(filename) + return res.out.decode('utf-8').strip() + + # no module indirection + with self.subTest('no indirection'): + text = run(f""" + from test.support import interpreters + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = interp.call(spam) + print(res) + """) + self.assertEqual(text, '') + + # indirect as func, direct interp + new_mod('mymod', f""" + def run(interp, func): + return interp.call(func) + """) + with self.subTest('indirect as func, direct interp'): + text = run(f""" + from test.support import interpreters + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = mymod.run(interp, spam) + print(res) + """) + self.assertEqual(text, '') + + # indirect as func, indirect interp + new_mod('mymod', f""" + from test.support import interpreters + def run(func): + interp = interpreters.create() + return interp.call(func) + """) + with self.subTest('indirect as func, indirect interp'): + text = run(f""" + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + res = mymod.run(spam) + print(res) + """) + self.assertEqual(text, '') + + def test_func_in___main___invalid(self): + interp = interpreters.create() + + funcname = f'{__name__.replace(".", "_")}_spam_okay' + script = dedent(f""" + def {funcname}(): + # This a global var... + return __name__ + """) + + with self.subTest('pickleable, added dynamically'): + with defined_in___main__(funcname, script) as arg: + with self.assert_fails_not_shareable(): + interp.call(defs.spam_returns_arg, arg) + + with self.subTest('lying about __main__'): + with defined_in___main__(funcname, script, remove=True) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_raises(self): + interp = interpreters.create() + with self.assertRaises(ExecutionFailed): + interp.call(call_func_failure) + + with self.assert_fails(ValueError): + interp.call(call_func_complex, '???', exc=ValueError('spam')) + + def test_call_valid(self): + interp = interpreters.create() + + for i, (callable, args, kwargs, expected) in enumerate([ + (call_func_noop, (), {}, None), + (call_func_ident, ('spamspamspam',), {}, 'spamspamspam'), + (call_func_return_shareable, (), {}, (1, None)), + (call_func_return_pickleable, (), {}, [1, 2, 3]), + (Spam.noop, (), {}, None), + (Spam.from_values, (), {}, Spam(())), + (Spam.from_values, (1, 2, 3), {}, Spam((1, 2, 3))), + (Spam, ('???',), {}, Spam('???')), + (Spam(101), (), {}, (101, (), {})), + (Spam(10101).run, (), {}, (10101, (), {})), + (call_func_complex, ('ident', 'spam'), {}, 'spam'), + (call_func_complex, ('full-ident', 'spam'), {}, ('spam', (), {})), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}, + ('spam', ('ham',), {'eggs': '!!!'})), + (call_func_complex, ('globals',), {}, __name__), + (call_func_complex, ('interpid',), {}, interp.id), + (call_func_complex, ('custom', 'spam!'), {}, Spam('spam!')), ]): with self.subTest(f'success case #{i+1}'): - res = interp.call(callable) - self.assertIs(res, None) + res = interp.call(callable, *args, **kwargs) + self.assertEqual(res, expected) + + def test_call_invalid(self): + interp = interpreters.create() + + func = get_call_func_closure + with self.subTest(func): + with self.assert_fails_not_shareable(): + interp.call(func, 42) + + func = get_call_func_closure(42) + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + func = call_func_complex + op = 'closure' + with self.subTest(f'{func} ({op})'): + with self.assert_fails_not_shareable(): + interp.call(func, op, value='~~~') + + op = 'custom-inner' + with self.subTest(f'{func} ({op})'): + with self.assert_fails_not_shareable(): + interp.call(func, op, 'eggs!') + + def test_call_in_thread(self): + interp = interpreters.create() for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), - (get_call_func_closure, (42,), {}), - (get_call_func_closure(42), (), {}), + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_pickleable, (), {}), (Spam.from_values, (), {}), (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), (Spam(101), (), {}), (Spam(10101).run, (), {}), + (Spam.noop, (), {}), (call_func_complex, ('ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), (call_func_complex, ('globals',), {}), (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), - ]): - with self.subTest(f'invalid case #{i+1}'): - with self.assertRaises(Exception): - if args or kwargs: - raise Exception((args, kwargs)) - interp.call(callable) - - with self.assertRaises(ExecutionFailed): - interp.call(call_func_failure) - - def test_call_in_thread(self): - interp = interpreters.create() - - for i, (callable, args, kwargs) in enumerate([ - (call_func_noop, (), {}), - (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) + t = interp.call_in_thread(callable, *args, **kwargs) t.join() self.assertIsNone(ctx.caught) for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), (get_call_func_closure, (42,), {}), (get_call_func_closure(42), (), {}), - (Spam.from_values, (), {}), - (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), - (Spam(101), (), {}), - (Spam(10101).run, (), {}), - (call_func_complex, ('ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), - (call_func_complex, ('globals',), {}), - (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), - (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): - if args or kwargs: - continue with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) + t = interp.call_in_thread(callable, *args, **kwargs) t.join() self.assertIsNotNone(ctx.caught) @@ -1623,14 +1947,43 @@ def test_exec(self): self.assertEqual(exc.msg, 'it worked!') def test_call(self): - with self.subTest('no args'): - interpid = _interpreters.create() - with self.assertRaises(ValueError): - _interpreters.call(interpid, call_func_return_shareable) + interpid = _interpreters.create() + + # Here we focus on basic args and return values. + # See TestInterpreterCall for full operational coverage, + # including supported callables. + + with self.subTest('no args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('empty args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func, (), {}) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('no args, return non-None'): + func = defs.script_with_return + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIs(res, True) + + with self.subTest('full args, return non-None'): + expected = (1, 2, 3, 4, 5, 6, (7, 8), {'g': 9, 'h': 0}) + func = defs.spam_full_args + args = (1, 2, 3, 4, 7, 8) + kwargs = dict(e=5, f=6, g=9, h=0) + res, exc = _interpreters.call(interpid, func, args, kwargs) + self.assertIsNone(exc) + self.assertEqual(res, expected) with self.subTest('uncaught exception'): - interpid = _interpreters.create() - exc = _interpreters.call(interpid, call_func_failure) + func = defs.spam_raises + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(res) self.assertEqual(exc, types.SimpleNamespace( type=types.SimpleNamespace( __name__='Exception', diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 66784c78c6fcf6..d4ada0a7ec47d5 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -316,14 +316,19 @@ get_module_state(PyObject *mod) } static module_state * -_get_current_module_state(void) +_get_current_module_state(int force) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + if (!force) { + PyErr_SetString(PyExc_RuntimeError, + MODULE_NAME_STR " module not imported yet"); + return NULL; + } + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); @@ -352,7 +357,8 @@ clear_module_state(module_state *state) static PyTypeObject * _get_current_xibufferview_type(void) { - module_state *state = _get_current_module_state(); + int force = 1; + module_state *state = _get_current_module_state(force); if (state == NULL) { return NULL; } @@ -422,6 +428,232 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } +struct interp_call { + _PyXIData_t *func; + _PyXIData_t *args; + _PyXIData_t *kwargs; + _PyXIData_t *result; // dynamically allocated + struct { + _PyXIData_t func; + _PyXIData_t args; + _PyXIData_t kwargs; + } _preallocated; +}; + +static void +_interp_call_clear(struct interp_call *call) +{ + struct interp_call temp = *call; + *call = (struct interp_call){0}; + if (temp.func != NULL) { + _PyXIData_Clear(NULL, temp.func); + } + if (temp.args != NULL) { + _PyXIData_Clear(NULL, temp.args); + } + if (temp.kwargs != NULL) { + _PyXIData_Clear(NULL, temp.kwargs); + } + if (temp.result != NULL) { + (void)_PyXIData_ReleaseAndRawFree(temp.result); + } +} + +static int +_interp_call_init_result(struct interp_call *call) +{ + call->result = _PyXIData_New(); + return (call->result == NULL) ? -1 : 0; +} + +static void +_interp_call_clear_result(struct interp_call *call) +{ + _PyXIData_t *xidata = call->result; + if (xidata == NULL) { + return; + } + call->result = NULL; + _PyXIData_Clear(NULL, xidata); + _PyXIData_Release(xidata); + PyMem_RawFree(xidata); +} + +static int +_interp_call_clear_result_immediately(PyThreadState *tstate, + struct interp_call *call, + PyInterpreterState *interp) +{ + _PyXIData_t *xidata = call->result; + if (xidata == NULL) { + return 0; + } + assert(tstate == _PyThreadState_GET()); + assert(interp != NULL); + assert(interp == _PyInterpreterState_LookUpID(_PyXIData_INTERPID(xidata))); + if (tstate->interp == interp) { + // There's no need to switch interpreters. + _interp_call_clear_result(call); + return 0; + } + + // It's from a different interpreter. + PyThreadState *temp_tstate = + _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); + if (temp_tstate == NULL) { + return -1; + } + PyThreadState *save_tstate = PyThreadState_Swap(temp_tstate); + assert(save_tstate == tstate); + + _interp_call_clear_result(call); + + PyThreadState_Clear(temp_tstate); + (void)PyThreadState_Swap(save_tstate); + PyThreadState_Delete(temp_tstate); + return 0; +} + +static int +_interp_call_pack(PyThreadState *tstate, struct interp_call *call, + PyObject *func, PyObject *args, PyObject *kwargs) +{ + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + assert(call->func == NULL); + assert(call->args == NULL); + assert(call->kwargs == NULL); + assert(call->result == NULL); + // Handle the func. + if (!PyCallable_Check(func)) { + _PyErr_Format(tstate, PyExc_TypeError, + "expected a callable, got %R", func); + return -1; + } + if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_SetRaisedException(tstate, exc); +//unwrap_not_shareable(tstate); + return -1; + } + } + call->func = &call->_preallocated.func; + // Handle the args. + if (args == NULL || args == Py_None) { + // Leave it empty. + } + else { + assert(PyTuple_Check(args)); + if (PyTuple_GET_SIZE(args) > 0) { + if (_PyObject_GetXIData( + tstate, args, fallback, &call->_preallocated.args) < 0) + { + return -1; + } + call->args = &call->_preallocated.args; + } + } + // Handle the kwargs. + if (kwargs == NULL || kwargs == Py_None) { + // Leave it empty. + } + else { + assert(PyDict_Check(kwargs)); + if (PyDict_GET_SIZE(kwargs) > 0) { + if (_PyObject_GetXIData( + tstate, kwargs, fallback, &call->_preallocated.kwargs) < 0) + { + return -1; + } + call->kwargs = &call->_preallocated.kwargs; + } + } + return 0; +} + +static PyObject * +_interp_call_pop_result(PyThreadState *tstate, struct interp_call *call, + PyInterpreterState *interp) +{ + assert(tstate == _PyThreadState_GET()); + assert(!_PyErr_Occurred(tstate)); + assert(call->result != NULL); + PyObject *res = _PyXIData_NewObject(call->result); + PyObject *exc = _PyErr_GetRaisedException(tstate); + + if (_interp_call_clear_result_immediately(tstate, call, interp) < 0) { + // We couldn't do it immediately, so we fall back to adding + // a pending call. If this fails then there are other, + // bigger problems. + (int)_PyXIData_ReleaseAndRawFree(call->result); + call->result = NULL; + } + + _PyErr_SetRaisedException(tstate, exc); + return res; +} + +static int +_make_call(struct interp_call *call) +{ + assert(call != NULL && call->func != NULL); + int res = -1; + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *resobj = NULL; + // Unpack the func. + PyObject *func = _PyXIData_NewObject(call->func); + if (func == NULL) { + return -1; + } + // Unpack the args. + if (call->args == NULL) { + args = PyTuple_New(0); + if (args == NULL) { + goto finally; + } + } + else { + args = _PyXIData_NewObject(call->args); + if (args == NULL) { + goto finally; + } + assert(PyTuple_Check(args)); + } + // Unpack the kwargs. + if (call->kwargs != NULL) { + kwargs = _PyXIData_NewObject(call->kwargs); + if (kwargs == NULL) { + goto finally; + } + assert(PyDict_Check(kwargs)); + } + // Prepare call->result. + if (_interp_call_init_result(call) < 0) { + goto finally; + } + // Make the call. + resobj = PyObject_Call(func, args, kwargs); + if (resobj == NULL) { + _interp_call_clear_result(call); + goto finally; + } + // Pack the result. + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + if (_PyObject_GetXIData(tstate, resobj, fallback, call->result) < 0) { + _interp_call_clear_result(call); + goto finally; + } + res = 0; + +finally: + Py_DECREF(func); + Py_XDECREF(args); + Py_XDECREF(kwargs); + return res; +} + static int _run_script(_PyXIData_t *script, PyObject *ns) { @@ -434,14 +666,15 @@ _run_script(_PyXIData_t *script, PyObject *ns) if (result == NULL) { return -1; } + assert(result == Py_None); Py_DECREF(result); // We throw away the result. return 0; } static int -_exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, - _PyXIData_t *script, PyObject *shareables, - _PyXI_session_result *result) +_run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, + _PyXIData_t *script, struct interp_call *call, + PyObject *shareables, _PyXI_session_result *result) { assert(!_PyErr_Occurred(tstate)); _PyXI_session *session = _PyXI_NewSession(); @@ -457,13 +690,20 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, return -1; } - // Run the script. int res = -1; - PyObject *mainns = _PyXI_GetMainNamespace(session); - if (mainns == NULL) { - goto finally; + if (script != NULL) { + // Run the script. + assert(call == NULL); + PyObject *mainns = _PyXI_GetMainNamespace(session); + if (mainns == NULL) { + goto finally; + } + res = _run_script(script, mainns); + } + else { + assert(call != NULL); + res = _make_call(call); } - res = _run_script(script, mainns); finally: // Clean up and switch back. @@ -898,7 +1138,8 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } _PyXI_session_result result = {0}; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &result); + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &result); _PyXIData_Release(&xidata); if (res < 0) { assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -961,7 +1202,8 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } _PyXI_session_result result = {0}; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &result); + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &result); _PyXIData_Release(&xidata); if (res < 0) { assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -1023,7 +1265,8 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) } _PyXI_session_result result = {0}; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &result); + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &result); _PyXIData_Release(&xidata); if (res < 0) { assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); @@ -1054,8 +1297,10 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject *kwargs_obj = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|OO$p:" FUNCNAME, kwlist, - &id, &callable, &args_obj, &kwargs_obj, + "OO|O!O!$p:" FUNCNAME, kwlist, + &id, &callable, + &PyTuple_Type, &args_obj, + &PyDict_Type, &kwargs_obj, &restricted)) { return NULL; @@ -1068,29 +1313,38 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (args_obj != NULL) { - _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected args"); - return NULL; - } - if (kwargs_obj != NULL) { - _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected kwargs"); - return NULL; - } - - _PyXIData_t xidata = {0}; - if (_PyCode_GetPureScriptXIData(tstate, callable, &xidata) < 0) { - unwrap_not_shareable(tstate); + struct interp_call call = {0}; + if (_interp_call_pack(tstate, &call, callable, args_obj, kwargs_obj) < 0) { return NULL; } + PyObject *res_and_exc = NULL; _PyXI_session_result result = {0}; - int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &result); - _PyXIData_Release(&xidata); - if (res < 0) { - assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return result.excinfo; + if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &result) < 0) { + assert(result.preserved == NULL); + if (result.excinfo == NULL) { + assert(_PyErr_Occurred(tstate)); + goto finally; + } + assert(!_PyErr_Occurred(tstate)); + assert(call.result == NULL); + res_and_exc = Py_BuildValue("OO", Py_None, result.excinfo); + Py_CLEAR(result.excinfo); } - Py_RETURN_NONE; + else { + assert(result.preserved == NULL); + assert(result.excinfo == NULL); + PyObject *res = _interp_call_pop_result(tstate, &call, interp); + if (res == NULL) { + goto finally; + } + res_and_exc = Py_BuildValue("OO", res, Py_None); + Py_DECREF(res); + } + +finally: + _interp_call_clear(&call); + return res_and_exc; #undef FUNCNAME } @@ -1098,13 +1352,7 @@ PyDoc_STRVAR(call_doc, "call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ \n\ Call the provided object in the identified interpreter.\n\ -Pass the given args and kwargs, if possible.\n\ -\n\ -\"callable\" may be a plain function with no free vars that takes\n\ -no arguments.\n\ -\n\ -The function's code object is used and all its state\n\ -is ignored, including its __globals__ dict."); +Pass the given args and kwargs, if possible."); static PyObject * From d173ec093150cf792d5fc699c661c7c01fbcd2ad Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 8 May 2025 16:17:13 -0600 Subject: [PATCH 04/11] Use _PyXI_Preserve(). --- Modules/_interpretersmodule.c | 193 +++++++++++----------------------- 1 file changed, 61 insertions(+), 132 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index d4ada0a7ec47d5..2cfdea9a5f6092 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -432,7 +432,6 @@ struct interp_call { _PyXIData_t *func; _PyXIData_t *args; _PyXIData_t *kwargs; - _PyXIData_t *result; // dynamically allocated struct { _PyXIData_t func; _PyXIData_t args; @@ -454,64 +453,6 @@ _interp_call_clear(struct interp_call *call) if (temp.kwargs != NULL) { _PyXIData_Clear(NULL, temp.kwargs); } - if (temp.result != NULL) { - (void)_PyXIData_ReleaseAndRawFree(temp.result); - } -} - -static int -_interp_call_init_result(struct interp_call *call) -{ - call->result = _PyXIData_New(); - return (call->result == NULL) ? -1 : 0; -} - -static void -_interp_call_clear_result(struct interp_call *call) -{ - _PyXIData_t *xidata = call->result; - if (xidata == NULL) { - return; - } - call->result = NULL; - _PyXIData_Clear(NULL, xidata); - _PyXIData_Release(xidata); - PyMem_RawFree(xidata); -} - -static int -_interp_call_clear_result_immediately(PyThreadState *tstate, - struct interp_call *call, - PyInterpreterState *interp) -{ - _PyXIData_t *xidata = call->result; - if (xidata == NULL) { - return 0; - } - assert(tstate == _PyThreadState_GET()); - assert(interp != NULL); - assert(interp == _PyInterpreterState_LookUpID(_PyXIData_INTERPID(xidata))); - if (tstate->interp == interp) { - // There's no need to switch interpreters. - _interp_call_clear_result(call); - return 0; - } - - // It's from a different interpreter. - PyThreadState *temp_tstate = - _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); - if (temp_tstate == NULL) { - return -1; - } - PyThreadState *save_tstate = PyThreadState_Swap(temp_tstate); - assert(save_tstate == tstate); - - _interp_call_clear_result(call); - - PyThreadState_Clear(temp_tstate); - (void)PyThreadState_Swap(save_tstate); - PyThreadState_Delete(temp_tstate); - return 0; } static int @@ -522,7 +463,6 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, assert(call->func == NULL); assert(call->args == NULL); assert(call->kwargs == NULL); - assert(call->result == NULL); // Handle the func. if (!PyCallable_Check(func)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -571,34 +511,11 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, return 0; } -static PyObject * -_interp_call_pop_result(PyThreadState *tstate, struct interp_call *call, - PyInterpreterState *interp) -{ - assert(tstate == _PyThreadState_GET()); - assert(!_PyErr_Occurred(tstate)); - assert(call->result != NULL); - PyObject *res = _PyXIData_NewObject(call->result); - PyObject *exc = _PyErr_GetRaisedException(tstate); - - if (_interp_call_clear_result_immediately(tstate, call, interp) < 0) { - // We couldn't do it immediately, so we fall back to adding - // a pending call. If this fails then there are other, - // bigger problems. - (int)_PyXIData_ReleaseAndRawFree(call->result); - call->result = NULL; - } - - _PyErr_SetRaisedException(tstate, exc); - return res; -} - static int -_make_call(struct interp_call *call) +_make_call(struct interp_call *call, PyObject **p_result) { assert(call != NULL && call->func != NULL); int res = -1; - PyThreadState *tstate = _PyThreadState_GET(); PyObject *args = NULL; PyObject *kwargs = NULL; PyObject *resobj = NULL; @@ -629,22 +546,12 @@ _make_call(struct interp_call *call) } assert(PyDict_Check(kwargs)); } - // Prepare call->result. - if (_interp_call_init_result(call) < 0) { - goto finally; - } // Make the call. resobj = PyObject_Call(func, args, kwargs); if (resobj == NULL) { - _interp_call_clear_result(call); - goto finally; - } - // Pack the result. - xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; - if (_PyObject_GetXIData(tstate, resobj, fallback, call->result) < 0) { - _interp_call_clear_result(call); goto finally; } + *p_result = resobj; res = 0; finally: @@ -671,19 +578,32 @@ _run_script(_PyXIData_t *script, PyObject *ns) return 0; } +struct run_result { + PyObject *result; + PyObject *excinfo; +}; + +static void +_run_result_clear(struct run_result *runres) +{ + Py_CLEAR(runres->result); + Py_CLEAR(runres->excinfo); +} + static int _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, _PyXIData_t *script, struct interp_call *call, - PyObject *shareables, _PyXI_session_result *result) + PyObject *shareables, struct run_result *runres) { assert(!_PyErr_Occurred(tstate)); _PyXI_session *session = _PyXI_NewSession(); if (session == NULL) { return -1; } + _PyXI_session_result result = {0}; // Prep and switch interpreters. - if (_PyXI_Enter(session, interp, shareables, result) < 0) { + if (_PyXI_Enter(session, interp, shareables, &result) < 0) { // If an error occured at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); @@ -692,7 +612,6 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, int res = -1; if (script != NULL) { - // Run the script. assert(call == NULL); PyObject *mainns = _PyXI_GetMainNamespace(session); if (mainns == NULL) { @@ -702,13 +621,31 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, } else { assert(call != NULL); - res = _make_call(call); + PyObject *resobj; + res = _make_call(call, &resobj); + if (res == 0) { + (void)_PyXI_Preserve(session, "resobj", resobj); + Py_DECREF(resobj); + } } finally: // Clean up and switch back. - (void)_PyXI_Exit(session, result); + (void)_PyXI_Exit(session, &result); _PyXI_FreeSession(session); + if (res < 0) { + runres->excinfo = result.excinfo; + } + else if (result.excinfo != NULL) { + runres->excinfo = result.excinfo; + res = -1; + } + else { + runres->result = _PyXI_GetPreserved(&result, "resobj"); + if (_PyErr_Occurred(tstate)) { + res = -1; + } + } return res; } @@ -1137,13 +1074,13 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - _PyXI_session_result result = {0}; + struct run_result runres = {0}; int res = _run_in_interpreter( - tstate, interp, &xidata, NULL, shared, &result); + tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return result.excinfo; + assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); + return runres.excinfo; } Py_RETURN_NONE; #undef FUNCNAME @@ -1201,13 +1138,13 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - _PyXI_session_result result = {0}; + struct run_result runres = {0}; int res = _run_in_interpreter( - tstate, interp, &xidata, NULL, shared, &result); + tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return result.excinfo; + assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); + return runres.excinfo; } Py_RETURN_NONE; #undef FUNCNAME @@ -1264,13 +1201,13 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - _PyXI_session_result result = {0}; + struct run_result runres = {0}; int res = _run_in_interpreter( - tstate, interp, &xidata, NULL, shared, &result); + tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((result.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return result.excinfo; + assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); + return runres.excinfo; } Py_RETURN_NONE; #undef FUNCNAME @@ -1291,17 +1228,18 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) #define FUNCNAME MODULE_NAME_STR ".call" PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "callable", "args", "kwargs", - "restrict", NULL}; + "preserve_exc", "restrict", NULL}; PyObject *id, *callable; PyObject *args_obj = NULL; PyObject *kwargs_obj = NULL; + int preserve_exc = 0; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!O!$p:" FUNCNAME, kwlist, + "OO|O!O!$pp:" FUNCNAME, kwlist, &id, &callable, &PyTuple_Type, &args_obj, &PyDict_Type, &kwargs_obj, - &restricted)) + &preserve_exc, &restricted)) { return NULL; } @@ -1319,31 +1257,22 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *res_and_exc = NULL; - _PyXI_session_result result = {0}; - if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &result) < 0) { - assert(result.preserved == NULL); - if (result.excinfo == NULL) { + struct run_result runres = {0}; + if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) { + if (runres.excinfo == NULL) { assert(_PyErr_Occurred(tstate)); goto finally; } assert(!_PyErr_Occurred(tstate)); - assert(call.result == NULL); - res_and_exc = Py_BuildValue("OO", Py_None, result.excinfo); - Py_CLEAR(result.excinfo); - } - else { - assert(result.preserved == NULL); - assert(result.excinfo == NULL); - PyObject *res = _interp_call_pop_result(tstate, &call, interp); - if (res == NULL) { - goto finally; - } - res_and_exc = Py_BuildValue("OO", res, Py_None); - Py_DECREF(res); } + assert(runres.result == NULL || runres.excinfo == NULL); + res_and_exc = Py_BuildValue("OO", + (runres.result ? runres.result : Py_None), + (runres.excinfo ? runres.excinfo : Py_None)); finally: _interp_call_clear(&call); + _run_result_clear(&runres); return res_and_exc; #undef FUNCNAME } From a9612e358444d232d7908c6d68c1886f8886c76f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 26 May 2025 11:32:13 -0600 Subject: [PATCH 05/11] Update the Interpreter.call() docstring. --- Lib/test/support/interpreters/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 93b367452a4f9f..6d1b0690805d2d 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -235,13 +235,13 @@ def _call(self, callable, args, kwargs): def call(self, callable, /, *args, **kwargs): """Call the object in the interpreter with given args/kwargs. - Only functions that take no arguments and have no closure - are supported. - - The return value is discarded. + Nearly all callables, args, kwargs, and return values are + supported. All "shareable" objects are supported, as are + "stateless" functions (meaning non-closures that do not use + any globals). This method will fall back to pickle. If the callable raises an exception then the error display - (including full traceback) is send back between the interpreters + (including full traceback) is sent back between the interpreters and an ExecutionFailed exception is raised, much like what happens with Interpreter.exec(). """ From 5d7141de581a5e349390826c99631ebb9aa341b4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 26 May 2025 16:43:18 -0600 Subject: [PATCH 06/11] Improve handling of xidata-to-object for scripts and calls. --- Modules/_interpretersmodule.c | 123 +++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 26 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 2cfdea9a5f6092..8e954d1c182f55 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -512,60 +512,78 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, } static int -_make_call(struct interp_call *call, PyObject **p_result) +_interp_call_unpack(struct interp_call *call, + PyObject **p_func, PyObject **p_args, PyObject **p_kwargs) { - assert(call != NULL && call->func != NULL); - int res = -1; - PyObject *args = NULL; - PyObject *kwargs = NULL; - PyObject *resobj = NULL; // Unpack the func. PyObject *func = _PyXIData_NewObject(call->func); if (func == NULL) { return -1; } // Unpack the args. + PyObject *args; if (call->args == NULL) { args = PyTuple_New(0); if (args == NULL) { - goto finally; + Py_DECREF(func); + return -1; } } else { args = _PyXIData_NewObject(call->args); if (args == NULL) { - goto finally; + Py_DECREF(func); + return -1; } assert(PyTuple_Check(args)); } // Unpack the kwargs. + PyObject *kwargs = NULL; if (call->kwargs != NULL) { kwargs = _PyXIData_NewObject(call->kwargs); if (kwargs == NULL) { - goto finally; + Py_DECREF(func); + Py_DECREF(args); + return -1; } assert(PyDict_Check(kwargs)); } - // Make the call. - resobj = PyObject_Call(func, args, kwargs); - if (resobj == NULL) { - goto finally; + *p_func = func; + *p_args = args; + *p_kwargs = kwargs; + return 0; +} + +static int +_make_call(struct interp_call *call, PyObject **p_result, int *p_badtarget) +{ + assert(call != NULL && call->func != NULL); + + PyObject *func, *args, *kwargs; + if (_interp_call_unpack(call, &func, &args, &kwargs) < 0) { + *p_badtarget = 1; + return -1; } - *p_result = resobj; - res = 0; + *p_badtarget = 0; -finally: + // Make the call. + PyObject *resobj = PyObject_Call(func, args, kwargs); Py_DECREF(func); Py_XDECREF(args); Py_XDECREF(kwargs); - return res; + if (resobj == NULL) { + return -1; + } + *p_result = resobj; + return 0; } static int -_run_script(_PyXIData_t *script, PyObject *ns) +_run_script(_PyXIData_t *script, PyObject *ns, int *p_badtarget) { PyObject *code = _PyXIData_NewObject(script); if (code == NULL) { + *p_badtarget = 1; return -1; } PyObject *result = PyEval_EvalCode(code, ns, ns); @@ -581,6 +599,7 @@ _run_script(_PyXIData_t *script, PyObject *ns) struct run_result { PyObject *result; PyObject *excinfo; + int badtarget; }; static void @@ -601,6 +620,7 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, return -1; } _PyXI_session_result result = {0}; + int badtarget = 0; // Prep and switch interpreters. if (_PyXI_Enter(session, interp, shareables, &result) < 0) { @@ -617,12 +637,12 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, if (mainns == NULL) { goto finally; } - res = _run_script(script, mainns); + res = _run_script(script, mainns, &badtarget); } else { assert(call != NULL); PyObject *resobj; - res = _make_call(call, &resobj); + res = _make_call(call, &resobj, &badtarget); if (res == 0) { (void)_PyXI_Preserve(session, "resobj", resobj); Py_DECREF(resobj); @@ -635,9 +655,11 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, _PyXI_FreeSession(session); if (res < 0) { runres->excinfo = result.excinfo; + runres->badtarget = badtarget; } else if (result.excinfo != NULL) { runres->excinfo = result.excinfo; + runres->badtarget = badtarget; res = -1; } else { @@ -1042,6 +1064,31 @@ unwrap_not_shareable(PyThreadState *tstate) _PyErr_SetRaisedException(tstate, exc); } +static PyObject * +_handle_script_error(_PyXIData_t *script, struct run_result *runres) +{ + if (runres->excinfo == NULL) { + assert(PyErr_Occurred()); + assert(!runres->badtarget); + return NULL; + } + assert(!PyErr_Occurred()); + if (!runres->badtarget) { + return runres->excinfo; + } + PyObject *code = _PyXIData_NewObject(script); + if (code != NULL) { + // Either the problem is intermittent or only affects subinterpreters. + // This is highly unlikely. + return runres->excinfo; + } + assert(PyErr_Occurred()); + PyThreadState *tstate = _PyThreadState_GET(); + unwrap_not_shareable(tstate); + Py_DECREF(runres->excinfo); + return NULL; +} + static PyObject * interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1080,7 +1127,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) _PyXIData_Release(&xidata); if (res < 0) { assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return runres.excinfo; + return _handle_script_error(&xidata, &runres); } Py_RETURN_NONE; #undef FUNCNAME @@ -1144,7 +1191,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) _PyXIData_Release(&xidata); if (res < 0) { assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return runres.excinfo; + return _handle_script_error(&xidata, &runres); } Py_RETURN_NONE; #undef FUNCNAME @@ -1207,7 +1254,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) _PyXIData_Release(&xidata); if (res < 0) { assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return runres.excinfo; + return _handle_script_error(&xidata, &runres); } Py_RETURN_NONE; #undef FUNCNAME @@ -1222,6 +1269,32 @@ are not supported. Methods and other callables are not supported either.\n\ \n\ (See " MODULE_NAME_STR ".exec()."); +static int +handle_call_error(struct interp_call *call, struct run_result *runres) +{ + if (runres->excinfo == NULL) { + assert(PyErr_Occurred()); + assert(!runres->badtarget); + return -1; + } + assert(!PyErr_Occurred()); + if (!runres->badtarget) { + return 0; + } + + PyObject *func, *args, *kwargs; + if (_interp_call_unpack(call, &func, &args, &kwargs) == 0) { + // Either the problem is intermittent or only affects subinterpreters. + // This is highly unlikely. + return 0; + } + assert(PyErr_Occurred()); + PyThreadState *tstate = _PyThreadState_GET(); + unwrap_not_shareable(tstate); + Py_CLEAR(runres->excinfo); + return -1; +} + static PyObject * interp_call(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1259,11 +1332,9 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject *res_and_exc = NULL; struct run_result runres = {0}; if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) { - if (runres.excinfo == NULL) { - assert(_PyErr_Occurred(tstate)); + if (handle_call_error(&call, &runres) < 0) { goto finally; } - assert(!_PyErr_Occurred(tstate)); } assert(runres.result == NULL || runres.excinfo == NULL); res_and_exc = Py_BuildValue("OO", From c25d9e1157f170175e64ec4d605402d0659dde42 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 26 May 2025 16:41:03 -0600 Subject: [PATCH 07/11] Always load the missing module in XI data situations. --- Modules/_interpchannelsmodule.c | 8 ++++---- Modules/_interpqueuesmodule.c | 8 ++++---- Modules/_interpretersmodule.c | 10 ++-------- Python/import.c | 4 +++- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index bfd805bf5e4072..ea2e5f99dfa308 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -254,10 +254,10 @@ _get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index ffc52c8ee74d85..71d8fd8716cd94 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1356,10 +1356,10 @@ _queueobj_from_xid(_PyXIData_t *data) PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } PyTypeObject *cls = get_external_queue_type(mod); diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 8e954d1c182f55..dd9449b6d4fd81 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -316,15 +316,10 @@ get_module_state(PyObject *mod) } static module_state * -_get_current_module_state(int force) +_get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - if (!force) { - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; - } mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { return NULL; @@ -357,8 +352,7 @@ clear_module_state(module_state *state) static PyTypeObject * _get_current_xibufferview_type(void) { - int force = 1; - module_state *state = _get_current_module_state(force); + module_state *state = _get_current_module_state(); if (state == NULL) { return NULL; } diff --git a/Python/import.c b/Python/import.c index e7be1b90751a6c..8c20c20a69539e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3964,8 +3964,10 @@ PyImport_Import(PyObject *module_name) if (globals != NULL) { Py_INCREF(globals); builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); - if (builtins == NULL) + if (builtins == NULL) { + // XXX Fall back to interp->builtins or sys.modules['builtins']? goto err; + } } else { /* No globals -- use standard builtins, and fake globals */ From 66bed1d0cadbe3596320459a0d0f692b5feeea40 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 28 May 2025 12:09:33 -0600 Subject: [PATCH 08/11] Fix refleaks. --- Include/internal/pycore_crossinterp.h | 1 + Modules/_interpretersmodule.c | 28 ++++++++++++++++----------- Python/crossinterp.c | 7 +++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index d8e87174988b6d..046d9765a718c0 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -354,6 +354,7 @@ typedef struct { PyObject *preserved; PyObject *excinfo; } _PyXI_session_result; +PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *); PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index dd9449b6d4fd81..8620127e02eea3 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -436,17 +436,16 @@ struct interp_call { static void _interp_call_clear(struct interp_call *call) { - struct interp_call temp = *call; - *call = (struct interp_call){0}; - if (temp.func != NULL) { - _PyXIData_Clear(NULL, temp.func); + if (call->func != NULL) { + _PyXIData_Clear(NULL, call->func); } - if (temp.args != NULL) { - _PyXIData_Clear(NULL, temp.args); + if (call->args != NULL) { + _PyXIData_Clear(NULL, call->args); } - if (temp.kwargs != NULL) { - _PyXIData_Clear(NULL, temp.kwargs); + if (call->kwargs != NULL) { + _PyXIData_Clear(NULL, call->kwargs); } + *call = (struct interp_call){0}; } static int @@ -467,9 +466,9 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, PyObject *exc = _PyErr_GetRaisedException(tstate); if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { _PyErr_SetRaisedException(tstate, exc); -//unwrap_not_shareable(tstate); return -1; } + Py_DECREF(exc); } call->func = &call->_preallocated.func; // Handle the args. @@ -482,6 +481,7 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, if (_PyObject_GetXIData( tstate, args, fallback, &call->_preallocated.args) < 0) { + _interp_call_clear(call); return -1; } call->args = &call->_preallocated.args; @@ -497,6 +497,7 @@ _interp_call_pack(PyThreadState *tstate, struct interp_call *call, if (_PyObject_GetXIData( tstate, kwargs, fallback, &call->_preallocated.kwargs) < 0) { + _interp_call_clear(call); return -1; } call->kwargs = &call->_preallocated.kwargs; @@ -648,11 +649,11 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, (void)_PyXI_Exit(session, &result); _PyXI_FreeSession(session); if (res < 0) { - runres->excinfo = result.excinfo; + runres->excinfo = Py_NewRef(result.excinfo); runres->badtarget = badtarget; } else if (result.excinfo != NULL) { - runres->excinfo = result.excinfo; + runres->excinfo = Py_NewRef(result.excinfo); runres->badtarget = badtarget; res = -1; } @@ -662,6 +663,7 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, res = -1; } } + _PyXI_ClearResult(&result); return res; } @@ -1276,12 +1278,16 @@ handle_call_error(struct interp_call *call, struct run_result *runres) return 0; } + // Handle the case where _PyXIData_NewObject() fails. PyObject *func, *args, *kwargs; if (_interp_call_unpack(call, &func, &args, &kwargs) == 0) { // Either the problem is intermittent or only affects subinterpreters. // This is highly unlikely. return 0; } + Py_DECREF(func); + Py_XDECREF(args); + Py_XDECREF(kwargs); assert(PyErr_Occurred()); PyThreadState *tstate = _PyThreadState_GET(); unwrap_not_shareable(tstate); diff --git a/Python/crossinterp.c b/Python/crossinterp.c index ba0457693b561e..1971aa47d4b1d7 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -2765,6 +2765,13 @@ _PyXI_GetPreserved(_PyXI_session_result *result, const char *name) return value; } +void +_PyXI_ClearResult(_PyXI_session_result *result) +{ + Py_CLEAR(result->preserved); + Py_CLEAR(result->excinfo); +} + /*********************/ /* runtime lifecycle */ From 3ed342338d8d82d9bc2417689d1dcffaf213ca5b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 28 May 2025 12:17:36 -0600 Subject: [PATCH 09/11] Fix _PyFunction_VerifyStateless(). --- Objects/funcobject.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 27214a129c2fb8..f87b0e5d8f1e47 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1264,26 +1264,32 @@ _PyFunction_VerifyStateless(PyThreadState *tstate, PyObject *func) } // Disallow __defaults__. PyObject *defaults = PyFunction_GET_DEFAULTS(func); - if (defaults != NULL && defaults != Py_None && PyDict_Size(defaults) > 0) - { - _PyErr_SetString(tstate, PyExc_ValueError, "defaults not supported"); - return -1; + if (defaults != NULL) { + assert(PyTuple_Check(defaults)); // per PyFunction_New() + if (PyTuple_GET_SIZE(defaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "defaults not supported"); + return -1; + } } // Disallow __kwdefaults__. PyObject *kwdefaults = PyFunction_GET_KW_DEFAULTS(func); - if (kwdefaults != NULL && kwdefaults != Py_None - && PyDict_Size(kwdefaults) > 0) - { - _PyErr_SetString(tstate, PyExc_ValueError, - "keyword defaults not supported"); - return -1; + if (kwdefaults != NULL) { + assert(PyDict_Check(kwdefaults)); // per PyFunction_New() + if (PyDict_Size(kwdefaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "keyword defaults not supported"); + return -1; + } } // Disallow __closure__. PyObject *closure = PyFunction_GET_CLOSURE(func); - if (closure != NULL && closure != Py_None && PyTuple_GET_SIZE(closure) > 0) - { - _PyErr_SetString(tstate, PyExc_ValueError, "closures not supported"); - return -1; + if (closure != NULL) { + assert(PyTuple_Check(closure)); // per PyFunction_New() + if (PyTuple_GET_SIZE(closure) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, "closures not supported"); + return -1; + } } // Check the code. PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); From 17beeda38ad32d39017a4fe5e97220736a1e7a4f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 28 May 2025 22:35:33 -0600 Subject: [PATCH 10/11] Fix exc propagation and an excinfo memory leak. --- Include/internal/pycore_crossinterp.h | 20 +- Lib/test/test_interpreters/test_api.py | 68 ++++-- Modules/_interpretersmodule.c | 165 ++++++-------- Python/crossinterp.c | 302 +++++++++++++++++-------- 4 files changed, 340 insertions(+), 215 deletions(-) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 046d9765a718c0..713ddc66ba7382 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -317,7 +317,9 @@ typedef enum error_code { _PyXI_ERR_ALREADY_RUNNING = -4, _PyXI_ERR_MAIN_NS_FAILURE = -5, _PyXI_ERR_APPLY_NS_FAILURE = -6, - _PyXI_ERR_NOT_SHAREABLE = -7, + _PyXI_ERR_PRESERVE_FAILURE = -7, + _PyXI_ERR_EXC_PROPAGATION_FAILURE = -8, + _PyXI_ERR_NOT_SHAREABLE = -9, } _PyXI_errcode; @@ -353,6 +355,7 @@ PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); typedef struct { PyObject *preserved; PyObject *excinfo; + _PyXI_errcode errcode; } _PyXI_session_result; PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *); @@ -361,11 +364,20 @@ PyAPI_FUNC(int) _PyXI_Enter( PyInterpreterState *interp, PyObject *nsupdates, _PyXI_session_result *); -PyAPI_FUNC(int) _PyXI_Exit(_PyXI_session *, _PyXI_session_result *); +PyAPI_FUNC(int) _PyXI_Exit( + _PyXI_session *, + _PyXI_errcode, + _PyXI_session_result *); -PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *); +PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace( + _PyXI_session *, + _PyXI_errcode *); -PyAPI_FUNC(int) _PyXI_Preserve(_PyXI_session *, const char *, PyObject *); +PyAPI_FUNC(int) _PyXI_Preserve( + _PyXI_session *, + const char *, + PyObject *, + _PyXI_errcode *); PyAPI_FUNC(PyObject *) _PyXI_GetPreserved(_PyXI_session_result *, const char *); diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 7133f4605ec753..b3c9ef8efba37a 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -57,6 +57,36 @@ def defined_in___main__(name, script, *, remove=False): mainns.pop(name, None) +def build_excinfo(exctype, msg=None, formatted=None, errdisplay=None): + if isinstance(exctype, type): + assert issubclass(exctype, BaseException), exctype + exctype = types.SimpleNamespace( + __name__=exctype.__name__, + __qualname__=exctype.__qualname__, + __module__=exctype.__module__, + ) + elif isinstance(exctype, str): + module, _, name = exctype.rpartition(exctype) + if not module and name in __builtins__: + module = 'builtins' + exctype = types.SimpleNamespace( + __name__=name, + __qualname__=exctype, + __module__=module or None, + ) + else: + assert isinstance(exctype, types.SimpleNamespace) + assert msg is None or isinstance(msg, str), msg + assert formatted is None or isinstance(formatted, str), formatted + assert errdisplay is None or isinstance(errdisplay, str), errdisplay + return types.SimpleNamespace( + type=exctype, + msg=msg, + formatted=formatted, + errdisplay=errdisplay, + ) + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -1121,7 +1151,7 @@ def test_stateless_funcs(self): func = call_func_return_unpickleable with self.subTest('no args, returns unpickleable'): - with self.assert_fails_not_shareable(): + with self.assertRaises(interpreters.NotShareableError): interp.call(func) def test_stateless_func_returns_arg(self): @@ -1318,7 +1348,7 @@ def {funcname}(): with self.subTest('pickleable, added dynamically'): with defined_in___main__(funcname, script) as arg: - with self.assert_fails_not_shareable(): + with self.assertRaises(interpreters.NotShareableError): interp.call(defs.spam_returns_arg, arg) with self.subTest('lying about __main__'): @@ -1365,7 +1395,7 @@ def test_call_invalid(self): func = get_call_func_closure with self.subTest(func): - with self.assert_fails_not_shareable(): + with self.assertRaises(interpreters.NotShareableError): interp.call(func, 42) func = get_call_func_closure(42) @@ -1376,12 +1406,12 @@ def test_call_invalid(self): func = call_func_complex op = 'closure' with self.subTest(f'{func} ({op})'): - with self.assert_fails_not_shareable(): + with self.assertRaises(interpreters.NotShareableError): interp.call(func, op, value='~~~') op = 'custom-inner' with self.subTest(f'{func} ({op})'): - with self.assert_fails_not_shareable(): + with self.assertRaises(interpreters.NotShareableError): interp.call(func, op, 'eggs!') def test_call_in_thread(self): @@ -1924,18 +1954,14 @@ def test_exec(self): with results: exc = _interpreters.exec(interpid, script) out = results.stdout() - self.assertEqual(out, '') - self.assert_ns_equal(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='uh-oh!', + expected = build_excinfo( + Exception, 'uh-oh!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertEqual(out, '') + self.assert_ns_equal(exc, expected) with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: @@ -1983,18 +2009,14 @@ def test_call(self): with self.subTest('uncaught exception'): func = defs.spam_raises res, exc = _interpreters.call(interpid, func) - self.assertIsNone(res) - self.assertEqual(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='spam!', + expected = build_excinfo( + Exception, 'spam!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertIsNone(res) + self.assertEqual(exc, expected) @requires_test_modules def test_set___main___attrs(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 8620127e02eea3..037e9544543c4d 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -72,6 +72,32 @@ is_running_main(PyInterpreterState *interp) } +static inline int +is_notshareable_raised(PyThreadState *tstate) +{ + PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); + return _PyErr_ExceptionMatches(tstate, exctype); +} + +static void +unwrap_not_shareable(PyThreadState *tstate) +{ + if (!is_notshareable_raised(tstate)) { + return; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *cause = PyException_GetCause(exc); + if (cause != NULL) { + Py_DECREF(exc); + exc = cause; + } + else { + assert(PyException_GetContext(exc) == NULL); + } + _PyErr_SetRaisedException(tstate, exc); +} + + /* Cross-interpreter Buffer Views *******************************************/ /* When a memoryview object is "shared" between interpreters, @@ -550,16 +576,24 @@ _interp_call_unpack(struct interp_call *call, } static int -_make_call(struct interp_call *call, PyObject **p_result, int *p_badtarget) +_make_call(struct interp_call *call, + PyObject **p_result, _PyXI_errcode *p_errcode) { assert(call != NULL && call->func != NULL); + PyThreadState *tstate = _PyThreadState_GET(); - PyObject *func, *args, *kwargs; + // Get the func and args. + PyObject *func = NULL, *args = NULL, *kwargs = NULL; if (_interp_call_unpack(call, &func, &args, &kwargs) < 0) { - *p_badtarget = 1; + assert(func == NULL); + assert(args == NULL); + assert(kwargs == NULL); + *p_errcode = is_notshareable_raised(tstate) + ? _PyXI_ERR_NOT_SHAREABLE + : _PyXI_ERR_OTHER; return -1; } - *p_badtarget = 0; + *p_errcode = _PyXI_ERR_NO_ERROR; // Make the call. PyObject *resobj = PyObject_Call(func, args, kwargs); @@ -574,16 +608,17 @@ _make_call(struct interp_call *call, PyObject **p_result, int *p_badtarget) } static int -_run_script(_PyXIData_t *script, PyObject *ns, int *p_badtarget) +_run_script(_PyXIData_t *script, PyObject *ns, _PyXI_errcode *p_errcode) { PyObject *code = _PyXIData_NewObject(script); if (code == NULL) { - *p_badtarget = 1; + *p_errcode = _PyXI_ERR_NOT_SHAREABLE; return -1; } PyObject *result = PyEval_EvalCode(code, ns, ns); Py_DECREF(code); if (result == NULL) { + *p_errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; return -1; } assert(result == Py_None); @@ -594,7 +629,6 @@ _run_script(_PyXIData_t *script, PyObject *ns, int *p_badtarget) struct run_result { PyObject *result; PyObject *excinfo; - int badtarget; }; static void @@ -615,49 +649,58 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, return -1; } _PyXI_session_result result = {0}; - int badtarget = 0; // Prep and switch interpreters. if (_PyXI_Enter(session, interp, shareables, &result) < 0) { // If an error occured at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); + assert(result.excinfo == NULL); return -1; } + // Run in the interpreter. int res = -1; + _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; if (script != NULL) { assert(call == NULL); - PyObject *mainns = _PyXI_GetMainNamespace(session); + PyObject *mainns = _PyXI_GetMainNamespace(session, &errcode); if (mainns == NULL) { goto finally; } - res = _run_script(script, mainns, &badtarget); + res = _run_script(script, mainns, &errcode); } else { assert(call != NULL); PyObject *resobj; - res = _make_call(call, &resobj, &badtarget); + res = _make_call(call, &resobj, &errcode); if (res == 0) { - (void)_PyXI_Preserve(session, "resobj", resobj); + res = _PyXI_Preserve(session, "resobj", resobj, &errcode); Py_DECREF(resobj); + if (res < 0) { + goto finally; + } } } + int exitres; finally: // Clean up and switch back. - (void)_PyXI_Exit(session, &result); + exitres = _PyXI_Exit(session, errcode, &result); + assert(res == 0 || exitres != 0); _PyXI_FreeSession(session); - if (res < 0) { - runres->excinfo = Py_NewRef(result.excinfo); - runres->badtarget = badtarget; + + res = exitres; + if (_PyErr_Occurred(tstate)) { + assert(res < 0); } - else if (result.excinfo != NULL) { + else if (res < 0) { + assert(result.excinfo != NULL); runres->excinfo = Py_NewRef(result.excinfo); - runres->badtarget = badtarget; res = -1; } else { + assert(result.excinfo == NULL); runres->result = _PyXI_GetPreserved(&result, "resobj"); if (_PyErr_Occurred(tstate)) { res = -1; @@ -1021,7 +1064,7 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) // Clean up and switch back. assert(!PyErr_Occurred()); - int res = _PyXI_Exit(session, NULL); + int res = _PyXI_Exit(session, _PyXI_ERR_NO_ERROR, NULL); _PyXI_FreeSession(session); assert(res == 0); if (res < 0) { @@ -1041,48 +1084,16 @@ PyDoc_STRVAR(set___main___attrs_doc, Bind the given attributes in the interpreter's __main__ module."); -static void -unwrap_not_shareable(PyThreadState *tstate) -{ - PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); - if (!_PyErr_ExceptionMatches(tstate, exctype)) { - return; - } - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *cause = PyException_GetCause(exc); - if (cause != NULL) { - Py_DECREF(exc); - exc = cause; - } - else { - assert(PyException_GetContext(exc) == NULL); - } - _PyErr_SetRaisedException(tstate, exc); -} - static PyObject * -_handle_script_error(_PyXIData_t *script, struct run_result *runres) +_handle_script_error(struct run_result *runres) { + assert(runres->result == NULL); if (runres->excinfo == NULL) { assert(PyErr_Occurred()); - assert(!runres->badtarget); return NULL; } assert(!PyErr_Occurred()); - if (!runres->badtarget) { - return runres->excinfo; - } - PyObject *code = _PyXIData_NewObject(script); - if (code != NULL) { - // Either the problem is intermittent or only affects subinterpreters. - // This is highly unlikely. - return runres->excinfo; - } - assert(PyErr_Occurred()); - PyThreadState *tstate = _PyThreadState_GET(); - unwrap_not_shareable(tstate); - Py_DECREF(runres->excinfo); - return NULL; + return runres->excinfo; } static PyObject * @@ -1122,9 +1133,9 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return _handle_script_error(&xidata, &runres); + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; #undef FUNCNAME } @@ -1186,9 +1197,9 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return _handle_script_error(&xidata, &runres); + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; #undef FUNCNAME } @@ -1249,9 +1260,9 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) tstate, interp, &xidata, NULL, shared, &runres); _PyXIData_Release(&xidata); if (res < 0) { - assert((runres.excinfo == NULL) != (PyErr_Occurred() == NULL)); - return _handle_script_error(&xidata, &runres); + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; #undef FUNCNAME } @@ -1265,36 +1276,6 @@ are not supported. Methods and other callables are not supported either.\n\ \n\ (See " MODULE_NAME_STR ".exec()."); -static int -handle_call_error(struct interp_call *call, struct run_result *runres) -{ - if (runres->excinfo == NULL) { - assert(PyErr_Occurred()); - assert(!runres->badtarget); - return -1; - } - assert(!PyErr_Occurred()); - if (!runres->badtarget) { - return 0; - } - - // Handle the case where _PyXIData_NewObject() fails. - PyObject *func, *args, *kwargs; - if (_interp_call_unpack(call, &func, &args, &kwargs) == 0) { - // Either the problem is intermittent or only affects subinterpreters. - // This is highly unlikely. - return 0; - } - Py_DECREF(func); - Py_XDECREF(args); - Py_XDECREF(kwargs); - assert(PyErr_Occurred()); - PyThreadState *tstate = _PyThreadState_GET(); - unwrap_not_shareable(tstate); - Py_CLEAR(runres->excinfo); - return -1; -} - static PyObject * interp_call(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1332,9 +1313,11 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject *res_and_exc = NULL; struct run_result runres = {0}; if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) { - if (handle_call_error(&call, &runres) < 0) { + if (runres.excinfo == NULL) { + assert(_PyErr_Occurred(tstate)); goto finally; } + assert(!_PyErr_Occurred(tstate)); } assert(runres.result == NULL || runres.excinfo == NULL); res_and_exc = Py_BuildValue("OO", diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 1971aa47d4b1d7..687f872cbd8b0e 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -70,6 +70,17 @@ runpy_run_path(const char *filename, const char *modname) } +static void +set_exc_with_cause(PyObject *exctype, const char *msg) +{ + PyObject *cause = PyErr_GetRaisedException(); + PyErr_SetString(exctype, msg); + PyObject *exc = PyErr_GetRaisedException(); + PyException_SetCause(exc, cause); + PyErr_SetRaisedException(exc); +} + + static PyObject * pyerr_get_message(PyObject *exc) { @@ -1314,7 +1325,7 @@ _excinfo_normalize_type(struct _excinfo_type *info, } static void -_PyXI_excinfo_Clear(_PyXI_excinfo *info) +_PyXI_excinfo_clear(_PyXI_excinfo *info) { _excinfo_clear_type(&info->type); if (info->msg != NULL) { @@ -1364,7 +1375,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) assert(exc != NULL); if (PyErr_GivenExceptionMatches(exc, PyExc_MemoryError)) { - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return NULL; } const char *failure = NULL; @@ -1410,7 +1421,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1461,7 +1472,7 @@ _PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1656,7 +1667,7 @@ _PyXI_ExcInfoAsObject(_PyXI_excinfo *info) void _PyXI_ClearExcInfo(_PyXI_excinfo *info) { - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); } @@ -1694,6 +1705,14 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) PyErr_SetString(PyExc_InterpreterError, "failed to apply namespace to __main__"); break; + case _PyXI_ERR_PRESERVE_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to preserve objects across session"); + break; + case _PyXI_ERR_EXC_PROPAGATION_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to transfer exception between interpreters"); + break; case _PyXI_ERR_NOT_SHAREABLE: _set_xid_lookup_failure(tstate, NULL, NULL, NULL); break; @@ -1743,7 +1762,7 @@ _PyXI_InitError(_PyXI_error *error, PyObject *excobj, _PyXI_errcode code) assert(excobj == NULL); assert(code != _PyXI_ERR_NO_ERROR); error->code = code; - _PyXI_excinfo_Clear(&error->uncaught); + _PyXI_excinfo_clear(&error->uncaught); } return failure; } @@ -1753,7 +1772,7 @@ _PyXI_ApplyError(_PyXI_error *error) { PyThreadState *tstate = PyThreadState_Get(); if (error->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // Raise an exception that proxies the propagated exception. + // We will raise an exception that proxies the propagated exception. return _PyXI_excinfo_AsObject(&error->uncaught); } else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { @@ -2145,11 +2164,11 @@ _create_sharedns(PyObject *names) return NULL; } -static void _propagate_not_shareable_error(_PyXI_session *); +static void _propagate_not_shareable_error(_PyXI_errcode *); static int _fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, - xidata_fallback_t fallback, _PyXI_session *session) + xidata_fallback_t fallback, _PyXI_errcode *p_errcode) { // All items are expected to be shareable. assert(_sharedns_check_counts(ns)); @@ -2157,8 +2176,8 @@ _fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, assert(ns->numvalues == 0); for (Py_ssize_t i=0; i < ns->maxitems; i++) { if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj, fallback) < 0) { - if (session != NULL) { - _propagate_not_shareable_error(session); + if (p_errcode != NULL) { + _propagate_not_shareable_error(p_errcode); } // Clear out the ones we set so far. for (Py_ssize_t j=0; j < i; j++) { @@ -2315,9 +2334,8 @@ _session_pop_error(_PyXI_session *session, struct xi_session_error *err) return 1; } -static int _ensure_main_ns(_PyXI_session *); +static int _ensure_main_ns(_PyXI_session *, _PyXI_errcode *); static inline void _session_set_error(_PyXI_session *, _PyXI_errcode); -static void _capture_current_exception(_PyXI_session *); /* enter/exit a cross-interpreter session */ @@ -2399,9 +2417,9 @@ _exit_session(_PyXI_session *session) } static void -_propagate_not_shareable_error(_PyXI_session *session) +_propagate_not_shareable_error(_PyXI_errcode *p_errcode) { - assert(session != NULL); + assert(p_errcode != NULL); PyThreadState *tstate = PyThreadState_Get(); PyObject *exctype = get_notshareableerror_type(tstate); if (exctype == NULL) { @@ -2411,7 +2429,7 @@ _propagate_not_shareable_error(_PyXI_session *session) } if (PyErr_ExceptionMatches(exctype)) { // We want to propagate the exception directly. - _session_set_error(session, _PyXI_ERR_NOT_SHAREABLE); + *p_errcode = _PyXI_ERR_NOT_SHAREABLE; } } @@ -2425,18 +2443,32 @@ _PyXI_Enter(_PyXI_session *session, if (nsupdates != NULL) { Py_ssize_t len = PyDict_Size(nsupdates); if (len < 0) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } return -1; } if (len > 0) { sharedns = _create_sharedns(nsupdates); if (sharedns == NULL) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } return -1; } // For now we limit it to shareable objects. xidata_fallback_t fallback = _PyXIDATA_XIDATA_ONLY; - if (_fill_sharedns(sharedns, nsupdates, fallback, NULL) < 0) { + _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; + if (_fill_sharedns(sharedns, nsupdates, fallback, &errcode) < 0) { + assert(PyErr_Occurred()); assert(session->error.info == NULL); + if (errcode == _PyXI_ERR_NO_ERROR) { + errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } _destroy_sharedns(sharedns); + if (result != NULL) { + result->errcode = errcode; + } return -1; } } @@ -2458,8 +2490,7 @@ _PyXI_Enter(_PyXI_session *session, // Apply the cross-interpreter data. if (sharedns != NULL) { - if (_ensure_main_ns(session) < 0) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; + if (_ensure_main_ns(session, &errcode) < 0) { goto error; } if (_apply_sharedns(sharedns, session->main_ns, NULL) < 0) { @@ -2479,6 +2510,7 @@ _PyXI_Enter(_PyXI_session *session, _session_set_error(session, errcode); assert(!PyErr_Occurred()); + // Exit the session. struct xi_session_error err; (void)_session_pop_error(session, &err); _exit_session(session); @@ -2489,6 +2521,7 @@ _PyXI_Enter(_PyXI_session *session, // Apply the error from the other interpreter. PyObject *excinfo = _PyXI_ApplyError(err.info); + _PyXI_excinfo_clear(&err.info->uncaught); if (excinfo != NULL) { if (result != NULL) { result->excinfo = excinfo; @@ -2504,52 +2537,93 @@ _PyXI_Enter(_PyXI_session *session, return -1; } -static PyObject * _capture_preserved(_PyXI_session *); +static int _pop_preserved(_PyXI_session *, _PyXI_namespace **, PyObject **, + _PyXI_errcode *); +static int _finish_preserved(_PyXI_namespace *, PyObject **); int -_PyXI_Exit(_PyXI_session *session, _PyXI_session_result *result) +_PyXI_Exit(_PyXI_session *session, _PyXI_errcode errcode, + _PyXI_session_result *result) { - _capture_current_exception(session); - assert(!PyErr_Occurred()); + int res = 0; + + // Capture the raised exception, if any. + assert(session->error.info == NULL); + if (PyErr_Occurred()) { + _session_set_error(session, errcode); + assert(!PyErr_Occurred()); + } + else { + assert(errcode == _PyXI_ERR_NO_ERROR); + assert(session->error.override == NULL); + } + // Capture the preserved namespace. + _PyXI_namespace *preserved = NULL; + PyObject *preservedobj = NULL; if (result != NULL) { - result->preserved = _capture_preserved(session); - if (result->preserved == NULL && PyErr_Occurred()) { + errcode = _PyXI_ERR_NO_ERROR; + if (_pop_preserved(session, &preserved, &preservedobj, &errcode) < 0) { if (session->error.info != NULL) { + // XXX Chain the exception (i.e. set __context__)? PyErr_FormatUnraisable( "Exception ignored while capturing preserved objects"); } else { - _capture_current_exception(session); + _session_set_error(session, errcode); } } } + // Exit the session. struct xi_session_error err; (void)_session_pop_error(session, &err); _exit_session(session); - if (err.info == NULL) { - return 0; - } - - // Apply the error from the other interpreter. - PyObject *excinfo = _PyXI_ApplyError(err.info); - if (excinfo == NULL) { - assert(PyErr_Occurred()); - if (result != NULL) { - *result = (_PyXI_session_result){0}; + // Restore the preserved namespace. + assert(preserved == NULL || preservedobj == NULL); + if (_finish_preserved(preserved, &preservedobj) < 0) { + assert(preservedobj == NULL); + if (err.info != NULL) { + // XXX Chain the exception (i.e. set __context__)? + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + } + else { + errcode = _PyXI_ERR_PRESERVE_FAILURE; + _propagate_not_shareable_error(&errcode); } } - else if (result != NULL) { - result->excinfo = excinfo; - } - else { + if (result != NULL) { + result->preserved = preservedobj; + result->errcode = errcode; + } + + // Apply the error from the other interpreter, if any. + if (err.info != NULL) { + res = -1; + assert(!PyErr_Occurred()); + PyObject *excinfo = _PyXI_ApplyError(err.info); + _PyXI_excinfo_clear(&err.info->uncaught); + if (excinfo == NULL) { + assert(PyErr_Occurred()); + if (result != NULL) { + _PyXI_ClearResult(result); + *result = (_PyXI_session_result){ + .errcode = _PyXI_ERR_EXC_PROPAGATION_FAILURE, + }; + } + } + else if (result != NULL) { + result->excinfo = excinfo; + } + else { #ifdef Py_DEBUG - fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded"); + fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded"); #endif + } } - return -1; + return res; } @@ -2624,6 +2698,10 @@ _session_set_error(_PyXI_session *session, _PyXI_errcode errcode) { assert(_session_is_active(session)); assert(PyErr_Occurred()); + if (errcode == _PyXI_ERR_NO_ERROR) { + // We're a bit forgiving here. + errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } if (errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION) { session->error._override = errcode; session->error.override = &session->error._override; @@ -2632,7 +2710,7 @@ _session_set_error(_PyXI_session *session, _PyXI_errcode errcode) } static int -_ensure_main_ns(_PyXI_session *session) +_ensure_main_ns(_PyXI_session *session, _PyXI_errcode *p_errcode) { assert(_session_is_active(session)); if (session->main_ns != NULL) { @@ -2641,11 +2719,17 @@ _ensure_main_ns(_PyXI_session *session) // Cache __main__.__dict__. PyObject *main_mod = _Py_GetMainModule(session->init_tstate); if (_Py_CheckMainModule(main_mod) < 0) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_MAIN_NS_FAILURE; + } return -1; } PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_MAIN_NS_FAILURE; + } return -1; } session->main_ns = Py_NewRef(ns); @@ -2653,90 +2737,108 @@ _ensure_main_ns(_PyXI_session *session) } PyObject * -_PyXI_GetMainNamespace(_PyXI_session *session) +_PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_errcode *p_errcode) { if (!_session_is_active(session)) { PyErr_SetString(PyExc_RuntimeError, "session not active"); return NULL; } - if (_ensure_main_ns(session) < 0) { - _session_set_error(session, _PyXI_ERR_MAIN_NS_FAILURE); - _capture_current_exception(session); + if (_ensure_main_ns(session, p_errcode) < 0) { return NULL; } return session->main_ns; } -static PyObject * -_capture_preserved(_PyXI_session *session) +static int +_pop_preserved(_PyXI_session *session, + _PyXI_namespace **p_xidata, PyObject **p_obj, + _PyXI_errcode *p_errcode) { assert(_PyThreadState_GET() == session->init_tstate); // active session + if (session->_preserved == NULL) { + *p_xidata = NULL; + *p_obj = NULL; + return 0; + } if (session->init_tstate == session->prev_tstate) { - // didn't switch - return Py_XNewRef(session->_preserved); + // We did not switch interpreters. + *p_xidata = NULL; + *p_obj = session->_preserved; + session->_preserved = NULL; + return 0; } + *p_obj = NULL; - _PyXI_namespace *preserved = NULL; - if (session->_preserved != NULL) { - Py_ssize_t len = PyDict_Size(session->_preserved); - if (len < 0) { - return NULL; + // We did switch interpreters. + Py_ssize_t len = PyDict_Size(session->_preserved); + if (len < 0) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; } - if (len > 0) { - preserved = _create_sharedns(session->_preserved); - if (preserved == NULL) { - return NULL; + return -1; + } + else if (len == 0) { + *p_xidata = NULL; + } + else { + _PyXI_namespace *xidata = _create_sharedns(session->_preserved); + if (xidata == NULL) { + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; } - if (_fill_sharedns(preserved, session->_preserved, - _PyXIDATA_FULL_FALLBACK, NULL) < 0) - { - assert(session->error.info == NULL); - _destroy_sharedns(preserved); - return NULL; + return -1; + } + _PyXI_errcode errcode = _PyXI_ERR_NO_ERROR; + if (_fill_sharedns(xidata, session->_preserved, + _PyXIDATA_FULL_FALLBACK, &errcode) < 0) + { + assert(session->error.info == NULL); + if (errcode != _PyXI_ERR_NOT_SHAREABLE) { + errcode = _PyXI_ERR_PRESERVE_FAILURE; } + if (p_errcode != NULL) { + *p_errcode = errcode; + } + _destroy_sharedns(xidata); + return -1; } - Py_CLEAR(session->_preserved); + *p_xidata = xidata; } - if (preserved == NULL) { - return NULL; - } - - // We need to switch back to the original interpreter long enough - // to restore the preservd objects. - (void)PyThreadState_Swap(session->prev_tstate); + Py_CLEAR(session->_preserved); + return 0; +} - PyObject *ns = PyDict_New(); - if (ns == NULL) { - goto finally; +static int +_finish_preserved(_PyXI_namespace *xidata, PyObject **p_preserved) +{ + if (xidata == NULL) { + return 0; } - if (_apply_sharedns(preserved, ns, NULL) < 0) { - Py_CLEAR(ns); - goto finally; + int res = -1; + if (p_preserved != NULL) { + PyObject *ns = PyDict_New(); + if (ns == NULL) { + goto finally; + } + if (_apply_sharedns(xidata, ns, NULL) < 0) { + Py_CLEAR(ns); + goto finally; + } + *p_preserved = ns; } + res = 0; finally: - // Swap back into the session. - (void)PyThreadState_Swap(session->init_tstate); - assert(preserved != NULL); - _destroy_sharedns(preserved); - return ns; -} - -static void -set_exc_with_cause(PyObject *exctype, const char *msg) -{ - PyObject *cause = PyErr_GetRaisedException(); - PyErr_SetString(exctype, msg); - PyObject *exc = PyErr_GetRaisedException(); - PyException_SetCause(exc, cause); - PyErr_SetRaisedException(exc); + _destroy_sharedns(xidata); + return res; } int -_PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value) +_PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value, + _PyXI_errcode *p_errcode) { - if (session->init_tstate == NULL) { + if (!_session_is_active(session)) { PyErr_SetString(PyExc_RuntimeError, "session not active"); return -1; } @@ -2745,11 +2847,17 @@ _PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value) if (session->_preserved == NULL) { set_exc_with_cause(PyExc_RuntimeError, "failed to initialize preserved objects"); + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; + } return -1; } } if (PyDict_SetItemString(session->_preserved, name, value) < 0) { set_exc_with_cause(PyExc_RuntimeError, "failed to preserve object"); + if (p_errcode != NULL) { + *p_errcode = _PyXI_ERR_PRESERVE_FAILURE; + } return -1; } return 0; From 2682157bbd54765039f651e549bd0a121bbe077d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 May 2025 14:01:07 -0600 Subject: [PATCH 11/11] Drop a stale comment. --- Python/crossinterp.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 687f872cbd8b0e..5e73ab28f2b663 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -2412,7 +2412,6 @@ _exit_session(_PyXI_session *session) assert(session->error.info == NULL); assert(session->error.override == _PyXI_ERR_NO_ERROR); - // For now the error data persists past the exit. *session = (_PyXI_session){0}; }