From b0ac76745abd8aa3c24220bc15f0c212c36010d7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 15 Apr 2024 20:17:53 -0700 Subject: [PATCH 1/5] Fix a bug with CALL_STAT_INC We were under-counting calls in `_PyEvalFramePushAndInit` because the `CALL_STAT_INC` macro was redefined to a no-op for the Tier 2 interpreter. The fix is a little convoluted. This ought to result in ~37% more "Frames pushed" reported under "Call stats". The new count is the correct one (I presume). --- Include/internal/pycore_code.h | 7 +++++-- Python/ceval.c | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 1ec0348d6e5e8b..8deab052afc0ec 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -300,7 +300,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) #define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) #define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) +#define REAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) #define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) #define OBJECT_STAT_INC_COND(name, cond) \ do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) @@ -336,7 +336,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define STAT_INC(opname, name) ((void)0) #define STAT_DEC(opname, name) ((void)0) #define OPCODE_EXE_INC(opname) ((void)0) -#define CALL_STAT_INC(name) ((void)0) +#define REAL_CALL_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) @@ -351,6 +351,9 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define RARE_EVENT_STAT_INC(name) ((void)0) #endif // !Py_STATS +// We do a little dance here so we can redefine and restore CALL_STAT_INC +#define CALL_STAT_INC(name) REAL_CALL_STAT_INC(name) + // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and // performant! They should compile to just "move" instructions on all supported diff --git a/Python/ceval.c b/Python/ceval.c index c0783f7377a8ee..1826a2d9c6d985 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1110,6 +1110,17 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif // _Py_JIT +// Undefine the macros we redefined, to avoid using the wrong ones below +#undef LOAD_IP +#undef GOTO_ERROR +#undef ENABLE_SPECIALIZATION +#undef STAT_INC +#undef STAT_DEC +#undef CALL_STAT_INC + +// Restore this one +#define CALL_STAT_INC(name) REAL_CALL_STAT_INC(name) + } #if defined(__GNUC__) From 779f7f4ee59ebde0b4f24984c99ecb1a41def44e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 16 Apr 2024 14:22:55 -0700 Subject: [PATCH 2/5] Revert "Fix a bug with CALL_STAT_INC" Let's try a different fix instead. --- Include/internal/pycore_code.h | 7 ++----- Python/ceval.c | 11 ----------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 8deab052afc0ec..1ec0348d6e5e8b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -300,7 +300,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) #define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) #define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) -#define REAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) +#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) #define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) #define OBJECT_STAT_INC_COND(name, cond) \ do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) @@ -336,7 +336,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define STAT_INC(opname, name) ((void)0) #define STAT_DEC(opname, name) ((void)0) #define OPCODE_EXE_INC(opname) ((void)0) -#define REAL_CALL_STAT_INC(name) ((void)0) +#define CALL_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) @@ -351,9 +351,6 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define RARE_EVENT_STAT_INC(name) ((void)0) #endif // !Py_STATS -// We do a little dance here so we can redefine and restore CALL_STAT_INC -#define CALL_STAT_INC(name) REAL_CALL_STAT_INC(name) - // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and // performant! They should compile to just "move" instructions on all supported diff --git a/Python/ceval.c b/Python/ceval.c index 1826a2d9c6d985..c0783f7377a8ee 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1110,17 +1110,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif // _Py_JIT -// Undefine the macros we redefined, to avoid using the wrong ones below -#undef LOAD_IP -#undef GOTO_ERROR -#undef ENABLE_SPECIALIZATION -#undef STAT_INC -#undef STAT_DEC -#undef CALL_STAT_INC - -// Restore this one -#define CALL_STAT_INC(name) REAL_CALL_STAT_INC(name) - } #if defined(__GNUC__) From 34584c3fca13bd1a560a0c52da373a154d9ffd10 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 16 Apr 2024 14:34:48 -0700 Subject: [PATCH 3/5] Move _PyEval_EvalFrameDefault to the end of the ceval.c This should fix the issue with CALL_STAT_INC in a cleaner way (even if the diff is much larger). --- Python/ceval.c | 3436 ++++++++++++++++++++++++------------------------ 1 file changed, 1718 insertions(+), 1718 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index c0783f7377a8ee..0fa94977de7230 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -50,6 +50,8 @@ # error "ceval.c must be build with Py_BUILD_CORE define for best performance" #endif +#include "ceval_macros.h" + #if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED) // GH-89279: The MSVC compiler does not inline these static inline functions // in PGO build in _PyEval_EvalFrameDefault(), because this function is over @@ -603,1907 +605,1391 @@ PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) return res; } - -/* Interpreter main loop */ - -PyObject * -PyEval_EvalFrame(PyFrameObject *f) +static void +format_missing(PyThreadState *tstate, const char *kind, + PyCodeObject *co, PyObject *names, PyObject *qualname) { - /* Function kept for backward compatibility */ - PyThreadState *tstate = _PyThreadState_GET(); - return _PyEval_EvalFrame(tstate, f->f_frame, 0); + int err; + Py_ssize_t len = PyList_GET_SIZE(names); + PyObject *name_str, *comma, *tail, *tmp; + + assert(PyList_CheckExact(names)); + assert(len >= 1); + /* Deal with the joys of natural language. */ + switch (len) { + case 1: + name_str = PyList_GET_ITEM(names, 0); + Py_INCREF(name_str); + break; + case 2: + name_str = PyUnicode_FromFormat("%U and %U", + PyList_GET_ITEM(names, len - 2), + PyList_GET_ITEM(names, len - 1)); + break; + default: + tail = PyUnicode_FromFormat(", %U, and %U", + PyList_GET_ITEM(names, len - 2), + PyList_GET_ITEM(names, len - 1)); + if (tail == NULL) + return; + /* Chop off the last two objects in the list. This shouldn't actually + fail, but we can't be too careful. */ + err = PyList_SetSlice(names, len - 2, len, NULL); + if (err == -1) { + Py_DECREF(tail); + return; + } + /* Stitch everything up into a nice comma-separated list. */ + comma = PyUnicode_FromString(", "); + if (comma == NULL) { + Py_DECREF(tail); + return; + } + tmp = PyUnicode_Join(comma, names); + Py_DECREF(comma); + if (tmp == NULL) { + Py_DECREF(tail); + return; + } + name_str = PyUnicode_Concat(tmp, tail); + Py_DECREF(tmp); + Py_DECREF(tail); + break; + } + if (name_str == NULL) + return; + _PyErr_Format(tstate, PyExc_TypeError, + "%U() missing %i required %s argument%s: %U", + qualname, + len, + kind, + len == 1 ? "" : "s", + name_str); + Py_DECREF(name_str); } -PyObject * -PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) +static void +missing_arguments(PyThreadState *tstate, PyCodeObject *co, + Py_ssize_t missing, Py_ssize_t defcount, + PyObject **localsplus, PyObject *qualname) { - PyThreadState *tstate = _PyThreadState_GET(); - return _PyEval_EvalFrame(tstate, f->f_frame, throwflag); -} + Py_ssize_t i, j = 0; + Py_ssize_t start, end; + int positional = (defcount != -1); + const char *kind = positional ? "positional" : "keyword-only"; + PyObject *missing_names; -#include "ceval_macros.h" + /* Compute the names of the arguments that are missing. */ + missing_names = PyList_New(missing); + if (missing_names == NULL) + return; + if (positional) { + start = 0; + end = co->co_argcount - defcount; + } + else { + start = co->co_argcount; + end = start + co->co_kwonlyargcount; + } + for (i = start; i < end; i++) { + if (localsplus[i] == NULL) { + PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *name = PyObject_Repr(raw); + if (name == NULL) { + Py_DECREF(missing_names); + return; + } + PyList_SET_ITEM(missing_names, j++, name); + } + } + assert(j == missing); + format_missing(tstate, kind, co, missing_names, qualname); + Py_DECREF(missing_names); +} -int _Py_CheckRecursiveCallPy( - PyThreadState *tstate) +static void +too_many_positional(PyThreadState *tstate, PyCodeObject *co, + Py_ssize_t given, PyObject *defaults, + PyObject **localsplus, PyObject *qualname) { - if (tstate->recursion_headroom) { - if (tstate->py_recursion_remaining < -50) { - /* Overflowing while handling an overflow. Give up. */ - Py_FatalError("Cannot recover from Python stack overflow."); + int plural; + Py_ssize_t kwonly_given = 0; + Py_ssize_t i; + PyObject *sig, *kwonly_sig; + Py_ssize_t co_argcount = co->co_argcount; + + assert((co->co_flags & CO_VARARGS) == 0); + /* Count missing keyword-only args. */ + for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { + if (localsplus[i] != NULL) { + kwonly_given++; } } + Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults); + if (defcount) { + Py_ssize_t atleast = co_argcount - defcount; + plural = 1; + sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount); + } else { - if (tstate->py_recursion_remaining <= 0) { - tstate->recursion_headroom++; - _PyErr_Format(tstate, PyExc_RecursionError, - "maximum recursion depth exceeded"); - tstate->recursion_headroom--; - return -1; + plural = (co_argcount != 1); + sig = PyUnicode_FromFormat("%zd", co_argcount); + } + if (sig == NULL) + return; + if (kwonly_given) { + const char *format = " positional argument%s (and %zd keyword-only argument%s)"; + kwonly_sig = PyUnicode_FromFormat(format, + given != 1 ? "s" : "", + kwonly_given, + kwonly_given != 1 ? "s" : ""); + if (kwonly_sig == NULL) { + Py_DECREF(sig); + return; } } - return 0; + else { + /* This will not fail. */ + kwonly_sig = PyUnicode_FromString(""); + assert(kwonly_sig != NULL); + } + _PyErr_Format(tstate, PyExc_TypeError, + "%U() takes %U positional argument%s but %zd%U %s given", + qualname, + sig, + plural ? "s" : "", + given, + kwonly_sig, + given == 1 && !kwonly_given ? "was" : "were"); + Py_DECREF(sig); + Py_DECREF(kwonly_sig); } -static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { - /* Put a NOP at the start, so that the IP points into - * the code, rather than before it */ - { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ - { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ - { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } -}; +static int +positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, + Py_ssize_t kwcount, PyObject* kwnames, + PyObject *qualname) +{ + int posonly_conflicts = 0; + PyObject* posonly_names = PyList_New(0); + if (posonly_names == NULL) { + goto fail; + } + for(int k=0; k < co->co_posonlyargcount; k++){ + PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k); -extern const struct _PyCode_DEF(8) _Py_InitCleanup; + for (int k2=0; k2 0) { + if(PyList_Append(posonly_names, kwname) != 0) { + goto fail; + } + posonly_conflicts++; + } else if (cmp < 0) { + goto fail; + } -/* Disable unused label warnings. They are handy for debugging, even - if computed gotos aren't used. */ + } + } + if (posonly_conflicts) { + PyObject* comma = PyUnicode_FromString(", "); + if (comma == NULL) { + goto fail; + } + PyObject* error_names = PyUnicode_Join(comma, posonly_names); + Py_DECREF(comma); + if (error_names == NULL) { + goto fail; + } + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got some positional-only arguments passed" + " as keyword arguments: '%U'", + qualname, error_names); + Py_DECREF(error_names); + goto fail; + } -/* TBD - what about other compilers? */ -#if defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-label" -#elif defined(_MSC_VER) /* MS_WINDOWS */ -# pragma warning(push) -# pragma warning(disable:4102) -#endif + Py_DECREF(posonly_names); + return 0; +fail: + Py_XDECREF(posonly_names); + return 1; -/* _PyEval_EvalFrameDefault() is a *big* function, - * so consume 3 units of C stack */ -#define PY_EVAL_C_STACK_UNITS 2 - -#if defined(_MSC_VER) && defined(_Py_USING_PGO) -/* gh-111786: _PyEval_EvalFrameDefault is too large to optimize for speed with - PGO on MSVC. Disable that optimization temporarily. If this is fixed - upstream, we should gate this on the version of MSVC. - */ -# pragma optimize("t", off) -/* This setting is reversed below following _PyEval_EvalFrameDefault */ -#endif - -PyObject* _Py_HOT_FUNCTION -_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) -{ - _Py_EnsureTstateNotNULL(tstate); - CALL_STAT_INC(pyeval_calls); +} -#if USE_COMPUTED_GOTOS -/* Import the static jump table */ -#include "opcode_targets.h" -#endif -#ifdef Py_STATS - int lastopcode = 0; -#endif - uint8_t opcode; /* Current opcode */ - int oparg; /* Current opcode argument, if any */ -#ifdef LLTRACE - int lltrace = 0; -#endif +static inline unsigned char * +scan_back_to_entry_start(unsigned char *p) { + for (; (p[0]&128) == 0; p--); + return p; +} - _PyInterpreterFrame entry_frame; +static inline unsigned char * +skip_to_next_entry(unsigned char *p, unsigned char *end) { + while (p < end && ((p[0] & 128) == 0)) { + p++; + } + return p; +} +#define MAX_LINEAR_SEARCH 40 -#ifdef Py_DEBUG - /* Set these to invalid but identifiable values for debugging. */ - entry_frame.f_funcobj = (PyObject*)0xaaa0; - entry_frame.f_locals = (PyObject*)0xaaa1; - entry_frame.frame_obj = (PyFrameObject*)0xaaa2; - entry_frame.f_globals = (PyObject*)0xaaa3; - entry_frame.f_builtins = (PyObject*)0xaaa4; -#endif - entry_frame.f_executable = Py_None; - entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - entry_frame.stacktop = 0; - entry_frame.owner = FRAME_OWNED_BY_CSTACK; - entry_frame.return_offset = 0; - /* Push frame */ - entry_frame.previous = tstate->current_frame; - frame->previous = &entry_frame; - tstate->current_frame = frame; +static int +get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) +{ + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); + /* Invariants: + * start_table == end_table OR + * start_table points to a legal entry and end_table points + * beyond the table or to a legal entry that is after index. + */ + if (end - start > MAX_LINEAR_SEARCH) { + int offset; + parse_varint(start, &offset); + if (offset > index) { + return 0; + } + do { + unsigned char * mid = start + ((end-start)>>1); + mid = scan_back_to_entry_start(mid); + parse_varint(mid, &offset); + if (offset > index) { + end = mid; + } + else { + start = mid; + } - tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); - if (_Py_EnterRecursiveCallTstate(tstate, "")) { - tstate->c_recursion_remaining--; - tstate->py_recursion_remaining--; - goto exit_unwind; + } while (end - start > MAX_LINEAR_SEARCH); } - - /* support for generator.throw() */ - if (throwflag) { - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; + unsigned char *scan = start; + while (scan < end) { + int start_offset, size; + scan = parse_varint(scan, &start_offset); + if (start_offset > index) { + break; } - /* Because this avoids the RESUME, - * we need to update instrumentation */ - _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - monitor_throw(tstate, frame, frame->instr_ptr); - /* TO DO -- Monitor throw entry. */ - goto resume_with_error; + scan = parse_varint(scan, &size); + if (start_offset + size > index) { + scan = parse_varint(scan, handler); + int depth_and_lasti; + parse_varint(scan, &depth_and_lasti); + *level = depth_and_lasti >> 1; + *lasti = depth_and_lasti & 1; + return 1; + } + scan = skip_to_next_entry(scan, end); } + return 0; +} - /* Local "register" variables. - * These are cached values from the frame and code object. */ - _Py_CODEUNIT *next_instr; - PyObject **stack_pointer; - -#ifndef _Py_JIT - /* Tier 2 interpreter state */ - _PyExecutorObject *current_executor = NULL; - const _PyUOpInstruction *next_uop = NULL; -#endif +static int +initialize_locals(PyThreadState *tstate, PyFunctionObject *func, + PyObject **localsplus, PyObject *const *args, + Py_ssize_t argcount, PyObject *kwnames) +{ + PyCodeObject *co = (PyCodeObject*)func->func_code; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; -start_frame: - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; + /* Create a dictionary for keyword parameters (**kwags) */ + PyObject *kwdict; + Py_ssize_t i; + if (co->co_flags & CO_VARKEYWORDS) { + kwdict = PyDict_New(); + if (kwdict == NULL) { + goto fail_pre_positional; + } + i = total_args; + if (co->co_flags & CO_VARARGS) { + i++; + } + assert(localsplus[i] == NULL); + localsplus[i] = kwdict; } - - next_instr = frame->instr_ptr; -resume_frame: - stack_pointer = _PyFrame_GetStackPointer(frame); - -#ifdef LLTRACE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; + else { + kwdict = NULL; } -#endif - -#ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); -#endif - - DISPATCH(); - - { - /* Start instructions */ -#if !USE_COMPUTED_GOTOS - dispatch_opcode: - switch (opcode) -#endif - { -#include "generated_cases.c.h" + /* Copy all positional arguments into local variables */ + Py_ssize_t j, n; + if (argcount > co->co_argcount) { + n = co->co_argcount; + } + else { + n = argcount; + } + for (j = 0; j < n; j++) { + PyObject *x = args[j]; + assert(localsplus[j] == NULL); + localsplus[j] = x; + } - /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, - * because it needs to capture frame->instr_ptr before it is updated, - * as happens in the standard instruction prologue. - */ -#if USE_COMPUTED_GOTOS - TARGET_INSTRUMENTED_LINE: -#else - case INSTRUMENTED_LINE: -#endif - { - _Py_CODEUNIT *prev = frame->instr_ptr; - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; + /* Pack other positional arguments into the *args argument */ + if (co->co_flags & CO_VARARGS) { + PyObject *u = NULL; + if (argcount == n) { + u = (PyObject *)&_Py_SINGLETON(tuple_empty); } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); + else { + assert(args != NULL); + u = _PyTuple_FromArraySteal(args + n, argcount - n); } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); + if (u == NULL) { + goto fail_post_positional; + } + assert(localsplus[total_args] == NULL); + localsplus[total_args] = u; + } + else if (argcount > n) { + /* Too many postional args. Error is reported later */ + for (j = n; j < argcount; j++) { + Py_DECREF(args[j]); } - opcode = original_opcode; - DISPATCH_GOTO(); } + /* Handle keyword arguments */ + if (kwnames != NULL) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < kwcount; i++) { + PyObject **co_varnames; + PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = args[i+argcount]; + Py_ssize_t j; -#if USE_COMPUTED_GOTOS - _unknown_opcode: -#else - EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode -#endif - /* Tell C compilers not to hold the opcode variable in the loop. - next_instr points the current instruction without TARGET(). */ - opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; - - } /* End instructions */ - - /* This should never be reached. Every opcode should end with DISPATCH() - or goto error. */ - Py_UNREACHABLE(); - -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != &entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + if (keyword == NULL || !PyUnicode_Check(keyword)) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() keywords must be strings", + func->func_qualname); + goto kw_fail; } - } - monitor_raise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ - PyObject **stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyObject *o = POP(); - Py_XDECREF(o); + /* Speed hack: do raw pointer compares. As names are + normally interned this should almost always hit. */ + co_varnames = ((PyTupleObject *)(co->co_localsplusnames))->ob_item; + for (j = co->co_posonlyargcount; j < total_args; j++) { + PyObject *varname = co_varnames[j]; + if (varname == keyword) { + goto kw_found; } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; } - assert(STACK_LEVEL() >= level); - PyObject **new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyObject *v = POP(); - Py_XDECREF(v); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; + /* Slow fallback, just in case */ + for (j = co->co_posonlyargcount; j < total_args; j++) { + PyObject *varname = co_varnames[j]; + int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ); + if (cmp > 0) { + goto kw_found; + } + else if (cmp < 0) { + goto kw_fail; } - PUSH(lasti); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(exc); - next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == &entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + assert(j >= total_args); + if (kwdict == NULL) { + if (co->co_posonlyargcount + && positional_only_passed_as_keyword(tstate, co, + kwcount, kwnames, + func->func_qualname)) + { + goto kw_fail; + } -// Tier 2 is also here! -enter_tier_two: - -#ifdef _Py_JIT - assert(0); -#else - -#undef LOAD_IP -#define LOAD_IP(UNUSED) (void)0 + PyObject* suggestion_keyword = NULL; + if (total_args > co->co_posonlyargcount) { + PyObject* possible_keywords = PyList_New(total_args - co->co_posonlyargcount); -#undef GOTO_ERROR -#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two + if (!possible_keywords) { + PyErr_Clear(); + } else { + for (Py_ssize_t k = co->co_posonlyargcount; k < total_args; k++) { + PyList_SET_ITEM(possible_keywords, k - co->co_posonlyargcount, co_varnames[k]); + } -#ifdef Py_STATS -// Disable these macros that apply to Tier 1 stats when we are in Tier 2 -#undef STAT_INC -#define STAT_INC(opname, name) ((void)0) -#undef STAT_DEC -#define STAT_DEC(opname, name) ((void)0) -#undef CALL_STAT_INC -#define CALL_STAT_INC(name) ((void)0) -#endif + suggestion_keyword = _Py_CalculateSuggestions(possible_keywords, keyword); + Py_DECREF(possible_keywords); + } + } -#undef ENABLE_SPECIALIZATION -#define ENABLE_SPECIALIZATION 0 + if (suggestion_keyword) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'. Did you mean '%S'?", + func->func_qualname, keyword, suggestion_keyword); + Py_DECREF(suggestion_keyword); + } else { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'", + func->func_qualname, keyword); + } -#ifdef Py_DEBUG - #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { printf(__VA_ARGS__); } -#else - #define DPRINTF(level, ...) -#endif + goto kw_fail; + } - ; // dummy statement after a label, before a declaration - uint16_t uopcode; -#ifdef Py_STATS - int lastuop = 0; - uint64_t trace_uop_execution_counter = 0; -#endif + if (PyDict_SetItem(kwdict, keyword, value) == -1) { + goto kw_fail; + } + Py_DECREF(value); + continue; - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); -tier2_dispatch: - for (;;) { - uopcode = next_uop->opcode; -#ifdef Py_DEBUG - if (lltrace >= 3) { - if (next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT) { - printf("%4d uop: ", 0); + kw_fail: + for (;i < kwcount; i++) { + PyObject *value = args[i+argcount]; + Py_DECREF(value); } - else { - printf("%4d uop: ", (int)(next_uop - current_executor->trace)); + goto fail_post_args; + + kw_found: + if (localsplus[j] != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got multiple values for argument '%S'", + func->func_qualname, keyword); + goto kw_fail; } - _PyUOpPrint(next_uop); - printf(" stack_level=%d\n", - (int)(stack_pointer - _PyFrame_Stackbase(frame))); + localsplus[j] = value; } -#endif - next_uop++; - OPT_STAT_INC(uops_executed); - UOP_STAT_INC(uopcode, execution_count); - UOP_PAIR_INC(uopcode, lastuop); -#ifdef Py_STATS - trace_uop_execution_counter++; -#endif - - switch (uopcode) { + } -#include "executor_cases.c.h" + /* Check the number of positional arguments */ + if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { + too_many_positional(tstate, co, argcount, func->func_defaults, localsplus, + func->func_qualname); + goto fail_post_args; + } - default: -#ifdef Py_DEBUG - { - printf("Unknown uop: "); - _PyUOpPrint(&next_uop[-1]); - printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); - Py_FatalError("Unknown uop"); + /* Add missing positional arguments (copy default values from defs) */ + if (argcount < co->co_argcount) { + Py_ssize_t defcount = func->func_defaults == NULL ? 0 : PyTuple_GET_SIZE(func->func_defaults); + Py_ssize_t m = co->co_argcount - defcount; + Py_ssize_t missing = 0; + for (i = argcount; i < m; i++) { + if (localsplus[i] == NULL) { + missing++; + } + } + if (missing) { + missing_arguments(tstate, co, missing, defcount, localsplus, + func->func_qualname); + goto fail_post_args; + } + if (n > m) + i = n - m; + else + i = 0; + if (defcount) { + PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0); + for (; i < defcount; i++) { + if (localsplus[m+i] == NULL) { + PyObject *def = defs[i]; + localsplus[m+i] = Py_NewRef(def); + } } -#else - Py_UNREACHABLE(); -#endif - } } -jump_to_error_target: -#ifdef Py_DEBUG - if (lltrace >= 2) { - printf("Error: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(" @ %d -> %s]\n", - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); + /* Add missing keyword arguments (copy default values from kwdefs) */ + if (co->co_kwonlyargcount > 0) { + Py_ssize_t missing = 0; + for (i = co->co_argcount; i < total_args; i++) { + if (localsplus[i] != NULL) + continue; + PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (func->func_kwdefaults != NULL) { + PyObject *def; + if (PyDict_GetItemRef(func->func_kwdefaults, varname, &def) < 0) { + goto fail_post_args; + } + if (def) { + localsplus[i] = def; + continue; + } + } + missing++; + } + if (missing) { + missing_arguments(tstate, co, missing, -1, localsplus, + func->func_qualname); + goto fail_post_args; + } } -#endif - assert (next_uop[-1].format == UOP_FORMAT_JUMP); - uint16_t target = uop_get_error_target(&next_uop[-1]); - next_uop = current_executor->trace + target; - goto tier2_dispatch; - -error_tier_two: - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - assert(next_uop[-1].format == UOP_FORMAT_TARGET); - frame->return_offset = 0; // Don't leave this random - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(current_executor); - tstate->previous_executor = NULL; - goto resume_with_error; - -jump_to_jump_target: - assert(next_uop[-1].format == UOP_FORMAT_JUMP); - target = uop_get_jump_target(&next_uop[-1]); - next_uop = current_executor->trace + target; - goto tier2_dispatch; + return 0; -exit_to_tier1: - assert(next_uop[-1].format == UOP_FORMAT_TARGET); - next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); -#ifdef Py_DEBUG - if (lltrace >= 2) { - printf("DEOPT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(" -> %s]\n", - _PyOpcode_OpName[next_instr->op.code]); +fail_pre_positional: + for (j = 0; j < argcount; j++) { + Py_DECREF(args[j]); } -#endif - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - Py_DECREF(current_executor); - tstate->previous_executor = NULL; - DISPATCH(); - -exit_to_trace: - assert(next_uop[-1].format == UOP_FORMAT_EXIT); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - uint32_t exit_index = next_uop[-1].exit_index; - assert(exit_index < current_executor->exit_count); - _PyExitData *exit = ¤t_executor->exits[exit_index]; -#ifdef Py_DEBUG - if (lltrace >= 2) { - printf("SIDE EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", - exit_index, exit->temperature.as_counter, exit->target, - _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); + /* fall through */ +fail_post_positional: + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (j = argcount; j < argcount+kwcount; j++) { + Py_DECREF(args[j]); + } } -#endif - Py_INCREF(exit->executor); - tstate->previous_executor = (PyObject *)current_executor; - GOTO_TIER_TWO(exit->executor); - -#endif // _Py_JIT - + /* fall through */ +fail_post_args: + return -1; } -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#elif defined(_MSC_VER) /* MS_WINDOWS */ -# pragma warning(pop) -# pragma optimize("", on) -#endif - static void -format_missing(PyThreadState *tstate, const char *kind, - PyCodeObject *co, PyObject *names, PyObject *qualname) +clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { - int err; - Py_ssize_t len = PyList_GET_SIZE(names); - PyObject *name_str, *comma, *tail, *tmp; - - assert(PyList_CheckExact(names)); - assert(len >= 1); - /* Deal with the joys of natural language. */ - switch (len) { - case 1: - name_str = PyList_GET_ITEM(names, 0); - Py_INCREF(name_str); - break; - case 2: - name_str = PyUnicode_FromFormat("%U and %U", - PyList_GET_ITEM(names, len - 2), - PyList_GET_ITEM(names, len - 1)); - break; - default: - tail = PyUnicode_FromFormat(", %U, and %U", - PyList_GET_ITEM(names, len - 2), - PyList_GET_ITEM(names, len - 1)); - if (tail == NULL) - return; - /* Chop off the last two objects in the list. This shouldn't actually - fail, but we can't be too careful. */ - err = PyList_SetSlice(names, len - 2, len, NULL); - if (err == -1) { - Py_DECREF(tail); - return; - } - /* Stitch everything up into a nice comma-separated list. */ - comma = PyUnicode_FromString(", "); - if (comma == NULL) { - Py_DECREF(tail); - return; - } - tmp = PyUnicode_Join(comma, names); - Py_DECREF(comma); - if (tmp == NULL) { - Py_DECREF(tail); - return; - } - name_str = PyUnicode_Concat(tmp, tail); - Py_DECREF(tmp); - Py_DECREF(tail); - break; - } - if (name_str == NULL) - return; - _PyErr_Format(tstate, PyExc_TypeError, - "%U() missing %i required %s argument%s: %U", - qualname, - len, - kind, - len == 1 ? "" : "s", - name_str); - Py_DECREF(name_str); + assert(frame->owner == FRAME_OWNED_BY_THREAD); + // Make sure that this is, indeed, the top frame. We can't check this in + // _PyThreadState_PopFrame, since f_code is already cleared at that point: + assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + tstate->datastack_top); + tstate->c_recursion_remaining--; + assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); + _PyFrame_ClearExceptCode(frame); + Py_DECREF(frame->f_executable); + tstate->c_recursion_remaining++; + _PyThreadState_PopFrame(tstate, frame); } static void -missing_arguments(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t missing, Py_ssize_t defcount, - PyObject **localsplus, PyObject *qualname) +clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { - Py_ssize_t i, j = 0; - Py_ssize_t start, end; - int positional = (defcount != -1); - const char *kind = positional ? "positional" : "keyword-only"; - PyObject *missing_names; + assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_CLEARED; + assert(tstate->exc_info == &gen->gi_exc_state); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + tstate->c_recursion_remaining--; + assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); + _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); + tstate->c_recursion_remaining++; + frame->previous = NULL; +} - /* Compute the names of the arguments that are missing. */ - missing_names = PyList_New(missing); - if (missing_names == NULL) - return; - if (positional) { - start = 0; - end = co->co_argcount - defcount; +void +_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) +{ + if (frame->owner == FRAME_OWNED_BY_THREAD) { + clear_thread_frame(tstate, frame); } else { - start = co->co_argcount; - end = start + co->co_kwonlyargcount; + clear_gen_frame(tstate, frame); } - for (i = start; i < end; i++) { - if (localsplus[i] == NULL) { - PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *name = PyObject_Repr(raw); - if (name == NULL) { - Py_DECREF(missing_names); - return; - } - PyList_SET_ITEM(missing_names, j++, name); +} + +/* Consumes references to func, locals and all the args */ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject* const* args, + size_t argcount, PyObject *kwnames) +{ + PyCodeObject * code = (PyCodeObject *)func->func_code; + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); + if (frame == NULL) { + goto fail; + } + _PyFrame_Initialize(frame, func, locals, code, 0); + if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + clear_thread_frame(tstate, frame); + return NULL; + } + return frame; +fail: + /* Consume the references */ + for (size_t i = 0; i < argcount; i++) { + Py_DECREF(args[i]); + } + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_DECREF(args[i+argcount]); } } - assert(j == missing); - format_missing(tstate, kind, co, missing_names, qualname); - Py_DECREF(missing_names); + PyErr_NoMemory(); + return NULL; } -static void -too_many_positional(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t given, PyObject *defaults, - PyObject **localsplus, PyObject *qualname) +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) { - int plural; - Py_ssize_t kwonly_given = 0; - Py_ssize_t i; - PyObject *sig, *kwonly_sig; - Py_ssize_t co_argcount = co->co_argcount; - - assert((co->co_flags & CO_VARARGS) == 0); - /* Count missing keyword-only args. */ - for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { - if (localsplus[i] != NULL) { - kwonly_given++; + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs; + if (has_dict) { + newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (newargs == NULL) { + Py_DECREF(func); + goto error; } } - Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults); - if (defcount) { - Py_ssize_t atleast = co_argcount - defcount; - plural = 1; - sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount); - } else { - plural = (co_argcount != 1); - sig = PyUnicode_FromFormat("%zd", co_argcount); - } - if (sig == NULL) - return; - if (kwonly_given) { - const char *format = " positional argument%s (and %zd keyword-only argument%s)"; - kwonly_sig = PyUnicode_FromFormat(format, - given != 1 ? "s" : "", - kwonly_given, - kwonly_given != 1 ? "s" : ""); - if (kwonly_sig == NULL) { - Py_DECREF(sig); - return; + newargs = &PyTuple_GET_ITEM(callargs, 0); + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); } } - else { - /* This will not fail. */ - kwonly_sig = PyUnicode_FromString(""); - assert(kwonly_sig != NULL); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); } - _PyErr_Format(tstate, PyExc_TypeError, - "%U() takes %U positional argument%s but %zd%U %s given", - qualname, - sig, - plural ? "s" : "", - given, - kwonly_sig, - given == 1 && !kwonly_given ? "was" : "were"); - Py_DECREF(sig); - Py_DECREF(kwonly_sig); + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; } -static int -positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t kwcount, PyObject* kwnames, - PyObject *qualname) +PyObject * +_PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames) { - int posonly_conflicts = 0; - PyObject* posonly_names = PyList_New(0); - if (posonly_names == NULL) { - goto fail; + /* _PyEvalFramePushAndInit consumes the references + * to func, locals and all its arguments */ + Py_INCREF(func); + Py_XINCREF(locals); + for (size_t i = 0; i < argcount; i++) { + Py_INCREF(args[i]); } - for(int k=0; k < co->co_posonlyargcount; k++){ - PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k); - - for (int k2=0; k2 0) { - if(PyList_Append(posonly_names, kwname) != 0) { - goto fail; - } - posonly_conflicts++; - } else if (cmp < 0) { - goto fail; - } - + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_INCREF(args[i+argcount]); } } - if (posonly_conflicts) { - PyObject* comma = PyUnicode_FromString(", "); - if (comma == NULL) { + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + tstate, func, locals, args, argcount, kwnames); + if (frame == NULL) { + return NULL; + } + EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR); + return _PyEval_EvalFrame(tstate, frame, 0); +} + +/* Legacy API */ +PyObject * +PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, + PyObject *const *args, int argcount, + PyObject *const *kws, int kwcount, + PyObject *const *defs, int defcount, + PyObject *kwdefs, PyObject *closure) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *res = NULL; + PyObject *defaults = _PyTuple_FromArray(defs, defcount); + if (defaults == NULL) { + return NULL; + } + PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + if (builtins == NULL) { + Py_DECREF(defaults); + return NULL; + } + if (locals == NULL) { + locals = globals; + } + PyObject *kwnames = NULL; + PyObject *const *allargs; + PyObject **newargs = NULL; + PyFunctionObject *func = NULL; + if (kwcount == 0) { + allargs = args; + } + else { + kwnames = PyTuple_New(kwcount); + if (kwnames == NULL) { goto fail; } - PyObject* error_names = PyUnicode_Join(comma, posonly_names); - Py_DECREF(comma); - if (error_names == NULL) { + newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount)); + if (newargs == NULL) { goto fail; } - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got some positional-only arguments passed" - " as keyword arguments: '%U'", - qualname, error_names); - Py_DECREF(error_names); + for (int i = 0; i < argcount; i++) { + newargs[i] = args[i]; + } + for (int i = 0; i < kwcount; i++) { + PyTuple_SET_ITEM(kwnames, i, Py_NewRef(kws[2*i])); + newargs[argcount+i] = kws[2*i+1]; + } + allargs = newargs; + } + PyFrameConstructor constr = { + .fc_globals = globals, + .fc_builtins = builtins, + .fc_name = ((PyCodeObject *)_co)->co_name, + .fc_qualname = ((PyCodeObject *)_co)->co_name, + .fc_code = _co, + .fc_defaults = defaults, + .fc_kwdefaults = kwdefs, + .fc_closure = closure + }; + func = _PyFunction_FromConstructor(&constr); + if (func == NULL) { goto fail; } - - Py_DECREF(posonly_names); - return 0; - + EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY); + res = _PyEval_Vector(tstate, func, locals, + allargs, argcount, + kwnames); fail: - Py_XDECREF(posonly_names); - return 1; - -} - - -static inline unsigned char * -scan_back_to_entry_start(unsigned char *p) { - for (; (p[0]&128) == 0; p--); - return p; -} - -static inline unsigned char * -skip_to_next_entry(unsigned char *p, unsigned char *end) { - while (p < end && ((p[0] & 128) == 0)) { - p++; - } - return p; + Py_XDECREF(func); + Py_XDECREF(kwnames); + PyMem_Free(newargs); + Py_DECREF(defaults); + return res; } -#define MAX_LINEAR_SEARCH 40 - +/* Logic for the raise statement (too complicated for inlining). + This *consumes* a reference count to each of its arguments. */ static int -get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) +do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) { - unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); - unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); - /* Invariants: - * start_table == end_table OR - * start_table points to a legal entry and end_table points - * beyond the table or to a legal entry that is after index. - */ - if (end - start > MAX_LINEAR_SEARCH) { - int offset; - parse_varint(start, &offset); - if (offset > index) { - return 0; - } - do { - unsigned char * mid = start + ((end-start)>>1); - mid = scan_back_to_entry_start(mid); - parse_varint(mid, &offset); - if (offset > index) { - end = mid; - } - else { - start = mid; - } + PyObject *type = NULL, *value = NULL; - } while (end - start > MAX_LINEAR_SEARCH); - } - unsigned char *scan = start; - while (scan < end) { - int start_offset, size; - scan = parse_varint(scan, &start_offset); - if (start_offset > index) { - break; - } - scan = parse_varint(scan, &size); - if (start_offset + size > index) { - scan = parse_varint(scan, handler); - int depth_and_lasti; - parse_varint(scan, &depth_and_lasti); - *level = depth_and_lasti >> 1; - *lasti = depth_and_lasti & 1; - return 1; + if (exc == NULL) { + /* Reraise */ + _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); + exc = exc_info->exc_value; + if (Py_IsNone(exc) || exc == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "No active exception to reraise"); + return 0; } - scan = skip_to_next_entry(scan, end); + Py_INCREF(exc); + assert(PyExceptionInstance_Check(exc)); + _PyErr_SetRaisedException(tstate, exc); + return 1; } - return 0; -} -static int -initialize_locals(PyThreadState *tstate, PyFunctionObject *func, - PyObject **localsplus, PyObject *const *args, - Py_ssize_t argcount, PyObject *kwnames) -{ - PyCodeObject *co = (PyCodeObject*)func->func_code; - const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; + /* We support the following forms of raise: + raise + raise + raise */ - /* Create a dictionary for keyword parameters (**kwags) */ - PyObject *kwdict; - Py_ssize_t i; - if (co->co_flags & CO_VARKEYWORDS) { - kwdict = PyDict_New(); - if (kwdict == NULL) { - goto fail_pre_positional; - } - i = total_args; - if (co->co_flags & CO_VARARGS) { - i++; + if (PyExceptionClass_Check(exc)) { + type = exc; + value = _PyObject_CallNoArgs(exc); + if (value == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(value)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto raise_error; } - assert(localsplus[i] == NULL); - localsplus[i] = kwdict; } - else { - kwdict = NULL; - } - - /* Copy all positional arguments into local variables */ - Py_ssize_t j, n; - if (argcount > co->co_argcount) { - n = co->co_argcount; + else if (PyExceptionInstance_Check(exc)) { + value = exc; + type = PyExceptionInstance_Class(exc); + Py_INCREF(type); } else { - n = argcount; - } - for (j = 0; j < n; j++) { - PyObject *x = args[j]; - assert(localsplus[j] == NULL); - localsplus[j] = x; + /* Not something you can raise. You get an exception + anyway, just not what you specified :-) */ + Py_DECREF(exc); + _PyErr_SetString(tstate, PyExc_TypeError, + "exceptions must derive from BaseException"); + goto raise_error; } - /* Pack other positional arguments into the *args argument */ - if (co->co_flags & CO_VARARGS) { - PyObject *u = NULL; - if (argcount == n) { - u = (PyObject *)&_Py_SINGLETON(tuple_empty); + assert(type != NULL); + assert(value != NULL); + + if (cause) { + PyObject *fixed_cause; + if (PyExceptionClass_Check(cause)) { + fixed_cause = _PyObject_CallNoArgs(cause); + if (fixed_cause == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + goto raise_error; + } + Py_DECREF(cause); } - else { - assert(args != NULL); - u = _PyTuple_FromArraySteal(args + n, argcount - n); + else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; } - if (u == NULL) { - goto fail_post_positional; + else if (Py_IsNone(cause)) { + Py_DECREF(cause); + fixed_cause = NULL; } - assert(localsplus[total_args] == NULL); - localsplus[total_args] = u; - } - else if (argcount > n) { - /* Too many postional args. Error is reported later */ - for (j = n; j < argcount; j++) { - Py_DECREF(args[j]); + else { + _PyErr_SetString(tstate, PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto raise_error; } + PyException_SetCause(value, fixed_cause); } - /* Handle keyword arguments */ - if (kwnames != NULL) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (i = 0; i < kwcount; i++) { - PyObject **co_varnames; - PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); - PyObject *value = args[i+argcount]; - Py_ssize_t j; - - if (keyword == NULL || !PyUnicode_Check(keyword)) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() keywords must be strings", - func->func_qualname); - goto kw_fail; - } - - /* Speed hack: do raw pointer compares. As names are - normally interned this should almost always hit. */ - co_varnames = ((PyTupleObject *)(co->co_localsplusnames))->ob_item; - for (j = co->co_posonlyargcount; j < total_args; j++) { - PyObject *varname = co_varnames[j]; - if (varname == keyword) { - goto kw_found; - } - } + _PyErr_SetObject(tstate, type, value); + /* _PyErr_SetObject incref's its arguments */ + Py_DECREF(value); + Py_DECREF(type); + return 0; - /* Slow fallback, just in case */ - for (j = co->co_posonlyargcount; j < total_args; j++) { - PyObject *varname = co_varnames[j]; - int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ); - if (cmp > 0) { - goto kw_found; - } - else if (cmp < 0) { - goto kw_fail; - } - } +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(cause); + return 0; +} - assert(j >= total_args); - if (kwdict == NULL) { +/* Logic for matching an exception in an except* clause (too + complicated for inlining). +*/ - if (co->co_posonlyargcount - && positional_only_passed_as_keyword(tstate, co, - kwcount, kwnames, - func->func_qualname)) - { - goto kw_fail; - } +int +_PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, + PyObject **match, PyObject **rest) +{ + if (Py_IsNone(exc_value)) { + *match = Py_NewRef(Py_None); + *rest = Py_NewRef(Py_None); + return 0; + } + assert(PyExceptionInstance_Check(exc_value)); - PyObject* suggestion_keyword = NULL; - if (total_args > co->co_posonlyargcount) { - PyObject* possible_keywords = PyList_New(total_args - co->co_posonlyargcount); - - if (!possible_keywords) { - PyErr_Clear(); - } else { - for (Py_ssize_t k = co->co_posonlyargcount; k < total_args; k++) { - PyList_SET_ITEM(possible_keywords, k - co->co_posonlyargcount, co_varnames[k]); - } - - suggestion_keyword = _Py_CalculateSuggestions(possible_keywords, keyword); - Py_DECREF(possible_keywords); - } - } - - if (suggestion_keyword) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got an unexpected keyword argument '%S'. Did you mean '%S'?", - func->func_qualname, keyword, suggestion_keyword); - Py_DECREF(suggestion_keyword); - } else { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got an unexpected keyword argument '%S'", - func->func_qualname, keyword); - } - - goto kw_fail; - } - - if (PyDict_SetItem(kwdict, keyword, value) == -1) { - goto kw_fail; - } - Py_DECREF(value); - continue; - - kw_fail: - for (;i < kwcount; i++) { - PyObject *value = args[i+argcount]; - Py_DECREF(value); + if (PyErr_GivenExceptionMatches(exc_value, match_type)) { + /* Full match of exc itself */ + bool is_eg = _PyBaseExceptionGroup_Check(exc_value); + if (is_eg) { + *match = Py_NewRef(exc_value); + } + else { + /* naked exception - wrap it */ + PyObject *excs = PyTuple_Pack(1, exc_value); + if (excs == NULL) { + return -1; } - goto fail_post_args; - - kw_found: - if (localsplus[j] != NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got multiple values for argument '%S'", - func->func_qualname, keyword); - goto kw_fail; + PyObject *wrapped = _PyExc_CreateExceptionGroup("", excs); + Py_DECREF(excs); + if (wrapped == NULL) { + return -1; } - localsplus[j] = value; + *match = wrapped; } + *rest = Py_NewRef(Py_None); + return 0; } - /* Check the number of positional arguments */ - if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { - too_many_positional(tstate, co, argcount, func->func_defaults, localsplus, - func->func_qualname); - goto fail_post_args; + /* exc_value does not match match_type. + * Check for partial match if it's an exception group. + */ + if (_PyBaseExceptionGroup_Check(exc_value)) { + PyObject *pair = PyObject_CallMethod(exc_value, "split", "(O)", + match_type); + if (pair == NULL) { + return -1; + } + assert(PyTuple_CheckExact(pair)); + assert(PyTuple_GET_SIZE(pair) == 2); + *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0)); + *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1)); + Py_DECREF(pair); + return 0; } + /* no match */ + *match = Py_NewRef(Py_None); + *rest = Py_NewRef(exc_value); + return 0; +} - /* Add missing positional arguments (copy default values from defs) */ - if (argcount < co->co_argcount) { - Py_ssize_t defcount = func->func_defaults == NULL ? 0 : PyTuple_GET_SIZE(func->func_defaults); - Py_ssize_t m = co->co_argcount - defcount; - Py_ssize_t missing = 0; - for (i = argcount; i < m; i++) { - if (localsplus[i] == NULL) { - missing++; - } - } - if (missing) { - missing_arguments(tstate, co, missing, defcount, localsplus, - func->func_qualname); - goto fail_post_args; - } - if (n > m) - i = n - m; - else - i = 0; - if (defcount) { - PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0); - for (; i < defcount; i++) { - if (localsplus[m+i] == NULL) { - PyObject *def = defs[i]; - localsplus[m+i] = Py_NewRef(def); - } - } +/* Iterate v argcnt times and store the results on the stack (via decreasing + sp). Return 1 for success, 0 if error. + + If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack + with a variable target. +*/ + +int +_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, + int argcnt, int argcntafter, PyObject **sp) +{ + int i = 0, j = 0; + Py_ssize_t ll = 0; + PyObject *it; /* iter(v) */ + PyObject *w; + PyObject *l = NULL; /* variable list */ + + assert(v != NULL); + + it = PyObject_GetIter(v); + if (it == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + Py_TYPE(v)->tp_iter == NULL && !PySequence_Check(v)) + { + _PyErr_Format(tstate, PyExc_TypeError, + "cannot unpack non-iterable %.200s object", + Py_TYPE(v)->tp_name); } + return 0; } - /* Add missing keyword arguments (copy default values from kwdefs) */ - if (co->co_kwonlyargcount > 0) { - Py_ssize_t missing = 0; - for (i = co->co_argcount; i < total_args; i++) { - if (localsplus[i] != NULL) - continue; - PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); - if (func->func_kwdefaults != NULL) { - PyObject *def; - if (PyDict_GetItemRef(func->func_kwdefaults, varname, &def) < 0) { - goto fail_post_args; + for (; i < argcnt; i++) { + w = PyIter_Next(it); + if (w == NULL) { + /* Iterator done, via error or exhaustion. */ + if (!_PyErr_Occurred(tstate)) { + if (argcntafter == -1) { + _PyErr_Format(tstate, PyExc_ValueError, + "not enough values to unpack " + "(expected %d, got %d)", + argcnt, i); } - if (def) { - localsplus[i] = def; - continue; + else { + _PyErr_Format(tstate, PyExc_ValueError, + "not enough values to unpack " + "(expected at least %d, got %d)", + argcnt + argcntafter, i); } } - missing++; + goto Error; } - if (missing) { - missing_arguments(tstate, co, missing, -1, localsplus, - func->func_qualname); - goto fail_post_args; + *--sp = w; + } + + if (argcntafter == -1) { + /* We better have exhausted the iterator now. */ + w = PyIter_Next(it); + if (w == NULL) { + if (_PyErr_Occurred(tstate)) + goto Error; + Py_DECREF(it); + return 1; } + Py_DECREF(w); + _PyErr_Format(tstate, PyExc_ValueError, + "too many values to unpack (expected %d)", + argcnt); + goto Error; } - return 0; -fail_pre_positional: - for (j = 0; j < argcount; j++) { - Py_DECREF(args[j]); + l = PySequence_List(it); + if (l == NULL) + goto Error; + *--sp = l; + i++; + + ll = PyList_GET_SIZE(l); + if (ll < argcntafter) { + _PyErr_Format(tstate, PyExc_ValueError, + "not enough values to unpack (expected at least %d, got %zd)", + argcnt + argcntafter, argcnt + ll); + goto Error; } - /* fall through */ -fail_post_positional: - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (j = argcount; j < argcount+kwcount; j++) { - Py_DECREF(args[j]); - } + + /* Pop the "after-variable" args off the list. */ + for (j = argcntafter; j > 0; j--, i++) { + *--sp = PyList_GET_ITEM(l, ll - j); } - /* fall through */ -fail_post_args: - return -1; -} + /* Resize the list. */ + Py_SET_SIZE(l, ll - argcntafter); + Py_DECREF(it); + return 1; -static void -clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) -{ - assert(frame->owner == FRAME_OWNED_BY_THREAD); - // Make sure that this is, indeed, the top frame. We can't check this in - // _PyThreadState_PopFrame, since f_code is already cleared at that point: - assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == - tstate->datastack_top); - tstate->c_recursion_remaining--; - assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); - _PyFrame_ClearExceptCode(frame); - Py_DECREF(frame->f_executable); - tstate->c_recursion_remaining++; - _PyThreadState_PopFrame(tstate, frame); +Error: + for (; i > 0; i--, sp++) + Py_DECREF(*sp); + Py_XDECREF(it); + return 0; } -static void -clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) +static int +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) { - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_CLEARED; - assert(tstate->exc_info == &gen->gi_exc_state); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - tstate->c_recursion_remaining--; - assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); - _PyFrame_ClearExceptCode(frame); - _PyErr_ClearExcState(&gen->gi_exc_state); - tstate->c_recursion_remaining++; - frame->previous = NULL; -} - -void -_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) -{ - if (frame->owner == FRAME_OWNED_BY_THREAD) { - clear_thread_frame(tstate, frame); + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + return 0; + } + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + if (err == 0) { + PyErr_SetRaisedException(exc); } else { - clear_gen_frame(tstate, frame); + assert(PyErr_Occurred()); + Py_DECREF(exc); } + return err; } -/* Consumes references to func, locals and all the args */ -static _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames) +static inline bool +no_tools_for_global_event(PyThreadState *tstate, int event) { - PyCodeObject * code = (PyCodeObject *)func->func_code; - CALL_STAT_INC(frames_pushed); - _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); - if (frame == NULL) { - goto fail; - } - _PyFrame_Initialize(frame, func, locals, code, 0); - if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { - assert(frame->owner == FRAME_OWNED_BY_THREAD); - clear_thread_frame(tstate, frame); - return NULL; - } - return frame; -fail: - /* Consume the references */ - for (size_t i = 0; i < argcount; i++) { - Py_DECREF(args[i]); - } - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_DECREF(args[i+argcount]); - } - } - PyErr_NoMemory(); - return NULL; + return tstate->interp->monitors.tools[event] == 0; } -/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. - Steals references to func, callargs and kwargs. -*/ -static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) +static inline bool +no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); - PyObject *kwnames = NULL; - PyObject *const *newargs; - if (has_dict) { - newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); - if (newargs == NULL) { - Py_DECREF(func); - goto error; - } + assert(event < _PY_MONITORING_LOCAL_EVENTS); + _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; + if (data) { + return data->active_monitors.tools[event] == 0; } else { - newargs = &PyTuple_GET_ITEM(callargs, 0); - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); - } - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); - if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + return no_tools_for_global_event(tstate, event); } - /* No need to decref func here because the reference has been stolen by - _PyEvalFramePushAndInit. - */ - Py_DECREF(callargs); - Py_XDECREF(kwargs); - return new_frame; -error: - Py_DECREF(callargs); - Py_XDECREF(kwargs); - return NULL; } -PyObject * -_PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, - PyObject* const* args, size_t argcount, - PyObject *kwnames) +static void +monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - /* _PyEvalFramePushAndInit consumes the references - * to func, locals and all its arguments */ - Py_INCREF(func); - Py_XINCREF(locals); - for (size_t i = 0; i < argcount; i++) { - Py_INCREF(args[i]); - } - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_INCREF(args[i+argcount]); - } - } - _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( - tstate, func, locals, args, argcount, kwnames); - if (frame == NULL) { - return NULL; + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { + return; } - EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR); - return _PyEval_EvalFrame(tstate, frame, 0); + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); } -/* Legacy API */ -PyObject * -PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, - PyObject *const *args, int argcount, - PyObject *const *kws, int kwcount, - PyObject *const *defs, int defcount, - PyObject *kwdefs, PyObject *closure) +static void +monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *res = NULL; - PyObject *defaults = _PyTuple_FromArray(defs, defcount); - if (defaults == NULL) { - return NULL; - } - PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref - if (builtins == NULL) { - Py_DECREF(defaults); - return NULL; - } - if (locals == NULL) { - locals = globals; - } - PyObject *kwnames = NULL; - PyObject *const *allargs; - PyObject **newargs = NULL; - PyFunctionObject *func = NULL; - if (kwcount == 0) { - allargs = args; + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { + return; } - else { - kwnames = PyTuple_New(kwcount); - if (kwnames == NULL) { - goto fail; - } - newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount)); - if (newargs == NULL) { - goto fail; - } - for (int i = 0; i < argcount; i++) { - newargs[i] = args[i]; - } - for (int i = 0; i < kwcount; i++) { - PyTuple_SET_ITEM(kwnames, i, Py_NewRef(kws[2*i])); - newargs[argcount+i] = kws[2*i+1]; - } - allargs = newargs; + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); +} + +static int +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + return 0; } - PyFrameConstructor constr = { - .fc_globals = globals, - .fc_builtins = builtins, - .fc_name = ((PyCodeObject *)_co)->co_name, - .fc_qualname = ((PyCodeObject *)_co)->co_name, - .fc_code = _co, - .fc_defaults = defaults, - .fc_kwdefaults = kwdefs, - .fc_closure = closure - }; - func = _PyFunction_FromConstructor(&constr); - if (func == NULL) { - goto fail; + return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); +} + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { + return; } - EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY); - res = _PyEval_Vector(tstate, func, locals, - allargs, argcount, - kwnames); -fail: - Py_XDECREF(func); - Py_XDECREF(kwnames); - PyMem_Free(newargs); - Py_DECREF(defaults); - return res; + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); } -/* Logic for the raise statement (too complicated for inlining). - This *consumes* a reference count to each of its arguments. */ static int -do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) { - PyObject *type = NULL, *value = NULL; - - if (exc == NULL) { - /* Reraise */ - _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); - exc = exc_info->exc_value; - if (Py_IsNone(exc) || exc == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "No active exception to reraise"); - return 0; - } - Py_INCREF(exc); - assert(PyExceptionInstance_Check(exc)); - _PyErr_SetRaisedException(tstate, exc); - return 1; + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + return 0; } + return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); +} - /* We support the following forms of raise: - raise - raise - raise */ - - if (PyExceptionClass_Check(exc)) { - type = exc; - value = _PyObject_CallNoArgs(exc); - if (value == NULL) - goto raise_error; - if (!PyExceptionInstance_Check(value)) { - _PyErr_Format(tstate, PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto raise_error; - } - } - else if (PyExceptionInstance_Check(exc)) { - value = exc; - type = PyExceptionInstance_Class(exc); - Py_INCREF(type); - } - else { - /* Not something you can raise. You get an exception - anyway, just not what you specified :-) */ - Py_DECREF(exc); - _PyErr_SetString(tstate, PyExc_TypeError, - "exceptions must derive from BaseException"); - goto raise_error; +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { + return; } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); +} - assert(type != NULL); - assert(value != NULL); +void +PyThreadState_EnterTracing(PyThreadState *tstate) +{ + assert(tstate->tracing >= 0); + tstate->tracing++; +} - if (cause) { - PyObject *fixed_cause; - if (PyExceptionClass_Check(cause)) { - fixed_cause = _PyObject_CallNoArgs(cause); - if (fixed_cause == NULL) - goto raise_error; - if (!PyExceptionInstance_Check(fixed_cause)) { - _PyErr_Format(tstate, PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - cause, Py_TYPE(fixed_cause)); - goto raise_error; - } - Py_DECREF(cause); - } - else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - } - else if (Py_IsNone(cause)) { - Py_DECREF(cause); - fixed_cause = NULL; - } - else { - _PyErr_SetString(tstate, PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto raise_error; - } - PyException_SetCause(value, fixed_cause); - } +void +PyThreadState_LeaveTracing(PyThreadState *tstate) +{ + assert(tstate->tracing > 0); + tstate->tracing--; +} - _PyErr_SetObject(tstate, type, value); - /* _PyErr_SetObject incref's its arguments */ - Py_DECREF(value); - Py_DECREF(type); - return 0; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(cause); - return 0; -} +PyObject* +_PyEval_CallTracing(PyObject *func, PyObject *args) +{ + // Save and disable tracing + PyThreadState *tstate = _PyThreadState_GET(); + int save_tracing = tstate->tracing; + tstate->tracing = 0; -/* Logic for matching an exception in an except* clause (too - complicated for inlining). -*/ + // Call the tracing function + PyObject *result = PyObject_Call(func, args, NULL); -int -_PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, - PyObject **match, PyObject **rest) + // Restore tracing + tstate->tracing = save_tracing; + return result; +} + +void +PyEval_SetProfile(Py_tracefunc func, PyObject *arg) { - if (Py_IsNone(exc_value)) { - *match = Py_NewRef(Py_None); - *rest = Py_NewRef(Py_None); - return 0; + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyEval_SetProfile(tstate, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfile"); } - assert(PyExceptionInstance_Check(exc_value)); +} - if (PyErr_GivenExceptionMatches(exc_value, match_type)) { - /* Full match of exc itself */ - bool is_eg = _PyBaseExceptionGroup_Check(exc_value); - if (is_eg) { - *match = Py_NewRef(exc_value); - } - else { - /* naked exception - wrap it */ - PyObject *excs = PyTuple_Pack(1, exc_value); - if (excs == NULL) { - return -1; - } - PyObject *wrapped = _PyExc_CreateExceptionGroup("", excs); - Py_DECREF(excs); - if (wrapped == NULL) { - return -1; - } - *match = wrapped; +void +PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) +{ + PyThreadState *this_tstate = _PyThreadState_GET(); + PyInterpreterState* interp = this_tstate->interp; + + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + + while (ts) { + if (_PyEval_SetProfile(ts, func, arg) < 0) { + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); } - *rest = Py_NewRef(Py_None); - return 0; + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); } +} - /* exc_value does not match match_type. - * Check for partial match if it's an exception group. - */ - if (_PyBaseExceptionGroup_Check(exc_value)) { - PyObject *pair = PyObject_CallMethod(exc_value, "split", "(O)", - match_type); - if (pair == NULL) { - return -1; +void +PyEval_SetTrace(Py_tracefunc func, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyEval_SetTrace(tstate, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTrace"); + } +} + +void +PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) +{ + PyThreadState *this_tstate = _PyThreadState_GET(); + PyInterpreterState* interp = this_tstate->interp; + + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + + while (ts) { + if (_PyEval_SetTrace(ts, func, arg) < 0) { + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); } - assert(PyTuple_CheckExact(pair)); - assert(PyTuple_GET_SIZE(pair) == 2); - *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0)); - *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1)); - Py_DECREF(pair); - return 0; + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); } - /* no match */ - *match = Py_NewRef(Py_None); - *rest = Py_NewRef(exc_value); - return 0; } -/* Iterate v argcnt times and store the results on the stack (via decreasing - sp). Return 1 for success, 0 if error. +int +_PyEval_SetCoroutineOriginTrackingDepth(int depth) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (depth < 0) { + _PyErr_SetString(tstate, PyExc_ValueError, "depth must be >= 0"); + return -1; + } + tstate->coroutine_origin_tracking_depth = depth; + return 0; +} - If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack - with a variable target. -*/ int -_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, - int argcnt, int argcntafter, PyObject **sp) +_PyEval_GetCoroutineOriginTrackingDepth(void) { - int i = 0, j = 0; - Py_ssize_t ll = 0; - PyObject *it; /* iter(v) */ - PyObject *w; - PyObject *l = NULL; /* variable list */ + PyThreadState *tstate = _PyThreadState_GET(); + return tstate->coroutine_origin_tracking_depth; +} - assert(v != NULL); +int +_PyEval_SetAsyncGenFirstiter(PyObject *firstiter) +{ + PyThreadState *tstate = _PyThreadState_GET(); - it = PyObject_GetIter(v); - if (it == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - Py_TYPE(v)->tp_iter == NULL && !PySequence_Check(v)) - { - _PyErr_Format(tstate, PyExc_TypeError, - "cannot unpack non-iterable %.200s object", - Py_TYPE(v)->tp_name); - } - return 0; + if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_firstiter", NULL) < 0) { + return -1; } - for (; i < argcnt; i++) { - w = PyIter_Next(it); - if (w == NULL) { - /* Iterator done, via error or exhaustion. */ - if (!_PyErr_Occurred(tstate)) { - if (argcntafter == -1) { - _PyErr_Format(tstate, PyExc_ValueError, - "not enough values to unpack " - "(expected %d, got %d)", - argcnt, i); - } - else { - _PyErr_Format(tstate, PyExc_ValueError, - "not enough values to unpack " - "(expected at least %d, got %d)", - argcnt + argcntafter, i); - } - } - goto Error; - } - *--sp = w; - } - - if (argcntafter == -1) { - /* We better have exhausted the iterator now. */ - w = PyIter_Next(it); - if (w == NULL) { - if (_PyErr_Occurred(tstate)) - goto Error; - Py_DECREF(it); - return 1; - } - Py_DECREF(w); - _PyErr_Format(tstate, PyExc_ValueError, - "too many values to unpack (expected %d)", - argcnt); - goto Error; - } + Py_XSETREF(tstate->async_gen_firstiter, Py_XNewRef(firstiter)); + return 0; +} - l = PySequence_List(it); - if (l == NULL) - goto Error; - *--sp = l; - i++; +PyObject * +_PyEval_GetAsyncGenFirstiter(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return tstate->async_gen_firstiter; +} - ll = PyList_GET_SIZE(l); - if (ll < argcntafter) { - _PyErr_Format(tstate, PyExc_ValueError, - "not enough values to unpack (expected at least %d, got %zd)", - argcnt + argcntafter, argcnt + ll); - goto Error; - } +int +_PyEval_SetAsyncGenFinalizer(PyObject *finalizer) +{ + PyThreadState *tstate = _PyThreadState_GET(); - /* Pop the "after-variable" args off the list. */ - for (j = argcntafter; j > 0; j--, i++) { - *--sp = PyList_GET_ITEM(l, ll - j); + if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_finalizer", NULL) < 0) { + return -1; } - /* Resize the list. */ - Py_SET_SIZE(l, ll - argcntafter); - Py_DECREF(it); - return 1; -Error: - for (; i > 0; i--, sp++) - Py_DECREF(*sp); - Py_XDECREF(it); + Py_XSETREF(tstate->async_gen_finalizer, Py_XNewRef(finalizer)); return 0; } -static int -do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event) +PyObject * +_PyEval_GetAsyncGenFinalizer(void) { - assert(event < _PY_MONITORING_UNGROUPED_EVENTS); - if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { - return 0; - } - PyObject *exc = PyErr_GetRaisedException(); - assert(exc != NULL); - int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); - if (err == 0) { - PyErr_SetRaisedException(exc); - } - else { - assert(PyErr_Occurred()); - Py_DECREF(exc); - } - return err; + PyThreadState *tstate = _PyThreadState_GET(); + return tstate->async_gen_finalizer; } -static inline bool -no_tools_for_global_event(PyThreadState *tstate, int event) +_PyInterpreterFrame * +_PyEval_GetFrame(void) { - return tstate->interp->monitors.tools[event] == 0; + PyThreadState *tstate = _PyThreadState_GET(); + return _PyThreadState_GetFrame(tstate); } -static inline bool -no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +PyFrameObject * +PyEval_GetFrame(void) { - assert(event < _PY_MONITORING_LOCAL_EVENTS); - _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; - if (data) { - return data->active_monitors.tools[event] == 0; + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + if (frame == NULL) { + return NULL; } - else { - return no_tools_for_global_event(tstate, event); + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f == NULL) { + PyErr_Clear(); } + return f; } -static void -monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) +PyObject * +_PyEval_GetBuiltins(PyThreadState *tstate) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { - return; + _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); + if (frame != NULL) { + return frame->f_builtins; } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); + return tstate->interp->builtins; } -static void -monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) +PyObject * +PyEval_GetBuiltins(void) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { - return; - } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_GetBuiltins(tstate); } -static int -monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) +/* Convenience function to get a builtin from its name */ +PyObject * +_PyEval_GetBuiltin(PyObject *name) { - if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { - return 0; + PyObject *attr; + if (PyMapping_GetOptionalItem(PyEval_GetBuiltins(), name, &attr) == 0) { + PyErr_SetObject(PyExc_AttributeError, name); } - return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + return attr; } -static void -monitor_unwind(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) +PyObject * +_PyEval_GetBuiltinId(_Py_Identifier *name) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { - return; - } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); + return _PyEval_GetBuiltin(_PyUnicode_FromId(name)); } - -static int -monitor_handled(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, PyObject *exc) +PyObject * +PyEval_GetLocals(void) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { - return 0; + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; } - return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); -} -static void -monitor_throw(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { - return; + if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { + return NULL; } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); -} - -void -PyThreadState_EnterTracing(PyThreadState *tstate) -{ - assert(tstate->tracing >= 0); - tstate->tracing++; -} -void -PyThreadState_LeaveTracing(PyThreadState *tstate) -{ - assert(tstate->tracing > 0); - tstate->tracing--; + PyObject *locals = current_frame->f_locals; + assert(locals != NULL); + return locals; } - -PyObject* -_PyEval_CallTracing(PyObject *func, PyObject *args) +PyObject * +_PyEval_GetFrameLocals(void) { - // Save and disable tracing PyThreadState *tstate = _PyThreadState_GET(); - int save_tracing = tstate->tracing; - tstate->tracing = 0; - - // Call the tracing function - PyObject *result = PyObject_Call(func, args, NULL); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } - // Restore tracing - tstate->tracing = save_tracing; - return result; + return _PyFrame_GetLocals(current_frame, 1); } -void -PyEval_SetProfile(Py_tracefunc func, PyObject *arg) +PyObject * +PyEval_GetGlobals(void) { PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, func, arg) < 0) { - /* Log _PySys_Audit() error */ - PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfile"); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + return NULL; } + return current_frame->f_globals; } -void -PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) +int +PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetProfile(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); - } -} - -void -PyEval_SetTrace(Py_tracefunc func, PyObject *arg) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetTrace(tstate, func, arg) < 0) { - /* Log _PySys_Audit() error */ - PyErr_FormatUnraisable("Exception ignored in PyEval_SetTrace"); - } -} - -void -PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) -{ - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetTrace(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); - } -} - -int -_PyEval_SetCoroutineOriginTrackingDepth(int depth) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (depth < 0) { - _PyErr_SetString(tstate, PyExc_ValueError, "depth must be >= 0"); - return -1; - } - tstate->coroutine_origin_tracking_depth = depth; - return 0; -} - - -int -_PyEval_GetCoroutineOriginTrackingDepth(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return tstate->coroutine_origin_tracking_depth; -} - -int -_PyEval_SetAsyncGenFirstiter(PyObject *firstiter) -{ - PyThreadState *tstate = _PyThreadState_GET(); - - if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_firstiter", NULL) < 0) { - return -1; - } - - Py_XSETREF(tstate->async_gen_firstiter, Py_XNewRef(firstiter)); - return 0; -} - -PyObject * -_PyEval_GetAsyncGenFirstiter(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return tstate->async_gen_firstiter; -} - -int -_PyEval_SetAsyncGenFinalizer(PyObject *finalizer) -{ - PyThreadState *tstate = _PyThreadState_GET(); - - if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_finalizer", NULL) < 0) { - return -1; - } - - Py_XSETREF(tstate->async_gen_finalizer, Py_XNewRef(finalizer)); - return 0; -} - -PyObject * -_PyEval_GetAsyncGenFinalizer(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return tstate->async_gen_finalizer; -} - -_PyInterpreterFrame * -_PyEval_GetFrame(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return _PyThreadState_GetFrame(tstate); -} - -PyFrameObject * -PyEval_GetFrame(void) -{ - _PyInterpreterFrame *frame = _PyEval_GetFrame(); - if (frame == NULL) { - return NULL; - } - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - PyErr_Clear(); - } - return f; -} - -PyObject * -_PyEval_GetBuiltins(PyThreadState *tstate) -{ - _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); - if (frame != NULL) { - return frame->f_builtins; - } - return tstate->interp->builtins; -} - -PyObject * -PyEval_GetBuiltins(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return _PyEval_GetBuiltins(tstate); -} - -/* Convenience function to get a builtin from its name */ -PyObject * -_PyEval_GetBuiltin(PyObject *name) -{ - PyObject *attr; - if (PyMapping_GetOptionalItem(PyEval_GetBuiltins(), name, &attr) == 0) { - PyErr_SetObject(PyExc_AttributeError, name); - } - return attr; -} - -PyObject * -_PyEval_GetBuiltinId(_Py_Identifier *name) -{ - return _PyEval_GetBuiltin(_PyUnicode_FromId(name)); -} - -PyObject * -PyEval_GetLocals(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); - if (current_frame == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); - return NULL; - } - - if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { - return NULL; - } - - PyObject *locals = current_frame->f_locals; - assert(locals != NULL); - return locals; -} - -PyObject * -_PyEval_GetFrameLocals(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); - if (current_frame == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); - return NULL; - } - - return _PyFrame_GetLocals(current_frame, 1); -} - -PyObject * -PyEval_GetGlobals(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); - if (current_frame == NULL) { - return NULL; - } - return current_frame->f_globals; -} - -int -PyEval_MergeCompilerFlags(PyCompilerFlags *cf) -{ - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = tstate->current_frame; - int result = cf->cf_flags != 0; + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = tstate->current_frame; + int result = cf->cf_flags != 0; if (current_frame != NULL) { const int codeflags = _PyFrame_GetCode(current_frame)->co_flags; @@ -2841,103 +2327,617 @@ _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwarg funcstr, key); Py_DECREF(funcstr); } - Py_XDECREF(exc); - } - else { - _PyErr_SetRaisedException(tstate, exc); + Py_XDECREF(exc); + } + else { + _PyErr_SetRaisedException(tstate, exc); + } + } +} + +void +_PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, + const char *format_str, PyObject *obj) +{ + const char *obj_str; + + if (!obj) + return; + + obj_str = PyUnicode_AsUTF8(obj); + if (!obj_str) + return; + + _PyErr_Format(tstate, exc, format_str, obj_str); + + if (exc == PyExc_NameError) { + // Include the name in the NameError exceptions to offer suggestions later. + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_GivenExceptionMatches(exc, PyExc_NameError)) { + if (((PyNameErrorObject*)exc)->name == NULL) { + // We do not care if this fails because we are going to restore the + // NameError anyway. + (void)PyObject_SetAttr(exc, &_Py_ID(name), obj); + } + } + PyErr_SetRaisedException(exc); + } +} + +void +_PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg) +{ + PyObject *name; + /* Don't stomp existing exception */ + if (_PyErr_Occurred(tstate)) + return; + name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); + if (oparg < PyUnstable_Code_GetFirstFree(co)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, name); + } else { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + UNBOUNDFREE_ERROR_MSG, name); + } +} + +void +_PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg) +{ + if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) { + if (oparg == 1) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async with' received an object from __aenter__ " + "that does not implement __await__: %.100s", + type->tp_name); + } + else if (oparg == 2) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async with' received an object from __aexit__ " + "that does not implement __await__: %.100s", + type->tp_name); + } + } +} + +Py_ssize_t +PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + Py_ssize_t new_index; + + if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { + return -1; + } + new_index = interp->co_extra_user_count++; + interp->co_extra_freefuncs[new_index] = free; + return new_index; +} + +/* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions + for the limited API. */ + +int Py_EnterRecursiveCall(const char *where) +{ + return _Py_EnterRecursiveCall(where); +} + +void Py_LeaveRecursiveCall(void) +{ + _Py_LeaveRecursiveCall(); +} + + +/* Interpreter main loop */ + +PyObject * +PyEval_EvalFrame(PyFrameObject *f) +{ + /* Function kept for backward compatibility */ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_EvalFrame(tstate, f->f_frame, 0); +} + +PyObject * +PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_EvalFrame(tstate, f->f_frame, throwflag); +} + +int _Py_CheckRecursiveCallPy( + PyThreadState *tstate) +{ + if (tstate->recursion_headroom) { + if (tstate->py_recursion_remaining < -50) { + /* Overflowing while handling an overflow. Give up. */ + Py_FatalError("Cannot recover from Python stack overflow."); + } + } + else { + if (tstate->py_recursion_remaining <= 0) { + tstate->recursion_headroom++; + _PyErr_Format(tstate, PyExc_RecursionError, + "maximum recursion depth exceeded"); + tstate->recursion_headroom--; + return -1; + } + } + return 0; +} + +static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { + /* Put a NOP at the start, so that the IP points into + * the code, rather than before it */ + { .op.code = NOP, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ + { .op.code = NOP, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ + { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } +}; + +extern const struct _PyCode_DEF(8) _Py_InitCleanup; + +#ifdef Py_DEBUG +extern void _PyUOpPrint(const _PyUOpInstruction *uop); +#endif + +/* Disable unused label warnings. They are handy for debugging, even + if computed gotos aren't used. */ + +/* TBD - what about other compilers? */ +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-label" +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(push) +# pragma warning(disable:4102) +#endif + + +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 2 + +#if defined(_MSC_VER) && defined(_Py_USING_PGO) +/* gh-111786: _PyEval_EvalFrameDefault is too large to optimize for speed with + PGO on MSVC. Disable that optimization temporarily. If this is fixed + upstream, we should gate this on the version of MSVC. + */ +# pragma optimize("t", off) +/* This setting is reversed below following _PyEval_EvalFrameDefault */ +#endif + +PyObject* _Py_HOT_FUNCTION +_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) +{ + _Py_EnsureTstateNotNULL(tstate); + CALL_STAT_INC(pyeval_calls); + +#if USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" +#endif + +#ifdef Py_STATS + int lastopcode = 0; +#endif + uint8_t opcode; /* Current opcode */ + int oparg; /* Current opcode argument, if any */ +#ifdef LLTRACE + int lltrace = 0; +#endif + + _PyInterpreterFrame entry_frame; + + + +#ifdef Py_DEBUG + /* Set these to invalid but identifiable values for debugging. */ + entry_frame.f_funcobj = (PyObject*)0xaaa0; + entry_frame.f_locals = (PyObject*)0xaaa1; + entry_frame.frame_obj = (PyFrameObject*)0xaaa2; + entry_frame.f_globals = (PyObject*)0xaaa3; + entry_frame.f_builtins = (PyObject*)0xaaa4; +#endif + entry_frame.f_executable = Py_None; + entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; + entry_frame.stacktop = 0; + entry_frame.owner = FRAME_OWNED_BY_CSTACK; + entry_frame.return_offset = 0; + /* Push frame */ + entry_frame.previous = tstate->current_frame; + frame->previous = &entry_frame; + tstate->current_frame = frame; + + tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); + if (_Py_EnterRecursiveCallTstate(tstate, "")) { + tstate->c_recursion_remaining--; + tstate->py_recursion_remaining--; + goto exit_unwind; + } + + /* support for generator.throw() */ + if (throwflag) { + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + /* Because this avoids the RESUME, + * we need to update instrumentation */ + _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + monitor_throw(tstate, frame, frame->instr_ptr); + /* TO DO -- Monitor throw entry. */ + goto resume_with_error; + } + + /* Local "register" variables. + * These are cached values from the frame and code object. */ + _Py_CODEUNIT *next_instr; + PyObject **stack_pointer; + +#ifndef _Py_JIT + /* Tier 2 interpreter state */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif + +start_frame: + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + + next_instr = frame->instr_ptr; +resume_frame: + stack_pointer = _PyFrame_GetStackPointer(frame); + +#ifdef LLTRACE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif + +#ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); +#endif + + DISPATCH(); + + { + /* Start instructions */ +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + { + +#include "generated_cases.c.h" + + /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, + * because it needs to capture frame->instr_ptr before it is updated, + * as happens in the standard instruction prologue. + */ +#if USE_COMPUTED_GOTOS + TARGET_INSTRUMENTED_LINE: +#else + case INSTRUMENTED_LINE: +#endif + { + _Py_CODEUNIT *prev = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + int original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + } /* End instructions */ + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != &entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + monitor_raise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + PyObject **stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyObject *o = POP(); + Py_XDECREF(o); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + PyObject **new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyObject *v = POP(); + Py_XDECREF(v); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(lasti); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(exc); + next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); } } -} -void -_PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, - const char *format_str, PyObject *obj) -{ - const char *obj_str; +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == &entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } - if (!obj) - return; +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; - obj_str = PyUnicode_AsUTF8(obj); - if (!obj_str) - return; - _PyErr_Format(tstate, exc, format_str, obj_str); - if (exc == PyExc_NameError) { - // Include the name in the NameError exceptions to offer suggestions later. - PyObject *exc = PyErr_GetRaisedException(); - if (PyErr_GivenExceptionMatches(exc, PyExc_NameError)) { - if (((PyNameErrorObject*)exc)->name == NULL) { - // We do not care if this fails because we are going to restore the - // NameError anyway. - (void)PyObject_SetAttr(exc, &_Py_ID(name), obj); +// Tier 2 is also here! +enter_tier_two: + +#ifdef _Py_JIT + assert(0); +#else + +#undef LOAD_IP +#define LOAD_IP(UNUSED) (void)0 + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two + +#ifdef Py_STATS +// Disable these macros that apply to Tier 1 stats when we are in Tier 2 +#undef STAT_INC +#define STAT_INC(opname, name) ((void)0) +#undef STAT_DEC +#define STAT_DEC(opname, name) ((void)0) +#undef CALL_STAT_INC +#define CALL_STAT_INC(name) ((void)0) +#endif + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION 0 + +#ifdef Py_DEBUG + #define DPRINTF(level, ...) \ + if (lltrace >= (level)) { printf(__VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif + + ; // dummy statement after a label, before a declaration + uint16_t uopcode; +#ifdef Py_STATS + int lastuop = 0; + uint64_t trace_uop_execution_counter = 0; +#endif + + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); +tier2_dispatch: + for (;;) { + uopcode = next_uop->opcode; +#ifdef Py_DEBUG + if (lltrace >= 3) { + if (next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT) { + printf("%4d uop: ", 0); + } + else { + printf("%4d uop: ", (int)(next_uop - current_executor->trace)); } + _PyUOpPrint(next_uop); + printf(" stack_level=%d\n", + (int)(stack_pointer - _PyFrame_Stackbase(frame))); } - PyErr_SetRaisedException(exc); - } -} +#endif + next_uop++; + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(uopcode, execution_count); + UOP_PAIR_INC(uopcode, lastuop); +#ifdef Py_STATS + trace_uop_execution_counter++; +#endif -void -_PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg) -{ - PyObject *name; - /* Don't stomp existing exception */ - if (_PyErr_Occurred(tstate)) - return; - name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); - if (oparg < PyUnstable_Code_GetFirstFree(co)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, name); - } else { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - UNBOUNDFREE_ERROR_MSG, name); - } -} + switch (uopcode) { + +#include "executor_cases.c.h" + + default: +#ifdef Py_DEBUG + { + printf("Unknown uop: "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); + Py_FatalError("Unknown uop"); + } +#else + Py_UNREACHABLE(); +#endif -void -_PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg) -{ - if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) { - if (oparg == 1) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async with' received an object from __aenter__ " - "that does not implement __await__: %.100s", - type->tp_name); - } - else if (oparg == 2) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async with' received an object from __aexit__ " - "that does not implement __await__: %.100s", - type->tp_name); } } -} +jump_to_error_target: +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("Error: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d -> %s]\n", + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif + assert (next_uop[-1].format == UOP_FORMAT_JUMP); + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; -Py_ssize_t -PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - Py_ssize_t new_index; +error_tier_two: + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + assert(next_uop[-1].format == UOP_FORMAT_TARGET); + frame->return_offset = 0; // Don't leave this random + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(current_executor); + tstate->previous_executor = NULL; + goto resume_with_error; - if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { - return -1; +jump_to_jump_target: + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + target = uop_get_jump_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +exit_to_tier1: + assert(next_uop[-1].format == UOP_FORMAT_TARGET); + next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("DEOPT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" -> %s]\n", + _PyOpcode_OpName[next_instr->op.code]); } - new_index = interp->co_extra_user_count++; - interp->co_extra_freefuncs[new_index] = free; - return new_index; -} +#endif + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + Py_DECREF(current_executor); + tstate->previous_executor = NULL; + DISPATCH(); -/* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions - for the limited API. */ +exit_to_trace: + assert(next_uop[-1].format == UOP_FORMAT_EXIT); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + uint32_t exit_index = next_uop[-1].exit_index; + assert(exit_index < current_executor->exit_count); + _PyExitData *exit = ¤t_executor->exits[exit_index]; +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + exit_index, exit->temperature.as_counter, exit->target, + _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); + } +#endif + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); -int Py_EnterRecursiveCall(const char *where) -{ - return _Py_EnterRecursiveCall(where); -} +#endif // _Py_JIT -void Py_LeaveRecursiveCall(void) -{ - _Py_LeaveRecursiveCall(); } + +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(pop) +# pragma optimize("", on) +#endif + +/* There should be no code below this point. */ From 681ca311c1abe80694754f14a483cc1709e38a77 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 17 Apr 2024 10:17:14 -0700 Subject: [PATCH 4/5] Revert "Move _PyEval_EvalFrameDefault to the end of the ceval.c" I'm going to try yet another approach. --- Python/ceval.c | 3938 ++++++++++++++++++++++++------------------------ 1 file changed, 1969 insertions(+), 1969 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 0fa94977de7230..c0783f7377a8ee 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -50,8 +50,6 @@ # error "ceval.c must be build with Py_BUILD_CORE define for best performance" #endif -#include "ceval_macros.h" - #if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED) // GH-89279: The MSVC compiler does not inline these static inline functions // in PGO build in _PyEval_EvalFrameDefault(), because this function is over @@ -605,2339 +603,2341 @@ PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) return res; } -static void -format_missing(PyThreadState *tstate, const char *kind, - PyCodeObject *co, PyObject *names, PyObject *qualname) -{ - int err; - Py_ssize_t len = PyList_GET_SIZE(names); - PyObject *name_str, *comma, *tail, *tmp; - assert(PyList_CheckExact(names)); - assert(len >= 1); - /* Deal with the joys of natural language. */ - switch (len) { - case 1: - name_str = PyList_GET_ITEM(names, 0); - Py_INCREF(name_str); - break; - case 2: - name_str = PyUnicode_FromFormat("%U and %U", - PyList_GET_ITEM(names, len - 2), - PyList_GET_ITEM(names, len - 1)); - break; - default: - tail = PyUnicode_FromFormat(", %U, and %U", - PyList_GET_ITEM(names, len - 2), - PyList_GET_ITEM(names, len - 1)); - if (tail == NULL) - return; - /* Chop off the last two objects in the list. This shouldn't actually - fail, but we can't be too careful. */ - err = PyList_SetSlice(names, len - 2, len, NULL); - if (err == -1) { - Py_DECREF(tail); - return; - } - /* Stitch everything up into a nice comma-separated list. */ - comma = PyUnicode_FromString(", "); - if (comma == NULL) { - Py_DECREF(tail); - return; - } - tmp = PyUnicode_Join(comma, names); - Py_DECREF(comma); - if (tmp == NULL) { - Py_DECREF(tail); - return; - } - name_str = PyUnicode_Concat(tmp, tail); - Py_DECREF(tmp); - Py_DECREF(tail); - break; - } - if (name_str == NULL) - return; - _PyErr_Format(tstate, PyExc_TypeError, - "%U() missing %i required %s argument%s: %U", - qualname, - len, - kind, - len == 1 ? "" : "s", - name_str); - Py_DECREF(name_str); -} +/* Interpreter main loop */ -static void -missing_arguments(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t missing, Py_ssize_t defcount, - PyObject **localsplus, PyObject *qualname) +PyObject * +PyEval_EvalFrame(PyFrameObject *f) { - Py_ssize_t i, j = 0; - Py_ssize_t start, end; - int positional = (defcount != -1); - const char *kind = positional ? "positional" : "keyword-only"; - PyObject *missing_names; - - /* Compute the names of the arguments that are missing. */ - missing_names = PyList_New(missing); - if (missing_names == NULL) - return; - if (positional) { - start = 0; - end = co->co_argcount - defcount; - } - else { - start = co->co_argcount; - end = start + co->co_kwonlyargcount; - } - for (i = start; i < end; i++) { - if (localsplus[i] == NULL) { - PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *name = PyObject_Repr(raw); - if (name == NULL) { - Py_DECREF(missing_names); - return; - } - PyList_SET_ITEM(missing_names, j++, name); - } - } - assert(j == missing); - format_missing(tstate, kind, co, missing_names, qualname); - Py_DECREF(missing_names); + /* Function kept for backward compatibility */ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_EvalFrame(tstate, f->f_frame, 0); } -static void -too_many_positional(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t given, PyObject *defaults, - PyObject **localsplus, PyObject *qualname) +PyObject * +PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { - int plural; - Py_ssize_t kwonly_given = 0; - Py_ssize_t i; - PyObject *sig, *kwonly_sig; - Py_ssize_t co_argcount = co->co_argcount; + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_EvalFrame(tstate, f->f_frame, throwflag); +} - assert((co->co_flags & CO_VARARGS) == 0); - /* Count missing keyword-only args. */ - for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { - if (localsplus[i] != NULL) { - kwonly_given++; +#include "ceval_macros.h" + +int _Py_CheckRecursiveCallPy( + PyThreadState *tstate) +{ + if (tstate->recursion_headroom) { + if (tstate->py_recursion_remaining < -50) { + /* Overflowing while handling an overflow. Give up. */ + Py_FatalError("Cannot recover from Python stack overflow."); } } - Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults); - if (defcount) { - Py_ssize_t atleast = co_argcount - defcount; - plural = 1; - sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount); - } else { - plural = (co_argcount != 1); - sig = PyUnicode_FromFormat("%zd", co_argcount); - } - if (sig == NULL) - return; - if (kwonly_given) { - const char *format = " positional argument%s (and %zd keyword-only argument%s)"; - kwonly_sig = PyUnicode_FromFormat(format, - given != 1 ? "s" : "", - kwonly_given, - kwonly_given != 1 ? "s" : ""); - if (kwonly_sig == NULL) { - Py_DECREF(sig); - return; + if (tstate->py_recursion_remaining <= 0) { + tstate->recursion_headroom++; + _PyErr_Format(tstate, PyExc_RecursionError, + "maximum recursion depth exceeded"); + tstate->recursion_headroom--; + return -1; } } - else { - /* This will not fail. */ - kwonly_sig = PyUnicode_FromString(""); - assert(kwonly_sig != NULL); - } - _PyErr_Format(tstate, PyExc_TypeError, - "%U() takes %U positional argument%s but %zd%U %s given", - qualname, - sig, - plural ? "s" : "", - given, - kwonly_sig, - given == 1 && !kwonly_given ? "was" : "were"); - Py_DECREF(sig); - Py_DECREF(kwonly_sig); + return 0; } -static int -positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, - Py_ssize_t kwcount, PyObject* kwnames, - PyObject *qualname) -{ - int posonly_conflicts = 0; - PyObject* posonly_names = PyList_New(0); - if (posonly_names == NULL) { - goto fail; - } - for(int k=0; k < co->co_posonlyargcount; k++){ - PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k); - - for (int k2=0; k2 0) { - if(PyList_Append(posonly_names, kwname) != 0) { - goto fail; - } - posonly_conflicts++; - } else if (cmp < 0) { - goto fail; - } +extern const struct _PyCode_DEF(8) _Py_InitCleanup; - } - } - if (posonly_conflicts) { - PyObject* comma = PyUnicode_FromString(", "); - if (comma == NULL) { - goto fail; - } - PyObject* error_names = PyUnicode_Join(comma, posonly_names); - Py_DECREF(comma); - if (error_names == NULL) { - goto fail; - } - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got some positional-only arguments passed" - " as keyword arguments: '%U'", - qualname, error_names); - Py_DECREF(error_names); - goto fail; - } +#ifdef Py_DEBUG +extern void _PyUOpPrint(const _PyUOpInstruction *uop); +#endif - Py_DECREF(posonly_names); - return 0; -fail: - Py_XDECREF(posonly_names); - return 1; +/* Disable unused label warnings. They are handy for debugging, even + if computed gotos aren't used. */ -} +/* TBD - what about other compilers? */ +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-label" +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(push) +# pragma warning(disable:4102) +#endif -static inline unsigned char * -scan_back_to_entry_start(unsigned char *p) { - for (; (p[0]&128) == 0; p--); - return p; -} +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 2 -static inline unsigned char * -skip_to_next_entry(unsigned char *p, unsigned char *end) { - while (p < end && ((p[0] & 128) == 0)) { - p++; - } - return p; -} +#if defined(_MSC_VER) && defined(_Py_USING_PGO) +/* gh-111786: _PyEval_EvalFrameDefault is too large to optimize for speed with + PGO on MSVC. Disable that optimization temporarily. If this is fixed + upstream, we should gate this on the version of MSVC. + */ +# pragma optimize("t", off) +/* This setting is reversed below following _PyEval_EvalFrameDefault */ +#endif +PyObject* _Py_HOT_FUNCTION +_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) +{ + _Py_EnsureTstateNotNULL(tstate); + CALL_STAT_INC(pyeval_calls); -#define MAX_LINEAR_SEARCH 40 +#if USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" +#endif -static int -get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) -{ - unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); - unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); - /* Invariants: - * start_table == end_table OR - * start_table points to a legal entry and end_table points - * beyond the table or to a legal entry that is after index. - */ - if (end - start > MAX_LINEAR_SEARCH) { - int offset; - parse_varint(start, &offset); - if (offset > index) { - return 0; - } - do { - unsigned char * mid = start + ((end-start)>>1); - mid = scan_back_to_entry_start(mid); - parse_varint(mid, &offset); - if (offset > index) { - end = mid; - } - else { - start = mid; - } +#ifdef Py_STATS + int lastopcode = 0; +#endif + uint8_t opcode; /* Current opcode */ + int oparg; /* Current opcode argument, if any */ +#ifdef LLTRACE + int lltrace = 0; +#endif - } while (end - start > MAX_LINEAR_SEARCH); - } - unsigned char *scan = start; - while (scan < end) { - int start_offset, size; - scan = parse_varint(scan, &start_offset); - if (start_offset > index) { - break; - } - scan = parse_varint(scan, &size); - if (start_offset + size > index) { - scan = parse_varint(scan, handler); - int depth_and_lasti; - parse_varint(scan, &depth_and_lasti); - *level = depth_and_lasti >> 1; - *lasti = depth_and_lasti & 1; - return 1; - } - scan = skip_to_next_entry(scan, end); - } - return 0; -} + _PyInterpreterFrame entry_frame; -static int -initialize_locals(PyThreadState *tstate, PyFunctionObject *func, - PyObject **localsplus, PyObject *const *args, - Py_ssize_t argcount, PyObject *kwnames) -{ - PyCodeObject *co = (PyCodeObject*)func->func_code; - const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; - /* Create a dictionary for keyword parameters (**kwags) */ - PyObject *kwdict; - Py_ssize_t i; - if (co->co_flags & CO_VARKEYWORDS) { - kwdict = PyDict_New(); - if (kwdict == NULL) { - goto fail_pre_positional; - } - i = total_args; - if (co->co_flags & CO_VARARGS) { - i++; - } - assert(localsplus[i] == NULL); - localsplus[i] = kwdict; - } - else { - kwdict = NULL; - } - /* Copy all positional arguments into local variables */ - Py_ssize_t j, n; - if (argcount > co->co_argcount) { - n = co->co_argcount; - } - else { - n = argcount; - } - for (j = 0; j < n; j++) { - PyObject *x = args[j]; - assert(localsplus[j] == NULL); - localsplus[j] = x; - } +#ifdef Py_DEBUG + /* Set these to invalid but identifiable values for debugging. */ + entry_frame.f_funcobj = (PyObject*)0xaaa0; + entry_frame.f_locals = (PyObject*)0xaaa1; + entry_frame.frame_obj = (PyFrameObject*)0xaaa2; + entry_frame.f_globals = (PyObject*)0xaaa3; + entry_frame.f_builtins = (PyObject*)0xaaa4; +#endif + entry_frame.f_executable = Py_None; + entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; + entry_frame.stacktop = 0; + entry_frame.owner = FRAME_OWNED_BY_CSTACK; + entry_frame.return_offset = 0; + /* Push frame */ + entry_frame.previous = tstate->current_frame; + frame->previous = &entry_frame; + tstate->current_frame = frame; - /* Pack other positional arguments into the *args argument */ - if (co->co_flags & CO_VARARGS) { - PyObject *u = NULL; - if (argcount == n) { - u = (PyObject *)&_Py_SINGLETON(tuple_empty); - } - else { - assert(args != NULL); - u = _PyTuple_FromArraySteal(args + n, argcount - n); - } - if (u == NULL) { - goto fail_post_positional; - } - assert(localsplus[total_args] == NULL); - localsplus[total_args] = u; + tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); + if (_Py_EnterRecursiveCallTstate(tstate, "")) { + tstate->c_recursion_remaining--; + tstate->py_recursion_remaining--; + goto exit_unwind; } - else if (argcount > n) { - /* Too many postional args. Error is reported later */ - for (j = n; j < argcount; j++) { - Py_DECREF(args[j]); + + /* support for generator.throw() */ + if (throwflag) { + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; } + /* Because this avoids the RESUME, + * we need to update instrumentation */ + _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + monitor_throw(tstate, frame, frame->instr_ptr); + /* TO DO -- Monitor throw entry. */ + goto resume_with_error; } - /* Handle keyword arguments */ - if (kwnames != NULL) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (i = 0; i < kwcount; i++) { - PyObject **co_varnames; - PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); - PyObject *value = args[i+argcount]; - Py_ssize_t j; + /* Local "register" variables. + * These are cached values from the frame and code object. */ + _Py_CODEUNIT *next_instr; + PyObject **stack_pointer; - if (keyword == NULL || !PyUnicode_Check(keyword)) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() keywords must be strings", - func->func_qualname); - goto kw_fail; - } +#ifndef _Py_JIT + /* Tier 2 interpreter state */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif - /* Speed hack: do raw pointer compares. As names are - normally interned this should almost always hit. */ - co_varnames = ((PyTupleObject *)(co->co_localsplusnames))->ob_item; - for (j = co->co_posonlyargcount; j < total_args; j++) { - PyObject *varname = co_varnames[j]; - if (varname == keyword) { - goto kw_found; - } - } +start_frame: + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } - /* Slow fallback, just in case */ - for (j = co->co_posonlyargcount; j < total_args; j++) { - PyObject *varname = co_varnames[j]; - int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ); - if (cmp > 0) { - goto kw_found; - } - else if (cmp < 0) { - goto kw_fail; - } - } + next_instr = frame->instr_ptr; +resume_frame: + stack_pointer = _PyFrame_GetStackPointer(frame); - assert(j >= total_args); - if (kwdict == NULL) { +#ifdef LLTRACE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif - if (co->co_posonlyargcount - && positional_only_passed_as_keyword(tstate, co, - kwcount, kwnames, - func->func_qualname)) - { - goto kw_fail; - } +#ifdef Py_DEBUG + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); +#endif - PyObject* suggestion_keyword = NULL; - if (total_args > co->co_posonlyargcount) { - PyObject* possible_keywords = PyList_New(total_args - co->co_posonlyargcount); + DISPATCH(); - if (!possible_keywords) { - PyErr_Clear(); - } else { - for (Py_ssize_t k = co->co_posonlyargcount; k < total_args; k++) { - PyList_SET_ITEM(possible_keywords, k - co->co_posonlyargcount, co_varnames[k]); - } + { + /* Start instructions */ +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + { - suggestion_keyword = _Py_CalculateSuggestions(possible_keywords, keyword); - Py_DECREF(possible_keywords); - } - } +#include "generated_cases.c.h" - if (suggestion_keyword) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got an unexpected keyword argument '%S'. Did you mean '%S'?", - func->func_qualname, keyword, suggestion_keyword); - Py_DECREF(suggestion_keyword); - } else { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got an unexpected keyword argument '%S'", - func->func_qualname, keyword); - } + /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, + * because it needs to capture frame->instr_ptr before it is updated, + * as happens in the standard instruction prologue. + */ +#if USE_COMPUTED_GOTOS + TARGET_INSTRUMENTED_LINE: +#else + case INSTRUMENTED_LINE: +#endif + { + _Py_CODEUNIT *prev = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + int original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } - goto kw_fail; - } - if (PyDict_SetItem(kwdict, keyword, value) == -1) { - goto kw_fail; - } - Py_DECREF(value); - continue; +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; - kw_fail: - for (;i < kwcount; i++) { - PyObject *value = args[i+argcount]; - Py_DECREF(value); - } - goto fail_post_args; + } /* End instructions */ - kw_found: - if (localsplus[j] != NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got multiple values for argument '%S'", - func->func_qualname, keyword); - goto kw_fail; - } - localsplus[j] = value; - } - } + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); - /* Check the number of positional arguments */ - if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { - too_many_positional(tstate, co, argcount, func->func_defaults, localsplus, - func->func_qualname); - goto fail_post_args; - } +pop_4_error: + STACK_SHRINK(1); +pop_3_error: + STACK_SHRINK(1); +pop_2_error: + STACK_SHRINK(1); +pop_1_error: + STACK_SHRINK(1); +error: + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif - /* Add missing positional arguments (copy default values from defs) */ - if (argcount < co->co_argcount) { - Py_ssize_t defcount = func->func_defaults == NULL ? 0 : PyTuple_GET_SIZE(func->func_defaults); - Py_ssize_t m = co->co_argcount - defcount; - Py_ssize_t missing = 0; - for (i = argcount; i < m; i++) { - if (localsplus[i] == NULL) { - missing++; + /* Log traceback info. */ + assert(frame != &entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); } } - if (missing) { - missing_arguments(tstate, co, missing, defcount, localsplus, - func->func_qualname); - goto fail_post_args; - } - if (n > m) - i = n - m; - else - i = 0; - if (defcount) { - PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0); - for (; i < defcount; i++) { - if (localsplus[m+i] == NULL) { - PyObject *def = defs[i]; - localsplus[m+i] = Py_NewRef(def); + monitor_raise(tstate, frame, next_instr-1); +exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + PyObject **stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyObject *o = POP(); + Py_XDECREF(o); } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; } - } - } - /* Add missing keyword arguments (copy default values from kwdefs) */ - if (co->co_kwonlyargcount > 0) { - Py_ssize_t missing = 0; - for (i = co->co_argcount; i < total_args; i++) { - if (localsplus[i] != NULL) - continue; - PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); - if (func->func_kwdefaults != NULL) { - PyObject *def; - if (PyDict_GetItemRef(func->func_kwdefaults, varname, &def) < 0) { - goto fail_post_args; - } - if (def) { - localsplus[i] = def; - continue; + assert(STACK_LEVEL() >= level); + PyObject **new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyObject *v = POP(); + Py_XDECREF(v); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; } + PUSH(lasti); } - missing++; - } - if (missing) { - missing_arguments(tstate, co, missing, -1, localsplus, - func->func_qualname); - goto fail_post_args; - } - } - return 0; -fail_pre_positional: - for (j = 0; j < argcount; j++) { - Py_DECREF(args[j]); - } - /* fall through */ -fail_post_positional: - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (j = argcount; j < argcount+kwcount; j++) { - Py_DECREF(args[j]); + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(exc); + next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); } } - /* fall through */ -fail_post_args: - return -1; -} -static void -clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) -{ - assert(frame->owner == FRAME_OWNED_BY_THREAD); - // Make sure that this is, indeed, the top frame. We can't check this in - // _PyThreadState_PopFrame, since f_code is already cleared at that point: - assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == - tstate->datastack_top); - tstate->c_recursion_remaining--; - assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); - _PyFrame_ClearExceptCode(frame); - Py_DECREF(frame->f_executable); - tstate->c_recursion_remaining++; - _PyThreadState_PopFrame(tstate, frame); -} +exit_unwind: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == &entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } -static void -clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) -{ - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); - PyGenObject *gen = _PyFrame_GetGenerator(frame); - gen->gi_frame_state = FRAME_CLEARED; - assert(tstate->exc_info == &gen->gi_exc_state); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - tstate->c_recursion_remaining--; - assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); - _PyFrame_ClearExceptCode(frame); - _PyErr_ClearExcState(&gen->gi_exc_state); - tstate->c_recursion_remaining++; - frame->previous = NULL; -} +resume_with_error: + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; -void -_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) -{ - if (frame->owner == FRAME_OWNED_BY_THREAD) { - clear_thread_frame(tstate, frame); - } - else { - clear_gen_frame(tstate, frame); - } -} -/* Consumes references to func, locals and all the args */ -static _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames) -{ - PyCodeObject * code = (PyCodeObject *)func->func_code; - CALL_STAT_INC(frames_pushed); - _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); - if (frame == NULL) { - goto fail; - } - _PyFrame_Initialize(frame, func, locals, code, 0); - if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { - assert(frame->owner == FRAME_OWNED_BY_THREAD); - clear_thread_frame(tstate, frame); - return NULL; - } - return frame; -fail: - /* Consume the references */ - for (size_t i = 0; i < argcount; i++) { - Py_DECREF(args[i]); - } - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_DECREF(args[i+argcount]); - } - } - PyErr_NoMemory(); - return NULL; -} -/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. - Steals references to func, callargs and kwargs. -*/ -static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) -{ - bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); - PyObject *kwnames = NULL; - PyObject *const *newargs; - if (has_dict) { - newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); - if (newargs == NULL) { - Py_DECREF(func); - goto error; - } - } - else { - newargs = &PyTuple_GET_ITEM(callargs, 0); - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); +// Tier 2 is also here! +enter_tier_two: + +#ifdef _Py_JIT + assert(0); +#else + +#undef LOAD_IP +#define LOAD_IP(UNUSED) (void)0 + +#undef GOTO_ERROR +#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two + +#ifdef Py_STATS +// Disable these macros that apply to Tier 1 stats when we are in Tier 2 +#undef STAT_INC +#define STAT_INC(opname, name) ((void)0) +#undef STAT_DEC +#define STAT_DEC(opname, name) ((void)0) +#undef CALL_STAT_INC +#define CALL_STAT_INC(name) ((void)0) +#endif + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION 0 + +#ifdef Py_DEBUG + #define DPRINTF(level, ...) \ + if (lltrace >= (level)) { printf(__VA_ARGS__); } +#else + #define DPRINTF(level, ...) +#endif + + ; // dummy statement after a label, before a declaration + uint16_t uopcode; +#ifdef Py_STATS + int lastuop = 0; + uint64_t trace_uop_execution_counter = 0; +#endif + + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); +tier2_dispatch: + for (;;) { + uopcode = next_uop->opcode; +#ifdef Py_DEBUG + if (lltrace >= 3) { + if (next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT) { + printf("%4d uop: ", 0); + } + else { + printf("%4d uop: ", (int)(next_uop - current_executor->trace)); + } + _PyUOpPrint(next_uop); + printf(" stack_level=%d\n", + (int)(stack_pointer - _PyFrame_Stackbase(frame))); } - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); - if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); - } - /* No need to decref func here because the reference has been stolen by - _PyEvalFramePushAndInit. - */ - Py_DECREF(callargs); - Py_XDECREF(kwargs); - return new_frame; -error: - Py_DECREF(callargs); - Py_XDECREF(kwargs); - return NULL; -} +#endif + next_uop++; + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(uopcode, execution_count); + UOP_PAIR_INC(uopcode, lastuop); +#ifdef Py_STATS + trace_uop_execution_counter++; +#endif + + switch (uopcode) { + +#include "executor_cases.c.h" + + default: +#ifdef Py_DEBUG + { + printf("Unknown uop: "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); + Py_FatalError("Unknown uop"); + } +#else + Py_UNREACHABLE(); +#endif -PyObject * -_PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, - PyObject* const* args, size_t argcount, - PyObject *kwnames) -{ - /* _PyEvalFramePushAndInit consumes the references - * to func, locals and all its arguments */ - Py_INCREF(func); - Py_XINCREF(locals); - for (size_t i = 0; i < argcount; i++) { - Py_INCREF(args[i]); - } - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_INCREF(args[i+argcount]); } } - _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( - tstate, func, locals, args, argcount, kwnames); - if (frame == NULL) { - return NULL; - } - EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR); - return _PyEval_EvalFrame(tstate, frame, 0); -} -/* Legacy API */ -PyObject * -PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, - PyObject *const *args, int argcount, - PyObject *const *kws, int kwcount, - PyObject *const *defs, int defcount, - PyObject *kwdefs, PyObject *closure) -{ - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *res = NULL; - PyObject *defaults = _PyTuple_FromArray(defs, defcount); - if (defaults == NULL) { - return NULL; - } - PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref - if (builtins == NULL) { - Py_DECREF(defaults); - return NULL; - } - if (locals == NULL) { - locals = globals; - } - PyObject *kwnames = NULL; - PyObject *const *allargs; - PyObject **newargs = NULL; - PyFunctionObject *func = NULL; - if (kwcount == 0) { - allargs = args; +jump_to_error_target: +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("Error: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d -> %s]\n", + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); } - else { - kwnames = PyTuple_New(kwcount); - if (kwnames == NULL) { - goto fail; - } - newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount)); - if (newargs == NULL) { - goto fail; - } - for (int i = 0; i < argcount; i++) { - newargs[i] = args[i]; - } - for (int i = 0; i < kwcount; i++) { - PyTuple_SET_ITEM(kwnames, i, Py_NewRef(kws[2*i])); - newargs[argcount+i] = kws[2*i+1]; - } - allargs = newargs; +#endif + assert (next_uop[-1].format == UOP_FORMAT_JUMP); + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +error_tier_two: + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + assert(next_uop[-1].format == UOP_FORMAT_TARGET); + frame->return_offset = 0; // Don't leave this random + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(current_executor); + tstate->previous_executor = NULL; + goto resume_with_error; + +jump_to_jump_target: + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + target = uop_get_jump_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +exit_to_tier1: + assert(next_uop[-1].format == UOP_FORMAT_TARGET); + next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("DEOPT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" -> %s]\n", + _PyOpcode_OpName[next_instr->op.code]); } - PyFrameConstructor constr = { - .fc_globals = globals, - .fc_builtins = builtins, - .fc_name = ((PyCodeObject *)_co)->co_name, - .fc_qualname = ((PyCodeObject *)_co)->co_name, - .fc_code = _co, - .fc_defaults = defaults, - .fc_kwdefaults = kwdefs, - .fc_closure = closure - }; - func = _PyFunction_FromConstructor(&constr); - if (func == NULL) { - goto fail; +#endif + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + Py_DECREF(current_executor); + tstate->previous_executor = NULL; + DISPATCH(); + +exit_to_trace: + assert(next_uop[-1].format == UOP_FORMAT_EXIT); + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + uint32_t exit_index = next_uop[-1].exit_index; + assert(exit_index < current_executor->exit_count); + _PyExitData *exit = ¤t_executor->exits[exit_index]; +#ifdef Py_DEBUG + if (lltrace >= 2) { + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %u, temp %d, target %d -> %s]\n", + exit_index, exit->temperature.as_counter, exit->target, + _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); } - EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY); - res = _PyEval_Vector(tstate, func, locals, - allargs, argcount, - kwnames); -fail: - Py_XDECREF(func); - Py_XDECREF(kwnames); - PyMem_Free(newargs); - Py_DECREF(defaults); - return res; +#endif + Py_INCREF(exit->executor); + tstate->previous_executor = (PyObject *)current_executor; + GOTO_TIER_TWO(exit->executor); + +#endif // _Py_JIT + } +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(pop) +# pragma optimize("", on) +#endif -/* Logic for the raise statement (too complicated for inlining). - This *consumes* a reference count to each of its arguments. */ -static int -do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) +static void +format_missing(PyThreadState *tstate, const char *kind, + PyCodeObject *co, PyObject *names, PyObject *qualname) { - PyObject *type = NULL, *value = NULL; + int err; + Py_ssize_t len = PyList_GET_SIZE(names); + PyObject *name_str, *comma, *tail, *tmp; - if (exc == NULL) { - /* Reraise */ - _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); - exc = exc_info->exc_value; - if (Py_IsNone(exc) || exc == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, - "No active exception to reraise"); - return 0; + assert(PyList_CheckExact(names)); + assert(len >= 1); + /* Deal with the joys of natural language. */ + switch (len) { + case 1: + name_str = PyList_GET_ITEM(names, 0); + Py_INCREF(name_str); + break; + case 2: + name_str = PyUnicode_FromFormat("%U and %U", + PyList_GET_ITEM(names, len - 2), + PyList_GET_ITEM(names, len - 1)); + break; + default: + tail = PyUnicode_FromFormat(", %U, and %U", + PyList_GET_ITEM(names, len - 2), + PyList_GET_ITEM(names, len - 1)); + if (tail == NULL) + return; + /* Chop off the last two objects in the list. This shouldn't actually + fail, but we can't be too careful. */ + err = PyList_SetSlice(names, len - 2, len, NULL); + if (err == -1) { + Py_DECREF(tail); + return; } - Py_INCREF(exc); - assert(PyExceptionInstance_Check(exc)); - _PyErr_SetRaisedException(tstate, exc); - return 1; + /* Stitch everything up into a nice comma-separated list. */ + comma = PyUnicode_FromString(", "); + if (comma == NULL) { + Py_DECREF(tail); + return; + } + tmp = PyUnicode_Join(comma, names); + Py_DECREF(comma); + if (tmp == NULL) { + Py_DECREF(tail); + return; + } + name_str = PyUnicode_Concat(tmp, tail); + Py_DECREF(tmp); + Py_DECREF(tail); + break; } + if (name_str == NULL) + return; + _PyErr_Format(tstate, PyExc_TypeError, + "%U() missing %i required %s argument%s: %U", + qualname, + len, + kind, + len == 1 ? "" : "s", + name_str); + Py_DECREF(name_str); +} - /* We support the following forms of raise: - raise - raise - raise */ +static void +missing_arguments(PyThreadState *tstate, PyCodeObject *co, + Py_ssize_t missing, Py_ssize_t defcount, + PyObject **localsplus, PyObject *qualname) +{ + Py_ssize_t i, j = 0; + Py_ssize_t start, end; + int positional = (defcount != -1); + const char *kind = positional ? "positional" : "keyword-only"; + PyObject *missing_names; - if (PyExceptionClass_Check(exc)) { - type = exc; - value = _PyObject_CallNoArgs(exc); - if (value == NULL) - goto raise_error; - if (!PyExceptionInstance_Check(value)) { - _PyErr_Format(tstate, PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - type, Py_TYPE(value)); - goto raise_error; - } - } - else if (PyExceptionInstance_Check(exc)) { - value = exc; - type = PyExceptionInstance_Class(exc); - Py_INCREF(type); + /* Compute the names of the arguments that are missing. */ + missing_names = PyList_New(missing); + if (missing_names == NULL) + return; + if (positional) { + start = 0; + end = co->co_argcount - defcount; } else { - /* Not something you can raise. You get an exception - anyway, just not what you specified :-) */ - Py_DECREF(exc); - _PyErr_SetString(tstate, PyExc_TypeError, - "exceptions must derive from BaseException"); - goto raise_error; + start = co->co_argcount; + end = start + co->co_kwonlyargcount; } - - assert(type != NULL); - assert(value != NULL); - - if (cause) { - PyObject *fixed_cause; - if (PyExceptionClass_Check(cause)) { - fixed_cause = _PyObject_CallNoArgs(cause); - if (fixed_cause == NULL) - goto raise_error; - if (!PyExceptionInstance_Check(fixed_cause)) { - _PyErr_Format(tstate, PyExc_TypeError, - "calling %R should have returned an instance of " - "BaseException, not %R", - cause, Py_TYPE(fixed_cause)); - goto raise_error; + for (i = start; i < end; i++) { + if (localsplus[i] == NULL) { + PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *name = PyObject_Repr(raw); + if (name == NULL) { + Py_DECREF(missing_names); + return; } - Py_DECREF(cause); - } - else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - } - else if (Py_IsNone(cause)) { - Py_DECREF(cause); - fixed_cause = NULL; - } - else { - _PyErr_SetString(tstate, PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto raise_error; + PyList_SET_ITEM(missing_names, j++, name); } - PyException_SetCause(value, fixed_cause); } - - _PyErr_SetObject(tstate, type, value); - /* _PyErr_SetObject incref's its arguments */ - Py_DECREF(value); - Py_DECREF(type); - return 0; - -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(cause); - return 0; + assert(j == missing); + format_missing(tstate, kind, co, missing_names, qualname); + Py_DECREF(missing_names); } -/* Logic for matching an exception in an except* clause (too - complicated for inlining). -*/ - -int -_PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, - PyObject **match, PyObject **rest) +static void +too_many_positional(PyThreadState *tstate, PyCodeObject *co, + Py_ssize_t given, PyObject *defaults, + PyObject **localsplus, PyObject *qualname) { - if (Py_IsNone(exc_value)) { - *match = Py_NewRef(Py_None); - *rest = Py_NewRef(Py_None); - return 0; - } - assert(PyExceptionInstance_Check(exc_value)); + int plural; + Py_ssize_t kwonly_given = 0; + Py_ssize_t i; + PyObject *sig, *kwonly_sig; + Py_ssize_t co_argcount = co->co_argcount; - if (PyErr_GivenExceptionMatches(exc_value, match_type)) { - /* Full match of exc itself */ - bool is_eg = _PyBaseExceptionGroup_Check(exc_value); - if (is_eg) { - *match = Py_NewRef(exc_value); - } - else { - /* naked exception - wrap it */ - PyObject *excs = PyTuple_Pack(1, exc_value); - if (excs == NULL) { - return -1; - } - PyObject *wrapped = _PyExc_CreateExceptionGroup("", excs); - Py_DECREF(excs); - if (wrapped == NULL) { - return -1; - } - *match = wrapped; + assert((co->co_flags & CO_VARARGS) == 0); + /* Count missing keyword-only args. */ + for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) { + if (localsplus[i] != NULL) { + kwonly_given++; } - *rest = Py_NewRef(Py_None); - return 0; } - - /* exc_value does not match match_type. - * Check for partial match if it's an exception group. - */ - if (_PyBaseExceptionGroup_Check(exc_value)) { - PyObject *pair = PyObject_CallMethod(exc_value, "split", "(O)", - match_type); - if (pair == NULL) { - return -1; + Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults); + if (defcount) { + Py_ssize_t atleast = co_argcount - defcount; + plural = 1; + sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount); + } + else { + plural = (co_argcount != 1); + sig = PyUnicode_FromFormat("%zd", co_argcount); + } + if (sig == NULL) + return; + if (kwonly_given) { + const char *format = " positional argument%s (and %zd keyword-only argument%s)"; + kwonly_sig = PyUnicode_FromFormat(format, + given != 1 ? "s" : "", + kwonly_given, + kwonly_given != 1 ? "s" : ""); + if (kwonly_sig == NULL) { + Py_DECREF(sig); + return; } - assert(PyTuple_CheckExact(pair)); - assert(PyTuple_GET_SIZE(pair) == 2); - *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0)); - *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1)); - Py_DECREF(pair); - return 0; } - /* no match */ - *match = Py_NewRef(Py_None); - *rest = Py_NewRef(exc_value); - return 0; + else { + /* This will not fail. */ + kwonly_sig = PyUnicode_FromString(""); + assert(kwonly_sig != NULL); + } + _PyErr_Format(tstate, PyExc_TypeError, + "%U() takes %U positional argument%s but %zd%U %s given", + qualname, + sig, + plural ? "s" : "", + given, + kwonly_sig, + given == 1 && !kwonly_given ? "was" : "were"); + Py_DECREF(sig); + Py_DECREF(kwonly_sig); } -/* Iterate v argcnt times and store the results on the stack (via decreasing - sp). Return 1 for success, 0 if error. - - If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack - with a variable target. -*/ - -int -_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, - int argcnt, int argcntafter, PyObject **sp) +static int +positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co, + Py_ssize_t kwcount, PyObject* kwnames, + PyObject *qualname) { - int i = 0, j = 0; - Py_ssize_t ll = 0; - PyObject *it; /* iter(v) */ - PyObject *w; - PyObject *l = NULL; /* variable list */ - - assert(v != NULL); - - it = PyObject_GetIter(v); - if (it == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - Py_TYPE(v)->tp_iter == NULL && !PySequence_Check(v)) - { - _PyErr_Format(tstate, PyExc_TypeError, - "cannot unpack non-iterable %.200s object", - Py_TYPE(v)->tp_name); - } - return 0; + int posonly_conflicts = 0; + PyObject* posonly_names = PyList_New(0); + if (posonly_names == NULL) { + goto fail; } + for(int k=0; k < co->co_posonlyargcount; k++){ + PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k); - for (; i < argcnt; i++) { - w = PyIter_Next(it); - if (w == NULL) { - /* Iterator done, via error or exhaustion. */ - if (!_PyErr_Occurred(tstate)) { - if (argcntafter == -1) { - _PyErr_Format(tstate, PyExc_ValueError, - "not enough values to unpack " - "(expected %d, got %d)", - argcnt, i); + for (int k2=0; k2 0) { + if(PyList_Append(posonly_names, kwname) != 0) { + goto fail; } + posonly_conflicts++; + } else if (cmp < 0) { + goto fail; } - goto Error; + } - *--sp = w; } - - if (argcntafter == -1) { - /* We better have exhausted the iterator now. */ - w = PyIter_Next(it); - if (w == NULL) { - if (_PyErr_Occurred(tstate)) - goto Error; - Py_DECREF(it); - return 1; + if (posonly_conflicts) { + PyObject* comma = PyUnicode_FromString(", "); + if (comma == NULL) { + goto fail; } - Py_DECREF(w); - _PyErr_Format(tstate, PyExc_ValueError, - "too many values to unpack (expected %d)", - argcnt); - goto Error; + PyObject* error_names = PyUnicode_Join(comma, posonly_names); + Py_DECREF(comma); + if (error_names == NULL) { + goto fail; + } + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got some positional-only arguments passed" + " as keyword arguments: '%U'", + qualname, error_names); + Py_DECREF(error_names); + goto fail; } - l = PySequence_List(it); - if (l == NULL) - goto Error; - *--sp = l; - i++; - - ll = PyList_GET_SIZE(l); - if (ll < argcntafter) { - _PyErr_Format(tstate, PyExc_ValueError, - "not enough values to unpack (expected at least %d, got %zd)", - argcnt + argcntafter, argcnt + ll); - goto Error; - } + Py_DECREF(posonly_names); + return 0; - /* Pop the "after-variable" args off the list. */ - for (j = argcntafter; j > 0; j--, i++) { - *--sp = PyList_GET_ITEM(l, ll - j); - } - /* Resize the list. */ - Py_SET_SIZE(l, ll - argcntafter); - Py_DECREF(it); +fail: + Py_XDECREF(posonly_names); return 1; -Error: - for (; i > 0; i--, sp++) - Py_DECREF(*sp); - Py_XDECREF(it); - return 0; } -static int -do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event) -{ - assert(event < _PY_MONITORING_UNGROUPED_EVENTS); - if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { - return 0; - } - PyObject *exc = PyErr_GetRaisedException(); - assert(exc != NULL); - int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); - if (err == 0) { - PyErr_SetRaisedException(exc); - } - else { - assert(PyErr_Occurred()); - Py_DECREF(exc); - } - return err; -} -static inline bool -no_tools_for_global_event(PyThreadState *tstate, int event) -{ - return tstate->interp->monitors.tools[event] == 0; +static inline unsigned char * +scan_back_to_entry_start(unsigned char *p) { + for (; (p[0]&128) == 0; p--); + return p; } -static inline bool -no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) -{ - assert(event < _PY_MONITORING_LOCAL_EVENTS); - _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; - if (data) { - return data->active_monitors.tools[event] == 0; - } - else { - return no_tools_for_global_event(tstate, event); +static inline unsigned char * +skip_to_next_entry(unsigned char *p, unsigned char *end) { + while (p < end && ((p[0] & 128) == 0)) { + p++; } + return p; } -static void -monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { - return; - } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); -} -static void -monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) +#define MAX_LINEAR_SEARCH 40 + +static int +get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { - return; + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); + /* Invariants: + * start_table == end_table OR + * start_table points to a legal entry and end_table points + * beyond the table or to a legal entry that is after index. + */ + if (end - start > MAX_LINEAR_SEARCH) { + int offset; + parse_varint(start, &offset); + if (offset > index) { + return 0; + } + do { + unsigned char * mid = start + ((end-start)>>1); + mid = scan_back_to_entry_start(mid); + parse_varint(mid, &offset); + if (offset > index) { + end = mid; + } + else { + start = mid; + } + + } while (end - start > MAX_LINEAR_SEARCH); + } + unsigned char *scan = start; + while (scan < end) { + int start_offset, size; + scan = parse_varint(scan, &start_offset); + if (start_offset > index) { + break; + } + scan = parse_varint(scan, &size); + if (start_offset + size > index) { + scan = parse_varint(scan, handler); + int depth_and_lasti; + parse_varint(scan, &depth_and_lasti); + *level = depth_and_lasti >> 1; + *lasti = depth_and_lasti & 1; + return 1; + } + scan = skip_to_next_entry(scan, end); } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); + return 0; } static int -monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) +initialize_locals(PyThreadState *tstate, PyFunctionObject *func, + PyObject **localsplus, PyObject *const *args, + Py_ssize_t argcount, PyObject *kwnames) { - if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { - return 0; - } - return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); -} + PyCodeObject *co = (PyCodeObject*)func->func_code; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; -static void -monitor_unwind(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { - return; + /* Create a dictionary for keyword parameters (**kwags) */ + PyObject *kwdict; + Py_ssize_t i; + if (co->co_flags & CO_VARKEYWORDS) { + kwdict = PyDict_New(); + if (kwdict == NULL) { + goto fail_pre_positional; + } + i = total_args; + if (co->co_flags & CO_VARARGS) { + i++; + } + assert(localsplus[i] == NULL); + localsplus[i] = kwdict; + } + else { + kwdict = NULL; } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); -} - -static int -monitor_handled(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, PyObject *exc) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { - return 0; + /* Copy all positional arguments into local variables */ + Py_ssize_t j, n; + if (argcount > co->co_argcount) { + n = co->co_argcount; + } + else { + n = argcount; + } + for (j = 0; j < n; j++) { + PyObject *x = args[j]; + assert(localsplus[j] == NULL); + localsplus[j] = x; } - return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); -} -static void -monitor_throw(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) -{ - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { - return; + /* Pack other positional arguments into the *args argument */ + if (co->co_flags & CO_VARARGS) { + PyObject *u = NULL; + if (argcount == n) { + u = (PyObject *)&_Py_SINGLETON(tuple_empty); + } + else { + assert(args != NULL); + u = _PyTuple_FromArraySteal(args + n, argcount - n); + } + if (u == NULL) { + goto fail_post_positional; + } + assert(localsplus[total_args] == NULL); + localsplus[total_args] = u; + } + else if (argcount > n) { + /* Too many postional args. Error is reported later */ + for (j = n; j < argcount; j++) { + Py_DECREF(args[j]); + } } - do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); -} -void -PyThreadState_EnterTracing(PyThreadState *tstate) -{ - assert(tstate->tracing >= 0); - tstate->tracing++; -} + /* Handle keyword arguments */ + if (kwnames != NULL) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < kwcount; i++) { + PyObject **co_varnames; + PyObject *keyword = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = args[i+argcount]; + Py_ssize_t j; -void -PyThreadState_LeaveTracing(PyThreadState *tstate) -{ - assert(tstate->tracing > 0); - tstate->tracing--; -} + if (keyword == NULL || !PyUnicode_Check(keyword)) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() keywords must be strings", + func->func_qualname); + goto kw_fail; + } + /* Speed hack: do raw pointer compares. As names are + normally interned this should almost always hit. */ + co_varnames = ((PyTupleObject *)(co->co_localsplusnames))->ob_item; + for (j = co->co_posonlyargcount; j < total_args; j++) { + PyObject *varname = co_varnames[j]; + if (varname == keyword) { + goto kw_found; + } + } -PyObject* -_PyEval_CallTracing(PyObject *func, PyObject *args) -{ - // Save and disable tracing - PyThreadState *tstate = _PyThreadState_GET(); - int save_tracing = tstate->tracing; - tstate->tracing = 0; + /* Slow fallback, just in case */ + for (j = co->co_posonlyargcount; j < total_args; j++) { + PyObject *varname = co_varnames[j]; + int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ); + if (cmp > 0) { + goto kw_found; + } + else if (cmp < 0) { + goto kw_fail; + } + } - // Call the tracing function - PyObject *result = PyObject_Call(func, args, NULL); + assert(j >= total_args); + if (kwdict == NULL) { - // Restore tracing - tstate->tracing = save_tracing; - return result; -} + if (co->co_posonlyargcount + && positional_only_passed_as_keyword(tstate, co, + kwcount, kwnames, + func->func_qualname)) + { + goto kw_fail; + } -void -PyEval_SetProfile(Py_tracefunc func, PyObject *arg) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, func, arg) < 0) { - /* Log _PySys_Audit() error */ - PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfile"); - } -} + PyObject* suggestion_keyword = NULL; + if (total_args > co->co_posonlyargcount) { + PyObject* possible_keywords = PyList_New(total_args - co->co_posonlyargcount); -void -PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) -{ - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; + if (!possible_keywords) { + PyErr_Clear(); + } else { + for (Py_ssize_t k = co->co_posonlyargcount; k < total_args; k++) { + PyList_SET_ITEM(possible_keywords, k - co->co_posonlyargcount, co_varnames[k]); + } - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); + suggestion_keyword = _Py_CalculateSuggestions(possible_keywords, keyword); + Py_DECREF(possible_keywords); + } + } - while (ts) { - if (_PyEval_SetProfile(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); - } -} + if (suggestion_keyword) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'. Did you mean '%S'?", + func->func_qualname, keyword, suggestion_keyword); + Py_DECREF(suggestion_keyword); + } else { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'", + func->func_qualname, keyword); + } -void -PyEval_SetTrace(Py_tracefunc func, PyObject *arg) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetTrace(tstate, func, arg) < 0) { - /* Log _PySys_Audit() error */ - PyErr_FormatUnraisable("Exception ignored in PyEval_SetTrace"); - } -} + goto kw_fail; + } -void -PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) -{ - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; + if (PyDict_SetItem(kwdict, keyword, value) == -1) { + goto kw_fail; + } + Py_DECREF(value); + continue; - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); + kw_fail: + for (;i < kwcount; i++) { + PyObject *value = args[i+argcount]; + Py_DECREF(value); + } + goto fail_post_args; - while (ts) { - if (_PyEval_SetTrace(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); + kw_found: + if (localsplus[j] != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got multiple values for argument '%S'", + func->func_qualname, keyword); + goto kw_fail; + } + localsplus[j] = value; } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); } -} -int -_PyEval_SetCoroutineOriginTrackingDepth(int depth) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (depth < 0) { - _PyErr_SetString(tstate, PyExc_ValueError, "depth must be >= 0"); - return -1; + /* Check the number of positional arguments */ + if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) { + too_many_positional(tstate, co, argcount, func->func_defaults, localsplus, + func->func_qualname); + goto fail_post_args; } - tstate->coroutine_origin_tracking_depth = depth; - return 0; -} - - -int -_PyEval_GetCoroutineOriginTrackingDepth(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return tstate->coroutine_origin_tracking_depth; -} - -int -_PyEval_SetAsyncGenFirstiter(PyObject *firstiter) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_firstiter", NULL) < 0) { - return -1; + /* Add missing positional arguments (copy default values from defs) */ + if (argcount < co->co_argcount) { + Py_ssize_t defcount = func->func_defaults == NULL ? 0 : PyTuple_GET_SIZE(func->func_defaults); + Py_ssize_t m = co->co_argcount - defcount; + Py_ssize_t missing = 0; + for (i = argcount; i < m; i++) { + if (localsplus[i] == NULL) { + missing++; + } + } + if (missing) { + missing_arguments(tstate, co, missing, defcount, localsplus, + func->func_qualname); + goto fail_post_args; + } + if (n > m) + i = n - m; + else + i = 0; + if (defcount) { + PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0); + for (; i < defcount; i++) { + if (localsplus[m+i] == NULL) { + PyObject *def = defs[i]; + localsplus[m+i] = Py_NewRef(def); + } + } + } } - Py_XSETREF(tstate->async_gen_firstiter, Py_XNewRef(firstiter)); + /* Add missing keyword arguments (copy default values from kwdefs) */ + if (co->co_kwonlyargcount > 0) { + Py_ssize_t missing = 0; + for (i = co->co_argcount; i < total_args; i++) { + if (localsplus[i] != NULL) + continue; + PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (func->func_kwdefaults != NULL) { + PyObject *def; + if (PyDict_GetItemRef(func->func_kwdefaults, varname, &def) < 0) { + goto fail_post_args; + } + if (def) { + localsplus[i] = def; + continue; + } + } + missing++; + } + if (missing) { + missing_arguments(tstate, co, missing, -1, localsplus, + func->func_qualname); + goto fail_post_args; + } + } return 0; -} -PyObject * -_PyEval_GetAsyncGenFirstiter(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return tstate->async_gen_firstiter; +fail_pre_positional: + for (j = 0; j < argcount; j++) { + Py_DECREF(args[j]); + } + /* fall through */ +fail_post_positional: + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (j = argcount; j < argcount+kwcount; j++) { + Py_DECREF(args[j]); + } + } + /* fall through */ +fail_post_args: + return -1; } -int -_PyEval_SetAsyncGenFinalizer(PyObject *finalizer) +static void +clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { - PyThreadState *tstate = _PyThreadState_GET(); - - if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_finalizer", NULL) < 0) { - return -1; - } - - Py_XSETREF(tstate->async_gen_finalizer, Py_XNewRef(finalizer)); - return 0; + assert(frame->owner == FRAME_OWNED_BY_THREAD); + // Make sure that this is, indeed, the top frame. We can't check this in + // _PyThreadState_PopFrame, since f_code is already cleared at that point: + assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + tstate->datastack_top); + tstate->c_recursion_remaining--; + assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); + _PyFrame_ClearExceptCode(frame); + Py_DECREF(frame->f_executable); + tstate->c_recursion_remaining++; + _PyThreadState_PopFrame(tstate, frame); } -PyObject * -_PyEval_GetAsyncGenFinalizer(void) +static void +clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { - PyThreadState *tstate = _PyThreadState_GET(); - return tstate->async_gen_finalizer; + assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_CLEARED; + assert(tstate->exc_info == &gen->gi_exc_state); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + tstate->c_recursion_remaining--; + assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); + _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); + tstate->c_recursion_remaining++; + frame->previous = NULL; } -_PyInterpreterFrame * -_PyEval_GetFrame(void) +void +_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) { - PyThreadState *tstate = _PyThreadState_GET(); - return _PyThreadState_GetFrame(tstate); + if (frame->owner == FRAME_OWNED_BY_THREAD) { + clear_thread_frame(tstate, frame); + } + else { + clear_gen_frame(tstate, frame); + } } -PyFrameObject * -PyEval_GetFrame(void) +/* Consumes references to func, locals and all the args */ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject* const* args, + size_t argcount, PyObject *kwnames) { - _PyInterpreterFrame *frame = _PyEval_GetFrame(); + PyCodeObject * code = (PyCodeObject *)func->func_code; + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); if (frame == NULL) { + goto fail; + } + _PyFrame_Initialize(frame, func, locals, code, 0); + if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + clear_thread_frame(tstate, frame); return NULL; } - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - PyErr_Clear(); + return frame; +fail: + /* Consume the references */ + for (size_t i = 0; i < argcount; i++) { + Py_DECREF(args[i]); } - return f; -} - -PyObject * -_PyEval_GetBuiltins(PyThreadState *tstate) -{ - _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); - if (frame != NULL) { - return frame->f_builtins; + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_DECREF(args[i+argcount]); + } } - return tstate->interp->builtins; -} - -PyObject * -PyEval_GetBuiltins(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return _PyEval_GetBuiltins(tstate); + PyErr_NoMemory(); + return NULL; } -/* Convenience function to get a builtin from its name */ -PyObject * -_PyEval_GetBuiltin(PyObject *name) +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) { - PyObject *attr; - if (PyMapping_GetOptionalItem(PyEval_GetBuiltins(), name, &attr) == 0) { - PyErr_SetObject(PyExc_AttributeError, name); + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs; + if (has_dict) { + newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (newargs == NULL) { + Py_DECREF(func); + goto error; + } } - return attr; -} - -PyObject * -_PyEval_GetBuiltinId(_Py_Identifier *name) -{ - return _PyEval_GetBuiltin(_PyUnicode_FromId(name)); + else { + newargs = &PyTuple_GET_ITEM(callargs, 0); + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; } PyObject * -PyEval_GetLocals(void) +_PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames) { - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); - if (current_frame == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); - return NULL; + /* _PyEvalFramePushAndInit consumes the references + * to func, locals and all its arguments */ + Py_INCREF(func); + Py_XINCREF(locals); + for (size_t i = 0; i < argcount; i++) { + Py_INCREF(args[i]); } - - if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_INCREF(args[i+argcount]); + } + } + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + tstate, func, locals, args, argcount, kwnames); + if (frame == NULL) { return NULL; } - - PyObject *locals = current_frame->f_locals; - assert(locals != NULL); - return locals; + EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR); + return _PyEval_EvalFrame(tstate, frame, 0); } +/* Legacy API */ PyObject * -_PyEval_GetFrameLocals(void) +PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, + PyObject *const *args, int argcount, + PyObject *const *kws, int kwcount, + PyObject *const *defs, int defcount, + PyObject *kwdefs, PyObject *closure) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); - if (current_frame == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + PyObject *res = NULL; + PyObject *defaults = _PyTuple_FromArray(defs, defcount); + if (defaults == NULL) { return NULL; } - - return _PyFrame_GetLocals(current_frame, 1); -} - -PyObject * -PyEval_GetGlobals(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); - if (current_frame == NULL) { + PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref + if (builtins == NULL) { + Py_DECREF(defaults); return NULL; } - return current_frame->f_globals; + if (locals == NULL) { + locals = globals; + } + PyObject *kwnames = NULL; + PyObject *const *allargs; + PyObject **newargs = NULL; + PyFunctionObject *func = NULL; + if (kwcount == 0) { + allargs = args; + } + else { + kwnames = PyTuple_New(kwcount); + if (kwnames == NULL) { + goto fail; + } + newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount)); + if (newargs == NULL) { + goto fail; + } + for (int i = 0; i < argcount; i++) { + newargs[i] = args[i]; + } + for (int i = 0; i < kwcount; i++) { + PyTuple_SET_ITEM(kwnames, i, Py_NewRef(kws[2*i])); + newargs[argcount+i] = kws[2*i+1]; + } + allargs = newargs; + } + PyFrameConstructor constr = { + .fc_globals = globals, + .fc_builtins = builtins, + .fc_name = ((PyCodeObject *)_co)->co_name, + .fc_qualname = ((PyCodeObject *)_co)->co_name, + .fc_code = _co, + .fc_defaults = defaults, + .fc_kwdefaults = kwdefs, + .fc_closure = closure + }; + func = _PyFunction_FromConstructor(&constr); + if (func == NULL) { + goto fail; + } + EVAL_CALL_STAT_INC(EVAL_CALL_LEGACY); + res = _PyEval_Vector(tstate, func, locals, + allargs, argcount, + kwnames); +fail: + Py_XDECREF(func); + Py_XDECREF(kwnames); + PyMem_Free(newargs); + Py_DECREF(defaults); + return res; } -int -PyEval_MergeCompilerFlags(PyCompilerFlags *cf) + +/* Logic for the raise statement (too complicated for inlining). + This *consumes* a reference count to each of its arguments. */ +static int +do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) { - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = tstate->current_frame; - int result = cf->cf_flags != 0; + PyObject *type = NULL, *value = NULL; - if (current_frame != NULL) { - const int codeflags = _PyFrame_GetCode(current_frame)->co_flags; - const int compilerflags = codeflags & PyCF_MASK; - if (compilerflags) { - result = 1; - cf->cf_flags |= compilerflags; + if (exc == NULL) { + /* Reraise */ + _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); + exc = exc_info->exc_value; + if (Py_IsNone(exc) || exc == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "No active exception to reraise"); + return 0; } + Py_INCREF(exc); + assert(PyExceptionInstance_Check(exc)); + _PyErr_SetRaisedException(tstate, exc); + return 1; } - return result; -} + /* We support the following forms of raise: + raise + raise + raise */ -const char * -PyEval_GetFuncName(PyObject *func) -{ - if (PyMethod_Check(func)) - return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func)); - else if (PyFunction_Check(func)) - return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name); - else if (PyCFunction_Check(func)) - return ((PyCFunctionObject*)func)->m_ml->ml_name; - else - return Py_TYPE(func)->tp_name; -} + if (PyExceptionClass_Check(exc)) { + type = exc; + value = _PyObject_CallNoArgs(exc); + if (value == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(value)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto raise_error; + } + } + else if (PyExceptionInstance_Check(exc)) { + value = exc; + type = PyExceptionInstance_Class(exc); + Py_INCREF(type); + } + else { + /* Not something you can raise. You get an exception + anyway, just not what you specified :-) */ + Py_DECREF(exc); + _PyErr_SetString(tstate, PyExc_TypeError, + "exceptions must derive from BaseException"); + goto raise_error; + } -const char * -PyEval_GetFuncDesc(PyObject *func) -{ - if (PyMethod_Check(func)) - return "()"; - else if (PyFunction_Check(func)) - return "()"; - else if (PyCFunction_Check(func)) - return "()"; - else - return " object"; -} + assert(type != NULL); + assert(value != NULL); -/* Extract a slice index from a PyLong or an object with the - nb_index slot defined, and store in *pi. - Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, - and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN. - Return 0 on error, 1 on success. -*/ -int -_PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (!Py_IsNone(v)) { - Py_ssize_t x; - if (_PyIndex_Check(v)) { - x = PyNumber_AsSsize_t(v, NULL); - if (x == -1 && _PyErr_Occurred(tstate)) - return 0; + if (cause) { + PyObject *fixed_cause; + if (PyExceptionClass_Check(cause)) { + fixed_cause = _PyObject_CallNoArgs(cause); + if (fixed_cause == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + goto raise_error; + } + Py_DECREF(cause); + } + else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + } + else if (Py_IsNone(cause)) { + Py_DECREF(cause); + fixed_cause = NULL; } else { _PyErr_SetString(tstate, PyExc_TypeError, - "slice indices must be integers or " - "None or have an __index__ method"); - return 0; + "exception causes must derive from " + "BaseException"); + goto raise_error; } - *pi = x; + PyException_SetCause(value, fixed_cause); } - return 1; + + _PyErr_SetObject(tstate, type, value); + /* _PyErr_SetObject incref's its arguments */ + Py_DECREF(value); + Py_DECREF(type); + return 0; + +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(cause); + return 0; } +/* Logic for matching an exception in an except* clause (too + complicated for inlining). +*/ + int -_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) +_PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, + PyObject **match, PyObject **rest) { - PyThreadState *tstate = _PyThreadState_GET(); - Py_ssize_t x; - if (_PyIndex_Check(v)) { - x = PyNumber_AsSsize_t(v, NULL); - if (x == -1 && _PyErr_Occurred(tstate)) - return 0; - } - else { - _PyErr_SetString(tstate, PyExc_TypeError, - "slice indices must be integers or " - "have an __index__ method"); + if (Py_IsNone(exc_value)) { + *match = Py_NewRef(Py_None); + *rest = Py_NewRef(Py_None); return 0; } - *pi = x; - return 1; -} - -static PyObject * -import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, - PyObject *name, PyObject *fromlist, PyObject *level) -{ - PyObject *import_func; - if (PyMapping_GetOptionalItem(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { - return NULL; - } - if (import_func == NULL) { - _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found"); - return NULL; - } + assert(PyExceptionInstance_Check(exc_value)); - PyObject *locals = frame->f_locals; - if (locals == NULL) { - locals = Py_None; + if (PyErr_GivenExceptionMatches(exc_value, match_type)) { + /* Full match of exc itself */ + bool is_eg = _PyBaseExceptionGroup_Check(exc_value); + if (is_eg) { + *match = Py_NewRef(exc_value); + } + else { + /* naked exception - wrap it */ + PyObject *excs = PyTuple_Pack(1, exc_value); + if (excs == NULL) { + return -1; + } + PyObject *wrapped = _PyExc_CreateExceptionGroup("", excs); + Py_DECREF(excs); + if (wrapped == NULL) { + return -1; + } + *match = wrapped; + } + *rest = Py_NewRef(Py_None); + return 0; } - /* Fast path for not overloaded __import__. */ - if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { - Py_DECREF(import_func); - int ilevel = PyLong_AsInt(level); - if (ilevel == -1 && _PyErr_Occurred(tstate)) { - return NULL; + /* exc_value does not match match_type. + * Check for partial match if it's an exception group. + */ + if (_PyBaseExceptionGroup_Check(exc_value)) { + PyObject *pair = PyObject_CallMethod(exc_value, "split", "(O)", + match_type); + if (pair == NULL) { + return -1; } - return PyImport_ImportModuleLevelObject( - name, - frame->f_globals, - locals, - fromlist, - ilevel); + assert(PyTuple_CheckExact(pair)); + assert(PyTuple_GET_SIZE(pair) == 2); + *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0)); + *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1)); + Py_DECREF(pair); + return 0; } - - PyObject* args[5] = {name, frame->f_globals, locals, fromlist, level}; - PyObject *res = PyObject_Vectorcall(import_func, args, 5, NULL); - Py_DECREF(import_func); - return res; + /* no match */ + *match = Py_NewRef(Py_None); + *rest = Py_NewRef(exc_value); + return 0; } -static PyObject * -import_from(PyThreadState *tstate, PyObject *v, PyObject *name) +/* Iterate v argcnt times and store the results on the stack (via decreasing + sp). Return 1 for success, 0 if error. + + If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack + with a variable target. +*/ + +int +_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, + int argcnt, int argcntafter, PyObject **sp) { - PyObject *x; - PyObject *fullmodname, *pkgname, *pkgpath, *pkgname_or_unknown, *errmsg; + int i = 0, j = 0; + Py_ssize_t ll = 0; + PyObject *it; /* iter(v) */ + PyObject *w; + PyObject *l = NULL; /* variable list */ - if (PyObject_GetOptionalAttr(v, name, &x) != 0) { - return x; - } - /* Issue #17636: in case this failed because of a circular relative - import, try to fallback on reading the module directly from - sys.modules. */ - if (PyObject_GetOptionalAttr(v, &_Py_ID(__name__), &pkgname) < 0) { - return NULL; - } - if (pkgname == NULL || !PyUnicode_Check(pkgname)) { - Py_CLEAR(pkgname); - goto error; - } - fullmodname = PyUnicode_FromFormat("%U.%U", pkgname, name); - if (fullmodname == NULL) { - Py_DECREF(pkgname); - return NULL; - } - x = PyImport_GetModule(fullmodname); - Py_DECREF(fullmodname); - if (x == NULL && !_PyErr_Occurred(tstate)) { - goto error; - } - Py_DECREF(pkgname); - return x; - error: - if (pkgname == NULL) { - pkgname_or_unknown = PyUnicode_FromString(""); - if (pkgname_or_unknown == NULL) { - return NULL; + assert(v != NULL); + + it = PyObject_GetIter(v); + if (it == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + Py_TYPE(v)->tp_iter == NULL && !PySequence_Check(v)) + { + _PyErr_Format(tstate, PyExc_TypeError, + "cannot unpack non-iterable %.200s object", + Py_TYPE(v)->tp_name); } - } else { - pkgname_or_unknown = pkgname; + return 0; } - pkgpath = NULL; - if (PyModule_Check(v)) { - pkgpath = PyModule_GetFilenameObject(v); - if (pkgpath == NULL) { - if (!PyErr_ExceptionMatches(PyExc_SystemError)) { - Py_DECREF(pkgname_or_unknown); - return NULL; + for (; i < argcnt; i++) { + w = PyIter_Next(it); + if (w == NULL) { + /* Iterator done, via error or exhaustion. */ + if (!_PyErr_Occurred(tstate)) { + if (argcntafter == -1) { + _PyErr_Format(tstate, PyExc_ValueError, + "not enough values to unpack " + "(expected %d, got %d)", + argcnt, i); + } + else { + _PyErr_Format(tstate, PyExc_ValueError, + "not enough values to unpack " + "(expected at least %d, got %d)", + argcnt + argcntafter, i); + } } - // module filename missing - _PyErr_Clear(tstate); + goto Error; } + *--sp = w; } - if (pkgpath == NULL || !PyUnicode_Check(pkgpath)) { - Py_CLEAR(pkgpath); - errmsg = PyUnicode_FromFormat( - "cannot import name %R from %R (unknown location)", - name, pkgname_or_unknown - ); - } - else { - PyObject *spec; - int rc = PyObject_GetOptionalAttr(v, &_Py_ID(__spec__), &spec); - if (rc > 0) { - rc = _PyModuleSpec_IsInitializing(spec); - Py_DECREF(spec); - } - if (rc < 0) { - Py_DECREF(pkgname_or_unknown); - Py_DECREF(pkgpath); - return NULL; + + if (argcntafter == -1) { + /* We better have exhausted the iterator now. */ + w = PyIter_Next(it); + if (w == NULL) { + if (_PyErr_Occurred(tstate)) + goto Error; + Py_DECREF(it); + return 1; } - const char *fmt = - rc ? - "cannot import name %R from partially initialized module %R " - "(most likely due to a circular import) (%S)" : - "cannot import name %R from %R (%S)"; + Py_DECREF(w); + _PyErr_Format(tstate, PyExc_ValueError, + "too many values to unpack (expected %d)", + argcnt); + goto Error; + } - errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath); + l = PySequence_List(it); + if (l == NULL) + goto Error; + *--sp = l; + i++; + + ll = PyList_GET_SIZE(l); + if (ll < argcntafter) { + _PyErr_Format(tstate, PyExc_ValueError, + "not enough values to unpack (expected at least %d, got %zd)", + argcnt + argcntafter, argcnt + ll); + goto Error; + } + + /* Pop the "after-variable" args off the list. */ + for (j = argcntafter; j > 0; j--, i++) { + *--sp = PyList_GET_ITEM(l, ll - j); } - /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ - _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, pkgpath, name); + /* Resize the list. */ + Py_SET_SIZE(l, ll - argcntafter); + Py_DECREF(it); + return 1; - Py_XDECREF(errmsg); - Py_DECREF(pkgname_or_unknown); - Py_XDECREF(pkgpath); - return NULL; +Error: + for (; i > 0; i--, sp++) + Py_DECREF(*sp); + Py_XDECREF(it); + return 0; } -#define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ - "BaseException is not allowed" +static int +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) +{ + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + return 0; + } + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_DECREF(exc); + } + return err; +} -#define CANNOT_EXCEPT_STAR_EG "catching ExceptionGroup with except* "\ - "is not allowed. Use except instead." +static inline bool +no_tools_for_global_event(PyThreadState *tstate, int event) +{ + return tstate->interp->monitors.tools[event] == 0; +} -int -_PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right) +static inline bool +no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - if (PyTuple_Check(right)) { - Py_ssize_t i, length; - length = PyTuple_GET_SIZE(right); - for (i = 0; i < length; i++) { - PyObject *exc = PyTuple_GET_ITEM(right, i); - if (!PyExceptionClass_Check(exc)) { - _PyErr_SetString(tstate, PyExc_TypeError, - CANNOT_CATCH_MSG); - return -1; - } - } + assert(event < _PY_MONITORING_LOCAL_EVENTS); + _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; + if (data) { + return data->active_monitors.tools[event] == 0; } else { - if (!PyExceptionClass_Check(right)) { - _PyErr_SetString(tstate, PyExc_TypeError, - CANNOT_CATCH_MSG); - return -1; - } + return no_tools_for_global_event(tstate, event); } - return 0; } -int -_PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right) +static void +monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - return -1; + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { + return; } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); +} - /* reject except *ExceptionGroup */ - - int is_subclass = 0; - if (PyTuple_Check(right)) { - Py_ssize_t length = PyTuple_GET_SIZE(right); - for (Py_ssize_t i = 0; i < length; i++) { - PyObject *exc = PyTuple_GET_ITEM(right, i); - is_subclass = PyObject_IsSubclass(exc, PyExc_BaseExceptionGroup); - if (is_subclass < 0) { - return -1; - } - if (is_subclass) { - break; - } - } +static void +monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { + return; } - else { - is_subclass = PyObject_IsSubclass(right, PyExc_BaseExceptionGroup); - if (is_subclass < 0) { - return -1; - } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); +} + +static int +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + return 0; } - if (is_subclass) { - _PyErr_SetString(tstate, PyExc_TypeError, - CANNOT_EXCEPT_STAR_EG); - return -1; + return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); +} + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { + return; } - return 0; + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); } + static int -check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args) +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) { - if (Py_TYPE(args)->tp_iter == NULL && !PySequence_Check(args)) { - /* check_args_iterable() may be called with a live exception: - * clear it to prevent calling _PyObject_FunctionStr() with an - * exception set. */ - _PyErr_Clear(tstate); - PyObject *funcstr = _PyObject_FunctionStr(func); - if (funcstr != NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U argument after * must be an iterable, not %.200s", - funcstr, Py_TYPE(args)->tp_name); - Py_DECREF(funcstr); - } - return -1; + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + return 0; } - return 0; + return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); +} + +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); } void -_PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs) +PyThreadState_EnterTracing(PyThreadState *tstate) { - /* _PyDict_MergeEx raises attribute - * error (percolated from an attempt - * to get 'keys' attribute) instead of - * a type error if its second argument - * is not a mapping. - */ - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Clear(tstate); - PyObject *funcstr = _PyObject_FunctionStr(func); - if (funcstr != NULL) { - _PyErr_Format( - tstate, PyExc_TypeError, - "%U argument after ** must be a mapping, not %.200s", - funcstr, Py_TYPE(kwargs)->tp_name); - Py_DECREF(funcstr); - } + assert(tstate->tracing >= 0); + tstate->tracing++; +} + +void +PyThreadState_LeaveTracing(PyThreadState *tstate) +{ + assert(tstate->tracing > 0); + tstate->tracing--; +} + + +PyObject* +_PyEval_CallTracing(PyObject *func, PyObject *args) +{ + // Save and disable tracing + PyThreadState *tstate = _PyThreadState_GET(); + int save_tracing = tstate->tracing; + tstate->tracing = 0; + + // Call the tracing function + PyObject *result = PyObject_Call(func, args, NULL); + + // Restore tracing + tstate->tracing = save_tracing; + return result; +} + +void +PyEval_SetProfile(Py_tracefunc func, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyEval_SetProfile(tstate, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfile"); } - else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *args = ((PyBaseExceptionObject *)exc)->args; - if (exc && PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1) { - _PyErr_Clear(tstate); - PyObject *funcstr = _PyObject_FunctionStr(func); - if (funcstr != NULL) { - PyObject *key = PyTuple_GET_ITEM(args, 0); - _PyErr_Format( - tstate, PyExc_TypeError, - "%U got multiple values for keyword argument '%S'", - funcstr, key); - Py_DECREF(funcstr); - } - Py_XDECREF(exc); - } - else { - _PyErr_SetRaisedException(tstate, exc); +} + +void +PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) +{ + PyThreadState *this_tstate = _PyThreadState_GET(); + PyInterpreterState* interp = this_tstate->interp; + + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + + while (ts) { + if (_PyEval_SetProfile(ts, func, arg) < 0) { + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); } + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); } } void -_PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, - const char *format_str, PyObject *obj) +PyEval_SetTrace(Py_tracefunc func, PyObject *arg) { - const char *obj_str; - - if (!obj) - return; + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyEval_SetTrace(tstate, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTrace"); + } +} - obj_str = PyUnicode_AsUTF8(obj); - if (!obj_str) - return; +void +PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) +{ + PyThreadState *this_tstate = _PyThreadState_GET(); + PyInterpreterState* interp = this_tstate->interp; - _PyErr_Format(tstate, exc, format_str, obj_str); + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); - if (exc == PyExc_NameError) { - // Include the name in the NameError exceptions to offer suggestions later. - PyObject *exc = PyErr_GetRaisedException(); - if (PyErr_GivenExceptionMatches(exc, PyExc_NameError)) { - if (((PyNameErrorObject*)exc)->name == NULL) { - // We do not care if this fails because we are going to restore the - // NameError anyway. - (void)PyObject_SetAttr(exc, &_Py_ID(name), obj); - } + while (ts) { + if (_PyEval_SetTrace(ts, func, arg) < 0) { + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); } - PyErr_SetRaisedException(exc); + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); } } -void -_PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg) +int +_PyEval_SetCoroutineOriginTrackingDepth(int depth) { - PyObject *name; - /* Don't stomp existing exception */ - if (_PyErr_Occurred(tstate)) - return; - name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); - if (oparg < PyUnstable_Code_GetFirstFree(co)) { - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, name); - } else { - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - UNBOUNDFREE_ERROR_MSG, name); + PyThreadState *tstate = _PyThreadState_GET(); + if (depth < 0) { + _PyErr_SetString(tstate, PyExc_ValueError, "depth must be >= 0"); + return -1; } + tstate->coroutine_origin_tracking_depth = depth; + return 0; } -void -_PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg) + +int +_PyEval_GetCoroutineOriginTrackingDepth(void) { - if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) { - if (oparg == 1) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async with' received an object from __aenter__ " - "that does not implement __await__: %.100s", - type->tp_name); - } - else if (oparg == 2) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async with' received an object from __aexit__ " - "that does not implement __await__: %.100s", - type->tp_name); - } - } + PyThreadState *tstate = _PyThreadState_GET(); + return tstate->coroutine_origin_tracking_depth; } -Py_ssize_t -PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) +int +_PyEval_SetAsyncGenFirstiter(PyObject *firstiter) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - Py_ssize_t new_index; + PyThreadState *tstate = _PyThreadState_GET(); - if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { + if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_firstiter", NULL) < 0) { return -1; } - new_index = interp->co_extra_user_count++; - interp->co_extra_freefuncs[new_index] = free; - return new_index; -} -/* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions - for the limited API. */ + Py_XSETREF(tstate->async_gen_firstiter, Py_XNewRef(firstiter)); + return 0; +} -int Py_EnterRecursiveCall(const char *where) +PyObject * +_PyEval_GetAsyncGenFirstiter(void) { - return _Py_EnterRecursiveCall(where); + PyThreadState *tstate = _PyThreadState_GET(); + return tstate->async_gen_firstiter; } -void Py_LeaveRecursiveCall(void) +int +_PyEval_SetAsyncGenFinalizer(PyObject *finalizer) { - _Py_LeaveRecursiveCall(); -} + PyThreadState *tstate = _PyThreadState_GET(); + if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_finalizer", NULL) < 0) { + return -1; + } -/* Interpreter main loop */ + Py_XSETREF(tstate->async_gen_finalizer, Py_XNewRef(finalizer)); + return 0; +} PyObject * -PyEval_EvalFrame(PyFrameObject *f) +_PyEval_GetAsyncGenFinalizer(void) { - /* Function kept for backward compatibility */ PyThreadState *tstate = _PyThreadState_GET(); - return _PyEval_EvalFrame(tstate, f->f_frame, 0); + return tstate->async_gen_finalizer; } -PyObject * -PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) +_PyInterpreterFrame * +_PyEval_GetFrame(void) { PyThreadState *tstate = _PyThreadState_GET(); - return _PyEval_EvalFrame(tstate, f->f_frame, throwflag); + return _PyThreadState_GetFrame(tstate); } -int _Py_CheckRecursiveCallPy( - PyThreadState *tstate) +PyFrameObject * +PyEval_GetFrame(void) { - if (tstate->recursion_headroom) { - if (tstate->py_recursion_remaining < -50) { - /* Overflowing while handling an overflow. Give up. */ - Py_FatalError("Cannot recover from Python stack overflow."); - } + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + if (frame == NULL) { + return NULL; } - else { - if (tstate->py_recursion_remaining <= 0) { - tstate->recursion_headroom++; - _PyErr_Format(tstate, PyExc_RecursionError, - "maximum recursion depth exceeded"); - tstate->recursion_headroom--; - return -1; - } + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f == NULL) { + PyErr_Clear(); } - return 0; + return f; } -static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { - /* Put a NOP at the start, so that the IP points into - * the code, rather than before it */ - { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ - { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ - { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } -}; - -extern const struct _PyCode_DEF(8) _Py_InitCleanup; - -#ifdef Py_DEBUG -extern void _PyUOpPrint(const _PyUOpInstruction *uop); -#endif - -/* Disable unused label warnings. They are handy for debugging, even - if computed gotos aren't used. */ - -/* TBD - what about other compilers? */ -#if defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-label" -#elif defined(_MSC_VER) /* MS_WINDOWS */ -# pragma warning(push) -# pragma warning(disable:4102) -#endif - - -/* _PyEval_EvalFrameDefault() is a *big* function, - * so consume 3 units of C stack */ -#define PY_EVAL_C_STACK_UNITS 2 +PyObject * +_PyEval_GetBuiltins(PyThreadState *tstate) +{ + _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); + if (frame != NULL) { + return frame->f_builtins; + } + return tstate->interp->builtins; +} -#if defined(_MSC_VER) && defined(_Py_USING_PGO) -/* gh-111786: _PyEval_EvalFrameDefault is too large to optimize for speed with - PGO on MSVC. Disable that optimization temporarily. If this is fixed - upstream, we should gate this on the version of MSVC. - */ -# pragma optimize("t", off) -/* This setting is reversed below following _PyEval_EvalFrameDefault */ -#endif +PyObject * +PyEval_GetBuiltins(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_GetBuiltins(tstate); +} -PyObject* _Py_HOT_FUNCTION -_PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) +/* Convenience function to get a builtin from its name */ +PyObject * +_PyEval_GetBuiltin(PyObject *name) { - _Py_EnsureTstateNotNULL(tstate); - CALL_STAT_INC(pyeval_calls); + PyObject *attr; + if (PyMapping_GetOptionalItem(PyEval_GetBuiltins(), name, &attr) == 0) { + PyErr_SetObject(PyExc_AttributeError, name); + } + return attr; +} -#if USE_COMPUTED_GOTOS -/* Import the static jump table */ -#include "opcode_targets.h" -#endif +PyObject * +_PyEval_GetBuiltinId(_Py_Identifier *name) +{ + return _PyEval_GetBuiltin(_PyUnicode_FromId(name)); +} -#ifdef Py_STATS - int lastopcode = 0; -#endif - uint8_t opcode; /* Current opcode */ - int oparg; /* Current opcode argument, if any */ -#ifdef LLTRACE - int lltrace = 0; -#endif +PyObject * +PyEval_GetLocals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } - _PyInterpreterFrame entry_frame; + if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { + return NULL; + } + PyObject *locals = current_frame->f_locals; + assert(locals != NULL); + return locals; +} +PyObject * +_PyEval_GetFrameLocals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } -#ifdef Py_DEBUG - /* Set these to invalid but identifiable values for debugging. */ - entry_frame.f_funcobj = (PyObject*)0xaaa0; - entry_frame.f_locals = (PyObject*)0xaaa1; - entry_frame.frame_obj = (PyFrameObject*)0xaaa2; - entry_frame.f_globals = (PyObject*)0xaaa3; - entry_frame.f_builtins = (PyObject*)0xaaa4; -#endif - entry_frame.f_executable = Py_None; - entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - entry_frame.stacktop = 0; - entry_frame.owner = FRAME_OWNED_BY_CSTACK; - entry_frame.return_offset = 0; - /* Push frame */ - entry_frame.previous = tstate->current_frame; - frame->previous = &entry_frame; - tstate->current_frame = frame; + return _PyFrame_GetLocals(current_frame, 1); +} - tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); - if (_Py_EnterRecursiveCallTstate(tstate, "")) { - tstate->c_recursion_remaining--; - tstate->py_recursion_remaining--; - goto exit_unwind; +PyObject * +PyEval_GetGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + return NULL; } + return current_frame->f_globals; +} - /* support for generator.throw() */ - if (throwflag) { - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; +int +PyEval_MergeCompilerFlags(PyCompilerFlags *cf) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = tstate->current_frame; + int result = cf->cf_flags != 0; + + if (current_frame != NULL) { + const int codeflags = _PyFrame_GetCode(current_frame)->co_flags; + const int compilerflags = codeflags & PyCF_MASK; + if (compilerflags) { + result = 1; + cf->cf_flags |= compilerflags; } - /* Because this avoids the RESUME, - * we need to update instrumentation */ - _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - monitor_throw(tstate, frame, frame->instr_ptr); - /* TO DO -- Monitor throw entry. */ - goto resume_with_error; } + return result; +} - /* Local "register" variables. - * These are cached values from the frame and code object. */ - _Py_CODEUNIT *next_instr; - PyObject **stack_pointer; -#ifndef _Py_JIT - /* Tier 2 interpreter state */ - _PyExecutorObject *current_executor = NULL; - const _PyUOpInstruction *next_uop = NULL; -#endif +const char * +PyEval_GetFuncName(PyObject *func) +{ + if (PyMethod_Check(func)) + return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func)); + else if (PyFunction_Check(func)) + return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name); + else if (PyCFunction_Check(func)) + return ((PyCFunctionObject*)func)->m_ml->ml_name; + else + return Py_TYPE(func)->tp_name; +} -start_frame: - if (_Py_EnterRecursivePy(tstate)) { - goto exit_unwind; +const char * +PyEval_GetFuncDesc(PyObject *func) +{ + if (PyMethod_Check(func)) + return "()"; + else if (PyFunction_Check(func)) + return "()"; + else if (PyCFunction_Check(func)) + return "()"; + else + return " object"; +} + +/* Extract a slice index from a PyLong or an object with the + nb_index slot defined, and store in *pi. + Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, + and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN. + Return 0 on error, 1 on success. +*/ +int +_PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (!Py_IsNone(v)) { + Py_ssize_t x; + if (_PyIndex_Check(v)) { + x = PyNumber_AsSsize_t(v, NULL); + if (x == -1 && _PyErr_Occurred(tstate)) + return 0; + } + else { + _PyErr_SetString(tstate, PyExc_TypeError, + "slice indices must be integers or " + "None or have an __index__ method"); + return 0; + } + *pi = x; } + return 1; +} - next_instr = frame->instr_ptr; -resume_frame: - stack_pointer = _PyFrame_GetStackPointer(frame); +int +_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) +{ + PyThreadState *tstate = _PyThreadState_GET(); + Py_ssize_t x; + if (_PyIndex_Check(v)) { + x = PyNumber_AsSsize_t(v, NULL); + if (x == -1 && _PyErr_Occurred(tstate)) + return 0; + } + else { + _PyErr_SetString(tstate, PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + return 0; + } + *pi = x; + return 1; +} -#ifdef LLTRACE - lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); - if (lltrace < 0) { - goto exit_unwind; +static PyObject * +import_name(PyThreadState *tstate, _PyInterpreterFrame *frame, + PyObject *name, PyObject *fromlist, PyObject *level) +{ + PyObject *import_func; + if (PyMapping_GetOptionalItem(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { + return NULL; + } + if (import_func == NULL) { + _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found"); + return NULL; } -#endif -#ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); -#endif + PyObject *locals = frame->f_locals; + if (locals == NULL) { + locals = Py_None; + } - DISPATCH(); + /* Fast path for not overloaded __import__. */ + if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) { + Py_DECREF(import_func); + int ilevel = PyLong_AsInt(level); + if (ilevel == -1 && _PyErr_Occurred(tstate)) { + return NULL; + } + return PyImport_ImportModuleLevelObject( + name, + frame->f_globals, + locals, + fromlist, + ilevel); + } - { - /* Start instructions */ -#if !USE_COMPUTED_GOTOS - dispatch_opcode: - switch (opcode) -#endif - { + PyObject* args[5] = {name, frame->f_globals, locals, fromlist, level}; + PyObject *res = PyObject_Vectorcall(import_func, args, 5, NULL); + Py_DECREF(import_func); + return res; +} -#include "generated_cases.c.h" +static PyObject * +import_from(PyThreadState *tstate, PyObject *v, PyObject *name) +{ + PyObject *x; + PyObject *fullmodname, *pkgname, *pkgpath, *pkgname_or_unknown, *errmsg; - /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, - * because it needs to capture frame->instr_ptr before it is updated, - * as happens in the standard instruction prologue. - */ -#if USE_COMPUTED_GOTOS - TARGET_INSTRUMENTED_LINE: -#else - case INSTRUMENTED_LINE: -#endif - { - _Py_CODEUNIT *prev = frame->instr_ptr; - _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - /* Prevent the underlying instruction from specializing - * and overwriting the instrumentation. */ - PAUSE_ADAPTIVE_COUNTER(cache->counter); + if (PyObject_GetOptionalAttr(v, name, &x) != 0) { + return x; + } + /* Issue #17636: in case this failed because of a circular relative + import, try to fallback on reading the module directly from + sys.modules. */ + if (PyObject_GetOptionalAttr(v, &_Py_ID(__name__), &pkgname) < 0) { + return NULL; + } + if (pkgname == NULL || !PyUnicode_Check(pkgname)) { + Py_CLEAR(pkgname); + goto error; + } + fullmodname = PyUnicode_FromFormat("%U.%U", pkgname, name); + if (fullmodname == NULL) { + Py_DECREF(pkgname); + return NULL; + } + x = PyImport_GetModule(fullmodname); + Py_DECREF(fullmodname); + if (x == NULL && !_PyErr_Occurred(tstate)) { + goto error; + } + Py_DECREF(pkgname); + return x; + error: + if (pkgname == NULL) { + pkgname_or_unknown = PyUnicode_FromString(""); + if (pkgname_or_unknown == NULL) { + return NULL; } - opcode = original_opcode; - DISPATCH_GOTO(); + } else { + pkgname_or_unknown = pkgname; } + pkgpath = NULL; + if (PyModule_Check(v)) { + pkgpath = PyModule_GetFilenameObject(v); + if (pkgpath == NULL) { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + Py_DECREF(pkgname_or_unknown); + return NULL; + } + // module filename missing + _PyErr_Clear(tstate); + } + } + if (pkgpath == NULL || !PyUnicode_Check(pkgpath)) { + Py_CLEAR(pkgpath); + errmsg = PyUnicode_FromFormat( + "cannot import name %R from %R (unknown location)", + name, pkgname_or_unknown + ); + } + else { + PyObject *spec; + int rc = PyObject_GetOptionalAttr(v, &_Py_ID(__spec__), &spec); + if (rc > 0) { + rc = _PyModuleSpec_IsInitializing(spec); + Py_DECREF(spec); + } + if (rc < 0) { + Py_DECREF(pkgname_or_unknown); + Py_DECREF(pkgpath); + return NULL; + } + const char *fmt = + rc ? + "cannot import name %R from partially initialized module %R " + "(most likely due to a circular import) (%S)" : + "cannot import name %R from %R (%S)"; -#if USE_COMPUTED_GOTOS - _unknown_opcode: -#else - EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode -#endif - /* Tell C compilers not to hold the opcode variable in the loop. - next_instr points the current instruction without TARGET(). */ - opcode = next_instr->op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; + errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath); + } + /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */ + _PyErr_SetImportErrorWithNameFrom(errmsg, pkgname, pkgpath, name); - } /* End instructions */ + Py_XDECREF(errmsg); + Py_DECREF(pkgname_or_unknown); + Py_XDECREF(pkgpath); + return NULL; +} - /* This should never be reached. Every opcode should end with DISPATCH() - or goto error. */ - Py_UNREACHABLE(); +#define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ + "BaseException is not allowed" -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif +#define CANNOT_EXCEPT_STAR_EG "catching ExceptionGroup with except* "\ + "is not allowed. Use except instead." - /* Log traceback info. */ - assert(frame != &entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); +int +_PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right) +{ + if (PyTuple_Check(right)) { + Py_ssize_t i, length; + length = PyTuple_GET_SIZE(right); + for (i = 0; i < length; i++) { + PyObject *exc = PyTuple_GET_ITEM(right, i); + if (!PyExceptionClass_Check(exc)) { + _PyErr_SetString(tstate, PyExc_TypeError, + CANNOT_CATCH_MSG); + return -1; } } - monitor_raise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + } + else { + if (!PyExceptionClass_Check(right)) { + _PyErr_SetString(tstate, PyExc_TypeError, + CANNOT_CATCH_MSG); + return -1; + } + } + return 0; +} - /* Pop remaining stack entries. */ - PyObject **stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyObject *o = POP(); - Py_XDECREF(o); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } +int +_PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right) +{ + if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + return -1; + } - assert(STACK_LEVEL() >= level); - PyObject **new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyObject *v = POP(); - Py_XDECREF(v); + /* reject except *ExceptionGroup */ + + int is_subclass = 0; + if (PyTuple_Check(right)) { + Py_ssize_t length = PyTuple_GET_SIZE(right); + for (Py_ssize_t i = 0; i < length; i++) { + PyObject *exc = PyTuple_GET_ITEM(right, i); + is_subclass = PyObject_IsSubclass(exc, PyExc_BaseExceptionGroup); + if (is_subclass < 0) { + return -1; } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(lasti); + if (is_subclass) { + break; } + } + } + else { + is_subclass = PyObject_IsSubclass(right, PyExc_BaseExceptionGroup); + if (is_subclass < 0) { + return -1; + } + } + if (is_subclass) { + _PyErr_SetString(tstate, PyExc_TypeError, + CANNOT_EXCEPT_STAR_EG); + return -1; + } + return 0; +} - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(exc); - next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; +static int +check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args) +{ + if (Py_TYPE(args)->tp_iter == NULL && !PySequence_Check(args)) { + /* check_args_iterable() may be called with a live exception: + * clear it to prevent calling _PyObject_FunctionStr() with an + * exception set. */ + _PyErr_Clear(tstate); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U argument after * must be an iterable, not %.200s", + funcstr, Py_TYPE(args)->tp_name); + Py_DECREF(funcstr); + } + return -1; + } + return 0; +} - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); +void +_PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs) +{ + /* _PyDict_MergeEx raises attribute + * error (percolated from an attempt + * to get 'keys' attribute) instead of + * a type error if its second argument + * is not a mapping. + */ + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Clear(tstate); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + _PyErr_Format( + tstate, PyExc_TypeError, + "%U argument after ** must be a mapping, not %.200s", + funcstr, Py_TYPE(kwargs)->tp_name); + Py_DECREF(funcstr); + } + } + else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *args = ((PyBaseExceptionObject *)exc)->args; + if (exc && PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1) { + _PyErr_Clear(tstate); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + PyObject *key = PyTuple_GET_ITEM(args, 0); + _PyErr_Format( + tstate, PyExc_TypeError, + "%U got multiple values for keyword argument '%S'", + funcstr, key); + Py_DECREF(funcstr); } -#endif - DISPATCH(); + Py_XDECREF(exc); + } + else { + _PyErr_SetRaisedException(tstate, exc); } } +} -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == &entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - - - -// Tier 2 is also here! -enter_tier_two: - -#ifdef _Py_JIT - assert(0); -#else - -#undef LOAD_IP -#define LOAD_IP(UNUSED) (void)0 - -#undef GOTO_ERROR -#define GOTO_ERROR(LABEL) goto LABEL ## _tier_two - -#ifdef Py_STATS -// Disable these macros that apply to Tier 1 stats when we are in Tier 2 -#undef STAT_INC -#define STAT_INC(opname, name) ((void)0) -#undef STAT_DEC -#define STAT_DEC(opname, name) ((void)0) -#undef CALL_STAT_INC -#define CALL_STAT_INC(name) ((void)0) -#endif - -#undef ENABLE_SPECIALIZATION -#define ENABLE_SPECIALIZATION 0 - -#ifdef Py_DEBUG - #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { printf(__VA_ARGS__); } -#else - #define DPRINTF(level, ...) -#endif - - ; // dummy statement after a label, before a declaration - uint16_t uopcode; -#ifdef Py_STATS - int lastuop = 0; - uint64_t trace_uop_execution_counter = 0; -#endif +void +_PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, + const char *format_str, PyObject *obj) +{ + const char *obj_str; - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); -tier2_dispatch: - for (;;) { - uopcode = next_uop->opcode; -#ifdef Py_DEBUG - if (lltrace >= 3) { - if (next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT) { - printf("%4d uop: ", 0); - } - else { - printf("%4d uop: ", (int)(next_uop - current_executor->trace)); - } - _PyUOpPrint(next_uop); - printf(" stack_level=%d\n", - (int)(stack_pointer - _PyFrame_Stackbase(frame))); - } -#endif - next_uop++; - OPT_STAT_INC(uops_executed); - UOP_STAT_INC(uopcode, execution_count); - UOP_PAIR_INC(uopcode, lastuop); -#ifdef Py_STATS - trace_uop_execution_counter++; -#endif + if (!obj) + return; - switch (uopcode) { + obj_str = PyUnicode_AsUTF8(obj); + if (!obj_str) + return; -#include "executor_cases.c.h" + _PyErr_Format(tstate, exc, format_str, obj_str); - default: -#ifdef Py_DEBUG - { - printf("Unknown uop: "); - _PyUOpPrint(&next_uop[-1]); - printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); - Py_FatalError("Unknown uop"); + if (exc == PyExc_NameError) { + // Include the name in the NameError exceptions to offer suggestions later. + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_GivenExceptionMatches(exc, PyExc_NameError)) { + if (((PyNameErrorObject*)exc)->name == NULL) { + // We do not care if this fails because we are going to restore the + // NameError anyway. + (void)PyObject_SetAttr(exc, &_Py_ID(name), obj); } -#else - Py_UNREACHABLE(); -#endif - } + PyErr_SetRaisedException(exc); } +} -jump_to_error_target: -#ifdef Py_DEBUG - if (lltrace >= 2) { - printf("Error: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(" @ %d -> %s]\n", - (int)(next_uop - current_executor->trace - 1), - _PyOpcode_OpName[frame->instr_ptr->op.code]); +void +_PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg) +{ + PyObject *name; + /* Don't stomp existing exception */ + if (_PyErr_Occurred(tstate)) + return; + name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); + if (oparg < PyUnstable_Code_GetFirstFree(co)) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, name); + } else { + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + UNBOUNDFREE_ERROR_MSG, name); } -#endif - assert (next_uop[-1].format == UOP_FORMAT_JUMP); - uint16_t target = uop_get_error_target(&next_uop[-1]); - next_uop = current_executor->trace + target; - goto tier2_dispatch; +} -error_tier_two: - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - assert(next_uop[-1].format == UOP_FORMAT_TARGET); - frame->return_offset = 0; // Don't leave this random - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(current_executor); - tstate->previous_executor = NULL; - goto resume_with_error; +void +_PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg) +{ + if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) { + if (oparg == 1) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async with' received an object from __aenter__ " + "that does not implement __await__: %.100s", + type->tp_name); + } + else if (oparg == 2) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async with' received an object from __aexit__ " + "that does not implement __await__: %.100s", + type->tp_name); + } + } +} -jump_to_jump_target: - assert(next_uop[-1].format == UOP_FORMAT_JUMP); - target = uop_get_jump_target(&next_uop[-1]); - next_uop = current_executor->trace + target; - goto tier2_dispatch; -exit_to_tier1: - assert(next_uop[-1].format == UOP_FORMAT_TARGET); - next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); -#ifdef Py_DEBUG - if (lltrace >= 2) { - printf("DEOPT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(" -> %s]\n", - _PyOpcode_OpName[next_instr->op.code]); - } -#endif - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - Py_DECREF(current_executor); - tstate->previous_executor = NULL; - DISPATCH(); +Py_ssize_t +PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + Py_ssize_t new_index; -exit_to_trace: - assert(next_uop[-1].format == UOP_FORMAT_EXIT); - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - uint32_t exit_index = next_uop[-1].exit_index; - assert(exit_index < current_executor->exit_count); - _PyExitData *exit = ¤t_executor->exits[exit_index]; -#ifdef Py_DEBUG - if (lltrace >= 2) { - printf("SIDE EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", - exit_index, exit->temperature.as_counter, exit->target, - _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); + if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { + return -1; } -#endif - Py_INCREF(exit->executor); - tstate->previous_executor = (PyObject *)current_executor; - GOTO_TIER_TWO(exit->executor); + new_index = interp->co_extra_user_count++; + interp->co_extra_freefuncs[new_index] = free; + return new_index; +} -#endif // _Py_JIT +/* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions + for the limited API. */ +int Py_EnterRecursiveCall(const char *where) +{ + return _Py_EnterRecursiveCall(where); } -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#elif defined(_MSC_VER) /* MS_WINDOWS */ -# pragma warning(pop) -# pragma optimize("", on) -#endif - -/* There should be no code below this point. */ +void Py_LeaveRecursiveCall(void) +{ + _Py_LeaveRecursiveCall(); +} From 5e20f0f7fb8af37de0e1649d2848816cbb2c76dc Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 17 Apr 2024 10:42:20 -0700 Subject: [PATCH 5/5] Just don't redefine CALL_STAT_INC --- Python/ceval.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index c0783f7377a8ee..b88e555ded5c2e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -978,8 +978,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #define STAT_INC(opname, name) ((void)0) #undef STAT_DEC #define STAT_DEC(opname, name) ((void)0) -#undef CALL_STAT_INC -#define CALL_STAT_INC(name) ((void)0) #endif #undef ENABLE_SPECIALIZATION