From cb3fd406cf95c3aab22edd7baff3098f9fea7aa0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Nov 2022 14:09:01 +0000 Subject: [PATCH 01/10] Specialize calls to normal Python classes. --- Include/cpython/object.h | 1 + Include/internal/pycore_code.h | 2 + Include/internal/pycore_frame.h | 24 +++++++ Include/internal/pycore_object.h | 2 + Include/internal/pycore_opcode.h | 58 ++++++++-------- Include/opcode.h | 116 ++++++++++++++++--------------- Lib/opcode.py | 2 + Lib/test/test_sys.py | 2 +- Lib/test/test_sys_settrace.py | 23 ++++++ Objects/dictobject.c | 7 +- Objects/typeobject.c | 21 +++++- Python/bytecodes.c | 68 ++++++++++++++++++ Python/compile.c | 3 + Python/generated_cases.c.h | 69 ++++++++++++++++++ Python/opcode_targets.h | 54 +++++++------- Python/pystate.c | 1 + Python/specialize.c | 90 ++++++++++++++++++++++++ 17 files changed, 424 insertions(+), 119 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index f4755a7b2fb852..0d53f43acb5820 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,6 +230,7 @@ struct _typeobject { * by code other than the specializer and interpreter. */ struct _specialization_cache { PyObject *getitem; + PyObject *init; }; /* The *real* layout of a type object when allocated on the heap */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0af240ca362103..16d69b57976416 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -97,6 +97,8 @@ struct callable_cache { PyObject *len; PyObject *list_append; PyObject *object__getattribute__; + /* This is a strong reference */ + PyCodeObject *init_cleanup; }; /* "Locals plus" for a code object is the set of locals + cell vars + diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 7fa410d288c33a..550a35034709cb 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -223,6 +223,30 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func) return new_frame; } +/* Pushes a trampoline frame without checking for space. + * Must be guarded by _PyThreadState_HasStackSpace() */ +static inline _PyInterpreterFrame * +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int start) +{ + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; + tstate->datastack_top += code->co_framesize; + assert(tstate->datastack_top < tstate->datastack_limit); + frame->f_funcobj = Py_NewRef(Py_None); + frame->f_code = (PyCodeObject *)Py_NewRef(code); +#ifdef Py_DEBUG + frame->f_builtins = (PyObject*)0xaaaa; + frame->f_globals = (PyObject*)0xaaab; +#endif + frame->f_locals = NULL; + frame->stacktop = code->co_nlocalsplus + stackdepth; + frame->frame_obj = NULL; + frame->prev_instr = _PyCode_CODE(code) + start; + frame->owner = FRAME_OWNED_BY_THREAD; + frame->yield_offset = 0; + return frame; +} + int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame); static inline diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8b78f79e950e92..be21671b1cbb8c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -300,8 +300,10 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); +PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern int _PyObject_InitializeDict(PyObject *obj); +int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 3f44511240aa14..37ac566e3e8700 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -87,6 +87,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, [CALL_NO_KW_ISINSTANCE] = CALL, @@ -121,6 +122,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, + [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, @@ -249,6 +251,7 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", @@ -257,31 +260,30 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -289,37 +291,37 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", - [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC", - [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", + [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC", [STOPITERATION_ERROR] = "STOPITERATION_ERROR", + [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", + [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -346,7 +348,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -354,7 +356,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -374,9 +376,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -386,29 +388,29 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [174] = "<174>", - [175] = "<175>", [176] = "<176>", [177] = "<177>", [178] = "<178>", @@ -500,8 +502,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 174: \ - case 175: \ case 176: \ case 177: \ case 178: \ diff --git a/Include/opcode.h b/Include/opcode.h index c18d9c057b849b..d33d1c9f6475ed 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -18,6 +18,7 @@ extern "C" { #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define EXIT_INIT_CHECK 16 #define BINARY_SUBSCR 25 #define BINARY_SLICE 26 #define STORE_SLICE 27 @@ -135,63 +136,64 @@ extern "C" { #define BINARY_OP_GENERIC 8 #define BINARY_OP_INPLACE_ADD_UNICODE 13 #define BINARY_OP_MULTIPLY_FLOAT 14 -#define BINARY_OP_MULTIPLY_INT 16 -#define BINARY_OP_SUBTRACT_FLOAT 17 -#define BINARY_OP_SUBTRACT_INT 18 -#define BINARY_SUBSCR_DICT 19 -#define BINARY_SUBSCR_GETITEM 20 -#define BINARY_SUBSCR_LIST_INT 21 -#define BINARY_SUBSCR_TUPLE_INT 22 -#define CALL_PY_EXACT_ARGS 23 -#define CALL_PY_WITH_DEFAULTS 24 -#define CALL_BOUND_METHOD_EXACT_ARGS 28 -#define CALL_BUILTIN_CLASS 29 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 -#define CALL_NO_KW_BUILTIN_FAST 39 -#define CALL_NO_KW_BUILTIN_O 40 -#define CALL_NO_KW_ISINSTANCE 41 -#define CALL_NO_KW_LEN 42 -#define CALL_NO_KW_LIST_APPEND 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 -#define CALL_NO_KW_STR_1 47 -#define CALL_NO_KW_TUPLE_1 48 -#define CALL_NO_KW_TYPE_1 56 -#define COMPARE_OP_FLOAT_JUMP 57 -#define COMPARE_OP_GENERIC 58 -#define COMPARE_OP_INT_JUMP 59 -#define COMPARE_OP_STR_JUMP 62 -#define FOR_ITER_LIST 64 -#define FOR_ITER_RANGE 65 -#define FOR_ITER_GEN 66 -#define LOAD_ATTR_CLASS 67 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 72 -#define LOAD_ATTR_INSTANCE_VALUE 73 -#define LOAD_ATTR_MODULE 76 -#define LOAD_ATTR_PROPERTY 77 -#define LOAD_ATTR_SLOT 78 -#define LOAD_ATTR_WITH_HINT 79 -#define LOAD_ATTR_METHOD_LAZY_DICT 80 -#define LOAD_ATTR_METHOD_NO_DICT 81 -#define LOAD_ATTR_METHOD_WITH_DICT 86 -#define LOAD_ATTR_METHOD_WITH_VALUES 113 -#define LOAD_CONST__LOAD_FAST 121 -#define LOAD_FAST__LOAD_CONST 141 -#define LOAD_FAST__LOAD_FAST 143 -#define LOAD_GLOBAL_BUILTIN 153 -#define LOAD_GLOBAL_MODULE 154 -#define STORE_ATTR_INSTANCE_VALUE 158 -#define STORE_ATTR_SLOT 159 -#define STORE_ATTR_WITH_HINT 160 -#define STORE_FAST__LOAD_FAST 161 -#define STORE_FAST__STORE_FAST 166 -#define STORE_SUBSCR_DICT 167 -#define STORE_SUBSCR_LIST_INT 168 -#define UNPACK_SEQUENCE_LIST 169 -#define UNPACK_SEQUENCE_TUPLE 170 -#define UNPACK_SEQUENCE_TWO_TUPLE 173 +#define BINARY_OP_MULTIPLY_INT 17 +#define BINARY_OP_SUBTRACT_FLOAT 18 +#define BINARY_OP_SUBTRACT_INT 19 +#define BINARY_SUBSCR_DICT 20 +#define BINARY_SUBSCR_GETITEM 21 +#define BINARY_SUBSCR_LIST_INT 22 +#define BINARY_SUBSCR_TUPLE_INT 23 +#define CALL_PY_EXACT_ARGS 24 +#define CALL_PY_WITH_DEFAULTS 28 +#define CALL_BOUND_METHOD_EXACT_ARGS 29 +#define CALL_BUILTIN_CLASS 34 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 38 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 39 +#define CALL_NO_KW_BUILTIN_FAST 40 +#define CALL_NO_KW_BUILTIN_O 41 +#define CALL_NO_KW_ISINSTANCE 42 +#define CALL_NO_KW_LEN 43 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 44 +#define CALL_NO_KW_LIST_APPEND 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 46 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 47 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 48 +#define CALL_NO_KW_STR_1 56 +#define CALL_NO_KW_TUPLE_1 57 +#define CALL_NO_KW_TYPE_1 58 +#define COMPARE_OP_FLOAT_JUMP 59 +#define COMPARE_OP_GENERIC 62 +#define COMPARE_OP_INT_JUMP 64 +#define COMPARE_OP_STR_JUMP 65 +#define FOR_ITER_LIST 66 +#define FOR_ITER_RANGE 67 +#define FOR_ITER_GEN 72 +#define LOAD_ATTR_CLASS 73 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 +#define LOAD_ATTR_INSTANCE_VALUE 77 +#define LOAD_ATTR_MODULE 78 +#define LOAD_ATTR_PROPERTY 79 +#define LOAD_ATTR_SLOT 80 +#define LOAD_ATTR_WITH_HINT 81 +#define LOAD_ATTR_METHOD_LAZY_DICT 86 +#define LOAD_ATTR_METHOD_NO_DICT 113 +#define LOAD_ATTR_METHOD_WITH_DICT 121 +#define LOAD_ATTR_METHOD_WITH_VALUES 141 +#define LOAD_CONST__LOAD_FAST 143 +#define LOAD_FAST__LOAD_CONST 153 +#define LOAD_FAST__LOAD_FAST 154 +#define LOAD_GLOBAL_BUILTIN 158 +#define LOAD_GLOBAL_MODULE 159 +#define STORE_ATTR_INSTANCE_VALUE 160 +#define STORE_ATTR_SLOT 161 +#define STORE_ATTR_WITH_HINT 166 +#define STORE_FAST__LOAD_FAST 167 +#define STORE_FAST__STORE_FAST 168 +#define STORE_SUBSCR_DICT 169 +#define STORE_SUBSCR_LIST_INT 170 +#define UNPACK_SEQUENCE_LIST 173 +#define UNPACK_SEQUENCE_TUPLE 174 +#define UNPACK_SEQUENCE_TWO_TUPLE 175 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 0ee75958508ac7..9557fe4cb05a8f 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -87,6 +87,7 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_NOT', 12) def_op('UNARY_INVERT', 15) +def_op('EXIT_INIT_CHECK', 16) def_op('BINARY_SUBSCR', 25) def_op('BINARY_SLICE', 26) @@ -306,6 +307,7 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_BUILTIN_O", "CALL_NO_KW_ISINSTANCE", "CALL_NO_KW_LEN", + "CALL_NO_KW_ALLOC_AND_ENTER_INIT", "CALL_NO_KW_LIST_APPEND", "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 2403c7c815f2c0..83bc8b454daca5 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1532,7 +1532,7 @@ def delx(self): del self.__x '10P' # PySequenceMethods '2P' # PyBufferProcs '6P' - '1P' # Specializer cache + '2P' # Specializer cache ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..d02aa197a43d53 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1569,6 +1569,29 @@ def func(): self.run_and_compare(func, EXPECTED_EVENTS) + def test_correct_tracing_quickened_call_class_init(self): + + class C: + def __init__(self): + self + + def func(): + C() + + EXPECTED_EVENTS = [ + (0, 'call'), + (1, 'line'), + (-3, 'call'), + (-2, 'line'), + (-2, 'return'), + (1, 'return')] + + self.run_and_compare(func, EXPECTED_EVENTS) + # Quicken + for _ in range(100): + func() + self.run_and_compare(func, EXPECTED_EVENTS) + def test_very_large_function(self): # There is a separate code path when the number of lines > (1 << 15). d = {} diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 97007479b1be91..3bb6ba93c5eea2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5324,11 +5324,10 @@ _PyDict_NewKeysForClass(void) #define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) -static int -init_inline_values(PyObject *obj, PyTypeObject *tp) +int +_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - // assert(type->tp_dictoffset > 0); -- TO DO Update this assert. assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); @@ -5360,7 +5359,7 @@ _PyObject_InitializeDict(PyObject *obj) } if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { OBJECT_STAT_INC(new_values); - return init_inline_values(obj, tp); + return _PyObject_InitInlineValues(obj, tp); } PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e2e40b5c15297d..2c5a392a977c84 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -502,7 +502,6 @@ PyType_Modified(PyTypeObject *type) } } - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; /* 0 is not a valid version tag */ } @@ -1309,6 +1308,26 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) return obj; } +PyObject * +_PyType_NewManagedObject(PyTypeObject *type) +{ + assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyType_IS_GC(type)); + assert(type->tp_new == PyBaseObject_Type.tp_new); + assert(type->tp_alloc == PyType_GenericAlloc); + assert(type->tp_itemsize == 0); + PyObject *obj = PyType_GenericAlloc(type, 0); + if (obj == NULL) { + return PyErr_NoMemory(); + } + _PyObject_DictOrValuesPointer(obj)->dict = NULL; + if (_PyObject_InitInlineValues(obj, type)) { + Py_DECREF(obj); + return NULL; + } + return obj; +} + PyObject * _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ed12b94e3142c1..3437fb0f036847 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3147,6 +3147,74 @@ dummy_func( CHECK_EVAL_BREAKER(); } + inst(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + assert(kwnames == NULL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + DEOPT_IF(is_method(stack_pointer, oparg), CALL); + PyObject *callable = PEEK(oparg+1); + DEOPT_IF(!PyType_Check(callable), CALL); + PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; + DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); + PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); + STAT_INC(CALL, hit); + DEOPT_IF(code->co_argcount != oparg+1, CALL); + PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + if (self == NULL) { + goto error; + } + PEEK(oparg+1) = self; + Py_DECREF(tp); + CALL_STAT_INC(inlined_py_calls); + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, interp->callable_cache.init_cleanup, 1, 1); + assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + shim->previous = frame; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init); + /* Copy self followed by args to __init__ frame */ + STACK_SHRINK(oparg+1); + for (int i = 0; i <= oparg; i++) { + init_frame->localsplus[i] = stack_pointer[i]; + } + STACK_SHRINK(1); // Clear the NULL. + _PyFrame_SetStackPointer(frame, stack_pointer); + for (int i = oparg+1; i < code->co_nlocalsplus; i++) { + init_frame->localsplus[i] = NULL; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + init_frame->previous = shim; + frame = cframe.current_frame = init_frame; + goto start_frame; + } + + inst(EXIT_INIT_CHECK) { + assert(STACK_LEVEL() == 2); + PyObject *should_be_none = TOP(); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + STACK_SHRINK(1); + _Py_DECREF_NO_DEALLOC(Py_None); + } + // stack effect: (__0, __array[oparg] -- ) inst(CALL_BUILTIN_CLASS) { int is_meth = is_method(stack_pointer, oparg); diff --git a/Python/compile.c b/Python/compile.c index c71563f81609a2..3adbfae80471a4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1138,6 +1138,9 @@ stack_effect(int opcode, int oparg, int jump) case LOAD_GLOBAL: return (oparg & 1) + 1; + case EXIT_INIT_CHECK: + return -1; + /* Exception handling pseudo-instructions */ case SETUP_FINALLY: /* 0 in the normal flow. diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0487851db6106e..4b830f20c7828a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3139,6 +3139,75 @@ DISPATCH(); } + TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + assert(kwnames == NULL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + DEOPT_IF(is_method(stack_pointer, oparg), CALL); + PyObject *callable = PEEK(oparg+1); + DEOPT_IF(!PyType_Check(callable), CALL); + PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; + DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); + PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); + STAT_INC(CALL, hit); + DEOPT_IF(code->co_argcount != oparg+1, CALL); + PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + if (self == NULL) { + goto error; + } + PEEK(oparg+1) = self; + Py_DECREF(tp); + CALL_STAT_INC(inlined_py_calls); + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, interp->callable_cache.init_cleanup, 1, 1); + assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + shim->previous = frame; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init); + /* Copy self followed by args to __init__ frame */ + STACK_SHRINK(oparg+1); + for (int i = 0; i <= oparg; i++) { + init_frame->localsplus[i] = stack_pointer[i]; + } + STACK_SHRINK(1); // Clear the NULL. + _PyFrame_SetStackPointer(frame, stack_pointer); + for (int i = oparg+1; i < code->co_nlocalsplus; i++) { + init_frame->localsplus[i] = NULL; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + init_frame->previous = shim; + frame = cframe.current_frame = init_frame; + goto start_frame; + } + + TARGET(EXIT_INIT_CHECK) { + assert(STACK_LEVEL() == 2); + PyObject *should_be_none = TOP(); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + STACK_SHRINK(1); + _Py_DECREF_NO_DEALLOC(Py_None); + DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 237d3b946b1066..23ed32c7ad2476 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -15,6 +15,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_EXIT_INIT_CHECK, &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, @@ -23,31 +24,30 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, + &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_NO_KW_LIST_APPEND, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,37 +55,37 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_COMPARE_OP_FLOAT_JUMP, - &&TARGET_COMPARE_OP_GENERIC, - &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_COMPARE_OP_STR_JUMP, + &&TARGET_COMPARE_OP_GENERIC, &&TARGET_STOPITERATION_ERROR, + &&TARGET_COMPARE_OP_INT_JUMP, + &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, - &&TARGET_FOR_ITER_GEN, - &&TARGET_LOAD_ATTR_CLASS, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_FOR_ITER_GEN, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,26 +152,28 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_CALL, &&TARGET_KW_NAMES, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, @@ -252,7 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/pystate.c b/Python/pystate.c index 04db1fb419af62..85586ca8f428a1 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -454,6 +454,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + Py_CLEAR(interp->callable_cache.init_cleanup); Py_CLEAR(interp->interpreter_trampoline); for (int i=0; i < DICT_MAX_WATCHERS; i++) { diff --git a/Python/specialize.c b/Python/specialize.c index f84596751f9186..dc1d9898c9dd05 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1436,11 +1436,45 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins cache->counter = adaptive_counter_cooldown(); } +static PyFunctionObject * +get_init_for_simple_managed_python_class(PyTypeObject *tp) +{ + if (tp->tp_new != PyBaseObject_Type.tp_new) { + return NULL; + } + if (tp->tp_alloc != PyType_GenericAlloc) { + return NULL; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return NULL; + } + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + return NULL; + } + PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); + if (init == NULL || !PyFunction_Check(init)) { + return NULL; + } + int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); + if (kind != SIMPLE_FUNCTION) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_INIT_NOT_SIMPLE); + return NULL; + } + Py_CLEAR(((PyHeapTypeObject *)tp)->_spec_cache.init); + ((PyHeapTypeObject *)tp)->_spec_cache.init = init; + return (PyFunctionObject *)init; +} +static int setup_init_cleanup_code(void); + static int specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { + assert(PyType_Check(callable)); PyTypeObject *tp = _PyType_CAST(callable); + if (setup_init_cleanup_code()) { + return -1; + } if (tp->tp_new == PyBaseObject_Type.tp_new) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); return -1; @@ -1469,6 +1503,23 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + else { + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init != NULL) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } + } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } @@ -2176,3 +2227,42 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) STAT_INC(FOR_ITER, success); cache->counter = adaptive_counter_cooldown(); } + +/* Code init cleanup. + * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up + * the frame to execute the EXIT_INIT_CHECK + * instruction. + * Starts with an assertion error, in case it is called + * directly. + * Ends with a RESUME so that it is not traced. + * This is used as a plain code object, not a function, + * so must not access globals or builtins. + */ +static const uint8_t INIT_CLEANUP_CODE[10] = { + LOAD_ASSERTION_ERROR, 0, + RAISE_VARARGS, 1, + EXIT_INIT_CHECK, 0, + RETURN_VALUE, 0, + RESUME, 0 +}; + +static const _PyShimCodeDef INIT_CLEANUP_DEF = { + INIT_CLEANUP_CODE, + sizeof(INIT_CLEANUP_CODE), + 2, + "type.__call__" +}; + +static int +setup_init_cleanup_code(void) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->callable_cache.init_cleanup != NULL) { + return 0; + } + PyCodeObject *init = _Py_MakeShimCode(&INIT_CLEANUP_DEF); + if (init == NULL) { + return -1; + } + interp->callable_cache.init_cleanup = init; + return 0; +} From 43edcd66a82099a8d2f430598edb6df548374d6f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Nov 2022 16:03:24 +0000 Subject: [PATCH 02/10] Add news --- .../2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst new file mode 100644 index 00000000000000..5633097f4a3fdd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst @@ -0,0 +1,11 @@ +Specializes calls to most Python classes. Specifically, any class that +inherits from ``object``, or another Python class, and does not override +``__new__``. + +The specialized instruction does the following: + +1. Creates the object (by calling ``object.__new__``) +2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) +3. Pushes the frame for ``__init__`` to the frame stack + +Speeds up the instantiation of most Python classes. From fbeaf9bab6a38b1da6c2cf8fed38914c441d7aa4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Mar 2023 03:50:37 +0000 Subject: [PATCH 03/10] Fix up after merge --- Include/internal/pycore_frame.h | 6 ++--- Python/bytecodes.c | 16 ++++++-------- Python/ceval.c | 2 +- Python/generated_cases.c.h | 18 +++++++-------- Python/specialize.c | 39 +++++++++++++++++---------------- 5 files changed, 39 insertions(+), 42 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8edf21447cefaa..bf5c30309f0aa3 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -255,7 +255,7 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int start) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int prev_instr) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; @@ -265,12 +265,12 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_code = (PyCodeObject *)Py_NewRef(code); #ifdef Py_DEBUG frame->f_builtins = (PyObject*)0xaaaa; - frame->f_globals = (PyObject*)0xaaab; + frame->f_globals = NULL; #endif frame->f_locals = NULL; frame->stacktop = code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code) + start; + frame->prev_instr = _PyCode_CODE(code) + prev_instr; frame->owner = FRAME_OWNED_BY_THREAD; frame->yield_offset = 0; return frame; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7407e40915c5f7..390b5b5de51ebb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2615,7 +2615,6 @@ dummy_func( if (self == NULL) { goto error; } - PEEK(oparg+1) = self; Py_DECREF(tp); CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { @@ -2624,25 +2623,24 @@ dummy_func( PyInterpreterState *interp = _PyInterpreterState_GET(); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, interp->callable_cache.init_cleanup, 1, 1); - assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + assert(_PyCode_CODE(shim->f_code)[2].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; - shim->previous = frame; Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, code->co_argcount); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); /* Copy self followed by args to __init__ frame */ - for (int i = 0; i <= oparg; i++) { - init_frame->localsplus[i] = args[i]; - } - for (int i = oparg+1; i < code->co_nlocalsplus; i++) { - init_frame->localsplus[i] = NULL; + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; } STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->prev_instr = next_instr - 1; + /* Link frames */ init_frame->previous = shim; + shim->previous = frame; frame = cframe.current_frame = init_frame; goto start_frame; } diff --git a/Python/ceval.c b/Python/ceval.c index b422d0ed34ede3..3b265ab4bbe8c3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -808,7 +808,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef LLTRACE { - if (frame != &entry_frame) { + if (frame != &entry_frame && GLOBALS()) { int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); if (r < 0) { goto exit_unwind; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9445ce2e5ec7c8..f238a4973581d4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3273,7 +3273,6 @@ if (self == NULL) { goto error; } - PEEK(oparg+1) = self; Py_DECREF(tp); CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { @@ -3282,25 +3281,24 @@ PyInterpreterState *interp = _PyInterpreterState_GET(); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, interp->callable_cache.init_cleanup, 1, 1); - assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + assert(_PyCode_CODE(shim->f_code)[2].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; - shim->previous = frame; Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, code->co_argcount); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; /* Copy self followed by args to __init__ frame */ - for (int i = 0; i <= oparg; i++) { - init_frame->localsplus[i] = args[i]; - } - for (int i = oparg+1; i < code->co_nlocalsplus; i++) { - init_frame->localsplus[i] = NULL; + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; } STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->prev_instr = next_instr - 1; - init_frame->previous = shim; frame = cframe.current_frame = init_frame; goto start_frame; } diff --git a/Python/specialize.c b/Python/specialize.c index ed0d97f485f129..4c3207ca6b3f98 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -434,6 +434,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_KWNAMES 27 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 +#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_DIFFERENT_TYPES 12 @@ -1478,6 +1479,9 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins cache->counter = adaptive_counter_cooldown(); } +/* Returns a borrowed reference. + * The reference is only valid if guarded by a type version check. + */ static PyFunctionObject * get_init_for_simple_managed_python_class(PyTypeObject *tp) { @@ -1499,10 +1503,9 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp) } int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); if (kind != SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_INIT_NOT_SIMPLE); + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE); return NULL; } - Py_CLEAR(((PyHeapTypeObject *)tp)->_spec_cache.init); ((PyHeapTypeObject *)tp)->_spec_cache.init = init; return (PyFunctionObject *)init; } @@ -1518,6 +1521,21 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, return -1; } if (tp->tp_new == PyBaseObject_Type.tp_new) { + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init != NULL) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); return -1; } @@ -1545,23 +1563,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } - else { - PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); - if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } - if (kwnames) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); - return -1; - } - _PyCallCache *cache = (_PyCallCache *)(instr + 1); - write_u32(cache->func_version, tp->tp_version_tag); - _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); - return 0; - } - } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } From b851a2a1d6bcdb9d7ba1269dc179cbce89e3354c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Mar 2023 04:15:39 +0000 Subject: [PATCH 04/10] Update generated cases. --- Python/generated_cases.c.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f238a4973581d4..734ed7b5ca61a6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3287,9 +3287,6 @@ shim->localsplus[0] = self; Py_INCREF(init); _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; /* Copy self followed by args to __init__ frame */ init_frame->localsplus[0] = self; for (int i = 0; i < oparg; i++) { @@ -3299,6 +3296,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->prev_instr = next_instr - 1; + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; frame = cframe.current_frame = init_frame; goto start_frame; } From e46f3241efe88cb72a16a5653a9845480fe99eb1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Mar 2023 06:39:38 +0000 Subject: [PATCH 05/10] Swap order of specializations and make sure we have meaningful stats. --- Python/specialize.c | 48 +++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 4c3207ca6b3f98..866eaba6d4699b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -425,7 +425,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18 #define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20 -#define SPEC_FAIL_CALL_PYTHON_CLASS 21 +#define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21 #define SPEC_FAIL_CALL_PEP_523 22 #define SPEC_FAIL_CALL_BOUND_METHOD 23 #define SPEC_FAIL_CALL_STR 24 @@ -1485,20 +1485,23 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins static PyFunctionObject * get_init_for_simple_managed_python_class(PyTypeObject *tp) { - if (tp->tp_new != PyBaseObject_Type.tp_new) { - return NULL; - } + assert(tp->tp_new == PyBaseObject_Type.tp_new); if (tp->tp_alloc != PyType_GenericAlloc) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT); return NULL; } if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + /* Is this possible? */ + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR); return NULL; } PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); if (init == NULL || !PyFunction_Check(init)) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON); return NULL; } int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); @@ -1520,25 +1523,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (setup_init_cleanup_code()) { return -1; } - if (tp->tp_new == PyBaseObject_Type.tp_new) { - PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); - if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } - if (kwnames) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); - return -1; - } - _PyCallCache *cache = (_PyCallCache *)(instr + 1); - write_u32(cache->func_version, tp->tp_version_tag); - _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); - return 0; - } - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); - return -1; - } if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { int oparg = instr->op.arg; if (nargs == 1 && kwnames == NULL && oparg == 1) { @@ -1563,6 +1547,24 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + if (tp->tp_new == PyBaseObject_Type.tp_new) { + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init != NULL) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } + return -1; + } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } From dab56e5a5a3301f8522fc71b1cebe13519997bc1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 09:53:28 +0100 Subject: [PATCH 06/10] Fix up new specialization to work with current interpreter generator and instrumentation. --- Include/internal/pycore_frame.h | 4 +- Python/bytecodes.c | 8 ++-- Python/generated_cases.c.h | 85 +++++++++++++++++---------------- Python/instrumentation.c | 2 +- Python/opcode_metadata.h | 2 +- Python/specialize.c | 4 +- 6 files changed, 53 insertions(+), 52 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index b9f6012545da0c..952a96009dda6c 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -281,10 +281,10 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - frame->f_funcobj = Py_NewRef(Py_None); + frame->f_funcobj = Py_None; frame->f_executable = Py_NewRef(code); #ifdef Py_DEBUG - frame->f_builtins = (PyObject*)0xaaaa; + frame->f_builtins = NULL; frame->f_globals = NULL; #endif frame->f_locals = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5c2a3dcbdfc0db..9970e6efbb7794 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2923,7 +2923,6 @@ dummy_func( goto error; } Py_DECREF(tp); - CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } @@ -2940,14 +2939,16 @@ dummy_func( for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } + SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + frame->return_offset = 0; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; /* Link frames */ init_frame->previous = shim; shim->previous = frame; frame = cframe.current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); goto start_frame; } @@ -2961,7 +2962,6 @@ dummy_func( } } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { int is_meth = method != NULL; int total_args = oparg; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8bdc17e4496ba5..0ada77e13cdf50 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4050,7 +4050,6 @@ goto error; } Py_DECREF(tp); - CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } @@ -4067,21 +4066,23 @@ for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } + SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + frame->return_offset = 0; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; /* Link frames */ init_frame->previous = shim; shim->previous = frame; frame = cframe.current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); goto start_frame; - #line 4080 "Python/generated_cases.c.h" + #line 4081 "Python/generated_cases.c.h" } TARGET(EXIT_INIT_CHECK) { PyObject *should_be_none = stack_pointer[-1]; - #line 2955 "Python/bytecodes.c" + #line 2956 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -4089,7 +4090,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 4093 "Python/generated_cases.c.h" + #line 4094 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -4121,7 +4122,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4125 "Python/generated_cases.c.h" + #line 4126 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4163,7 +4164,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4167 "Python/generated_cases.c.h" + #line 4168 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4209,7 +4210,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4213 "Python/generated_cases.c.h" + #line 4214 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4255,7 +4256,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4259 "Python/generated_cases.c.h" + #line 4260 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4294,7 +4295,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4298 "Python/generated_cases.c.h" + #line 4299 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4334,7 +4335,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4338 "Python/generated_cases.c.h" + #line 4339 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4364,7 +4365,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4368 "Python/generated_cases.c.h" + #line 4369 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { @@ -4402,7 +4403,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4406 "Python/generated_cases.c.h" + #line 4407 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4444,7 +4445,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4448 "Python/generated_cases.c.h" + #line 4449 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4486,7 +4487,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4490 "Python/generated_cases.c.h" + #line 4491 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4527,7 +4528,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4531 "Python/generated_cases.c.h" + #line 4532 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4539,7 +4540,7 @@ TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { #line 3298 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4543 "Python/generated_cases.c.h" + #line 4544 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4610,14 +4611,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4614 "Python/generated_cases.c.h" + #line 4615 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); #line 3364 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4621 "Python/generated_cases.c.h" + #line 4622 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4640,7 +4641,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4644 "Python/generated_cases.c.h" + #line 4645 "Python/generated_cases.c.h" stack_pointer[-1] = func; DISPATCH(); } @@ -4673,7 +4674,7 @@ default: Py_UNREACHABLE(); } - #line 4677 "Python/generated_cases.c.h" + #line 4678 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; DISPATCH(); @@ -4701,7 +4702,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4705 "Python/generated_cases.c.h" + #line 4706 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4711,13 +4712,13 @@ PyObject *slice; #line 3434 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4715 "Python/generated_cases.c.h" + #line 4716 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); #line 3436 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4721 "Python/generated_cases.c.h" + #line 4722 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4734,7 +4735,7 @@ result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 4738 "Python/generated_cases.c.h" + #line 4739 "Python/generated_cases.c.h" stack_pointer[-1] = result; DISPATCH(); } @@ -4753,7 +4754,7 @@ else { res = value; } - #line 4757 "Python/generated_cases.c.h" + #line 4758 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -4767,7 +4768,7 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 4771 "Python/generated_cases.c.h" + #line 4772 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -4779,7 +4780,7 @@ #line 3469 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4783 "Python/generated_cases.c.h" + #line 4784 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4806,12 +4807,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4810 "Python/generated_cases.c.h" + #line 4811 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); #line 3489 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4815 "Python/generated_cases.c.h" + #line 4816 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4823,7 +4824,7 @@ PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; #line 3494 "Python/bytecodes.c" assert(oparg >= 2); - #line 4827 "Python/generated_cases.c.h" + #line 4828 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); @@ -4842,20 +4843,20 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4846 "Python/generated_cases.c.h" + #line 4847 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { #line 3512 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4852 "Python/generated_cases.c.h" + #line 4853 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { #line 3516 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); - #line 4859 "Python/generated_cases.c.h" + #line 4860 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -4870,7 +4871,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4874 "Python/generated_cases.c.h" + #line 4875 "Python/generated_cases.c.h" DISPATCH(); } @@ -4884,7 +4885,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4888 "Python/generated_cases.c.h" + #line 4889 "Python/generated_cases.c.h" DISPATCH(); } @@ -4901,7 +4902,7 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4905 "Python/generated_cases.c.h" + #line 4906 "Python/generated_cases.c.h" DISPATCH(); } @@ -4918,7 +4919,7 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4922 "Python/generated_cases.c.h" + #line 4923 "Python/generated_cases.c.h" DISPATCH(); } @@ -4929,19 +4930,19 @@ oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4933 "Python/generated_cases.c.h" + #line 4934 "Python/generated_cases.c.h" } TARGET(CACHE) { #line 3579 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4940 "Python/generated_cases.c.h" + #line 4941 "Python/generated_cases.c.h" } TARGET(RESERVED) { #line 3584 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4947 "Python/generated_cases.c.h" + #line 4948 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index fcaccc022c8b2d..524a82c076e99d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1526,7 +1526,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } /* Insert instrumentation */ - for (int i = 0; i < code_len; i+= _PyInstruction_GetLength(code, i)) { + for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; CHECK(instr->op.code != 0); int base_opcode = _Py_GetBaseOpcode(code, i); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index b0f759e0147584..421d9795e7da76 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -1065,7 +1065,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[512] = { [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, 0 }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, diff --git a/Python/specialize.c b/Python/specialize.c index f0d663fe239410..66a4e7669022da 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2291,11 +2291,11 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) * so must not access globals or builtins. */ -#define NO_LOC (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 7) +#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) static PyBytesObject no_location = { PyVarObject_HEAD_INIT(&PyBytes_Type, 1) - .ob_sval = { NO_LOC } + .ob_sval = { NO_LOC_4 } }; struct _PyCode_DEF(8) _Py_InitCleanup = { From 4f4de5f3cac12a2bc0135222f8df78b0486aecb3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 10:26:25 +0100 Subject: [PATCH 07/10] Address review comments --- Python/bytecodes.c | 13 +-- Python/generated_cases.c.h | 163 +++++++++++++++++++------------------ 2 files changed, 89 insertions(+), 87 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9970e6efbb7794..3ef8ed0ccaafdc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2911,14 +2911,15 @@ dummy_func( _PyCallCache *cache = (_PyCallCache *)next_instr; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); - PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; - DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); - PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); - STAT_INC(CALL, hit); DEOPT_IF(code->co_argcount != oparg+1, CALL); - PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { goto error; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0ada77e13cdf50..155f41fc5367f6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4038,14 +4038,15 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); - PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; - DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); - PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); - STAT_INC(CALL, hit); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); DEOPT_IF(code->co_argcount != oparg+1, CALL); - PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { goto error; } @@ -4077,12 +4078,12 @@ frame = cframe.current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); goto start_frame; - #line 4081 "Python/generated_cases.c.h" + #line 4082 "Python/generated_cases.c.h" } TARGET(EXIT_INIT_CHECK) { PyObject *should_be_none = stack_pointer[-1]; - #line 2956 "Python/bytecodes.c" + #line 2957 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -4090,7 +4091,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 4094 "Python/generated_cases.c.h" + #line 4095 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -4100,7 +4101,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2966 "Python/bytecodes.c" + #line 2967 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4122,7 +4123,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4126 "Python/generated_cases.c.h" + #line 4127 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4136,7 +4137,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2991 "Python/bytecodes.c" + #line 2992 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4164,7 +4165,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4168 "Python/generated_cases.c.h" + #line 4169 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4178,7 +4179,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3022 "Python/bytecodes.c" + #line 3023 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4210,7 +4211,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4214 "Python/generated_cases.c.h" + #line 4215 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4224,7 +4225,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3057 "Python/bytecodes.c" + #line 3058 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4256,7 +4257,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4260 "Python/generated_cases.c.h" + #line 4261 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4270,7 +4271,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3092 "Python/bytecodes.c" + #line 3093 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4295,7 +4296,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4299 "Python/generated_cases.c.h" + #line 4300 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4308,7 +4309,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3119 "Python/bytecodes.c" + #line 3120 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4335,7 +4336,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4339 "Python/generated_cases.c.h" + #line 4340 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4347,7 +4348,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3149 "Python/bytecodes.c" + #line 3150 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4365,14 +4366,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4369 "Python/generated_cases.c.h" + #line 4370 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3169 "Python/bytecodes.c" + #line 3170 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4403,7 +4404,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4407 "Python/generated_cases.c.h" + #line 4408 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4416,7 +4417,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3203 "Python/bytecodes.c" + #line 3204 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4445,7 +4446,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4449 "Python/generated_cases.c.h" + #line 4450 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4458,7 +4459,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3235 "Python/bytecodes.c" + #line 3236 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4487,7 +4488,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4491 "Python/generated_cases.c.h" + #line 4492 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4500,7 +4501,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3267 "Python/bytecodes.c" + #line 3268 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4528,7 +4529,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4532 "Python/generated_cases.c.h" + #line 4533 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4538,9 +4539,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3298 "Python/bytecodes.c" + #line 3299 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4544 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4549,7 +4550,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3302 "Python/bytecodes.c" + #line 3303 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4611,14 +4612,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4615 "Python/generated_cases.c.h" + #line 4616 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3364 "Python/bytecodes.c" + #line 3365 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4622 "Python/generated_cases.c.h" + #line 4623 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4629,7 +4630,7 @@ TARGET(MAKE_FUNCTION) { PyObject *codeobj = stack_pointer[-1]; PyObject *func; - #line 3370 "Python/bytecodes.c" + #line 3371 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4641,7 +4642,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4645 "Python/generated_cases.c.h" + #line 4646 "Python/generated_cases.c.h" stack_pointer[-1] = func; DISPATCH(); } @@ -4649,7 +4650,7 @@ TARGET(SET_FUNCTION_ATTRIBUTE) { PyObject *func = stack_pointer[-1]; PyObject *attr = stack_pointer[-2]; - #line 3384 "Python/bytecodes.c" + #line 3385 "Python/bytecodes.c" assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4674,14 +4675,14 @@ default: Py_UNREACHABLE(); } - #line 4678 "Python/generated_cases.c.h" + #line 4679 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3411 "Python/bytecodes.c" + #line 3412 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4702,7 +4703,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4706 "Python/generated_cases.c.h" + #line 4707 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4710,15 +4711,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3434 "Python/bytecodes.c" + #line 3435 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4716 "Python/generated_cases.c.h" + #line 4717 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3436 "Python/bytecodes.c" + #line 3437 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4722 "Python/generated_cases.c.h" + #line 4723 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4728,14 +4729,14 @@ TARGET(CONVERT_VALUE) { PyObject *value = stack_pointer[-1]; PyObject *result; - #line 3440 "Python/bytecodes.c" + #line 3441 "Python/bytecodes.c" convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 4739 "Python/generated_cases.c.h" + #line 4740 "Python/generated_cases.c.h" stack_pointer[-1] = result; DISPATCH(); } @@ -4743,7 +4744,7 @@ TARGET(FORMAT_SIMPLE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 3449 "Python/bytecodes.c" + #line 3450 "Python/bytecodes.c" /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4754,7 +4755,7 @@ else { res = value; } - #line 4758 "Python/generated_cases.c.h" + #line 4759 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -4763,12 +4764,12 @@ PyObject *fmt_spec = stack_pointer[-1]; PyObject *value = stack_pointer[-2]; PyObject *res; - #line 3462 "Python/bytecodes.c" + #line 3463 "Python/bytecodes.c" res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 4772 "Python/generated_cases.c.h" + #line 4773 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -4777,10 +4778,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3469 "Python/bytecodes.c" + #line 3470 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4784 "Python/generated_cases.c.h" + #line 4785 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4792,7 +4793,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3474 "Python/bytecodes.c" + #line 3475 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4807,12 +4808,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4811 "Python/generated_cases.c.h" + #line 4812 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3489 "Python/bytecodes.c" + #line 3490 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4816 "Python/generated_cases.c.h" + #line 4817 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4822,16 +4823,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3494 "Python/bytecodes.c" + #line 3495 "Python/bytecodes.c" assert(oparg >= 2); - #line 4828 "Python/generated_cases.c.h" + #line 4829 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3498 "Python/bytecodes.c" + #line 3499 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4843,26 +4844,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4847 "Python/generated_cases.c.h" + #line 4848 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3512 "Python/bytecodes.c" + #line 3513 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4853 "Python/generated_cases.c.h" + #line 4854 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3516 "Python/bytecodes.c" + #line 3517 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); - #line 4860 "Python/generated_cases.c.h" + #line 4861 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3521 "Python/bytecodes.c" + #line 3522 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4871,12 +4872,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4875 "Python/generated_cases.c.h" + #line 4876 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3532 "Python/bytecodes.c" + #line 3533 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4885,12 +4886,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4889 "Python/generated_cases.c.h" + #line 4890 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3543 "Python/bytecodes.c" + #line 3544 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4902,12 +4903,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4906 "Python/generated_cases.c.h" + #line 4907 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3557 "Python/bytecodes.c" + #line 3558 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4919,30 +4920,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4923 "Python/generated_cases.c.h" + #line 4924 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3571 "Python/bytecodes.c" + #line 3572 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4934 "Python/generated_cases.c.h" + #line 4935 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3579 "Python/bytecodes.c" + #line 3580 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4941 "Python/generated_cases.c.h" + #line 4942 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3584 "Python/bytecodes.c" + #line 3585 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4948 "Python/generated_cases.c.h" + #line 4949 "Python/generated_cases.c.h" } From fbe0afe0c89f2c41d2484c9aa055216af94c9d41 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 10:27:33 +0100 Subject: [PATCH 08/10] Update generated cases. --- Python/generated_cases.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 155f41fc5367f6..11ca535adfb19b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4043,8 +4043,8 @@ PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { From f76ab3dfa3153d198a98485c22fbaf3cd363a432 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 11:24:14 +0100 Subject: [PATCH 09/10] Make structs const. --- Python/ceval.c | 2 +- Python/specialize.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index e0b18c9387e169..53107018978c0b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -641,7 +641,7 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = 0 } }; -extern struct _PyCode_DEF(8) _Py_InitCleanup; +extern const struct _PyCode_DEF(8) _Py_InitCleanup; /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ diff --git a/Python/specialize.c b/Python/specialize.c index 66a4e7669022da..1a630821577ca1 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2293,12 +2293,12 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) #define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) -static PyBytesObject no_location = { +static const PyBytesObject no_location = { PyVarObject_HEAD_INIT(&PyBytes_Type, 1) .ob_sval = { NO_LOC_4 } }; -struct _PyCode_DEF(8) _Py_InitCleanup = { +const struct _PyCode_DEF(8) _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 4) .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), From 1df353e6f533475d037af1f7a1e3ed3b6835dd8c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 16:15:13 +0100 Subject: [PATCH 10/10] Fix comment. --- Python/specialize.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 1a630821577ca1..0006aa733bd6cb 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2284,8 +2284,6 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up * the frame to execute the EXIT_INIT_CHECK * instruction. - * Starts with an assertion error, in case it is called - * directly. * Ends with a RESUME so that it is not traced. * This is used as a plain code object, not a function, * so must not access globals or builtins.