diff --git a/Include/cpython/object.h b/Include/cpython/object.h index c80fc1df0e0ba4..29431e782f4746 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -231,6 +231,7 @@ struct _typeobject { * by code other than the specializer and interpreter. */ struct _specialization_cache { PyObject *getitem; + PyObject *init; }; /* The *real* layout of a type object when allocated on the heap */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 7d5fe941fcb4d0..46b7e2c3864f49 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -119,6 +119,8 @@ struct callable_cache { PyObject *len; PyObject *list_append; PyObject *object__getattribute__; + /* This is a strong reference */ + PyCodeObject *init_cleanup; }; /* "Locals plus" for a code object is the set of locals + cell vars + diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index decaafd141e9e1..0d523cb5bf3349 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -95,9 +95,27 @@ static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest); +static inline void +_PyFrame_InitializeSpecialsTrampoline( + _PyInterpreterFrame *frame, PyCodeObject *code, int stackdepth, int start) +{ + frame->f_funcobj = Py_NewRef(Py_None); + frame->f_code = (PyCodeObject *)Py_NewRef(code); +#ifdef Py_DEBUG + frame->f_builtins = NULL; + frame->f_globals = NULL; +#endif + frame->f_locals = NULL; + frame->stacktop = code->co_nlocalsplus + stackdepth; + frame->frame_obj = NULL; + frame->prev_instr = _PyCode_CODE(code) + start; + frame->is_entry = false; + frame->owner = FRAME_OWNED_BY_THREAD; +} + /* Consumes reference to func and locals */ static inline void -_PyFrame_InitializeSpecials( +_PyFrame_InitializeSpecialsFromFunction( _PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals, PyCodeObject *code) { @@ -213,7 +231,20 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func) _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - _PyFrame_InitializeSpecials(new_frame, func, NULL, code); + _PyFrame_InitializeSpecialsFromFunction(new_frame, func, NULL, code); + return new_frame; +} + +/* Pushes a trampoline frame without checking for space. + * Must be guarded by _PyThreadState_HasStackSpace() */ +static inline _PyInterpreterFrame * +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int start) +{ + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; + tstate->datastack_top += code->co_framesize; + assert(tstate->datastack_top < tstate->datastack_limit); + _PyFrame_InitializeSpecialsTrampoline(new_frame, code, stackdepth, start); return new_frame; } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8b78f79e950e92..be21671b1cbb8c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -300,8 +300,10 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); +PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern int _PyObject_InitializeDict(PyObject *obj); +int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 587590172b5615..ae21bf5b434107 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -89,6 +89,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, [CALL_NO_KW_ISINSTANCE] = CALL, @@ -122,6 +123,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_MERGE] = DICT_MERGE, [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, + [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, [EXTENDED_ARG] = EXTENDED_ARG, [EXTENDED_ARG_QUICK] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, @@ -260,6 +262,7 @@ static const char *const _PyOpcode_OpName[267] = { [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_ADAPTIVE] = "BINARY_SUBSCR_ADAPTIVE", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", @@ -268,31 +271,30 @@ static const char *const _PyOpcode_OpName[267] = { [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_ADAPTIVE] = "CALL_ADAPTIVE", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -300,37 +302,37 @@ static const char *const _PyOpcode_OpName[267] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [COMPARE_OP_ADAPTIVE] = "COMPARE_OP_ADAPTIVE", - [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", - [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", + [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -357,7 +359,7 @@ static const char *const _PyOpcode_OpName[267] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [POP_JUMP_FORWARD_IF_FALSE] = "POP_JUMP_FORWARD_IF_FALSE", [POP_JUMP_FORWARD_IF_TRUE] = "POP_JUMP_FORWARD_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -365,7 +367,7 @@ static const char *const _PyOpcode_OpName[267] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -385,9 +387,9 @@ static const char *const _PyOpcode_OpName[267] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -397,30 +399,32 @@ static const char *const _PyOpcode_OpName[267] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [RESUME_QUICK] = "RESUME_QUICK", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [RESUME_QUICK] = "RESUME_QUICK", [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [POP_JUMP_BACKWARD_IF_NOT_NONE] = "POP_JUMP_BACKWARD_IF_NOT_NONE", [POP_JUMP_BACKWARD_IF_NONE] = "POP_JUMP_BACKWARD_IF_NONE", [POP_JUMP_BACKWARD_IF_FALSE] = "POP_JUMP_BACKWARD_IF_FALSE", [POP_JUMP_BACKWARD_IF_TRUE] = "POP_JUMP_BACKWARD_IF_TRUE", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", @@ -429,8 +433,6 @@ static const char *const _PyOpcode_OpName[267] = { [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [185] = "<185>", - [186] = "<186>", [187] = "<187>", [188] = "<188>", [189] = "<189>", @@ -515,8 +517,6 @@ static const char *const _PyOpcode_OpName[267] = { #endif #define EXTRA_CASES \ - case 185: \ - case 186: \ case 187: \ case 188: \ case 189: \ diff --git a/Include/opcode.h b/Include/opcode.h index cf11e5560674e1..347987eb17e99b 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -16,6 +16,7 @@ extern "C" { #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define EXIT_INIT_CHECK 16 #define BINARY_SUBSCR 25 #define BINARY_SLICE 26 #define STORE_SLICE 27 @@ -142,71 +143,72 @@ extern "C" { #define BINARY_OP_MULTIPLY_FLOAT 8 #define BINARY_OP_MULTIPLY_INT 13 #define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_ADAPTIVE 17 -#define BINARY_SUBSCR_DICT 18 -#define BINARY_SUBSCR_GETITEM 19 -#define BINARY_SUBSCR_LIST_INT 20 -#define BINARY_SUBSCR_TUPLE_INT 21 -#define CALL_ADAPTIVE 22 -#define CALL_PY_EXACT_ARGS 23 -#define CALL_PY_WITH_DEFAULTS 24 -#define CALL_BOUND_METHOD_EXACT_ARGS 28 -#define CALL_BUILTIN_CLASS 29 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 -#define CALL_NO_KW_BUILTIN_FAST 39 -#define CALL_NO_KW_BUILTIN_O 40 -#define CALL_NO_KW_ISINSTANCE 41 -#define CALL_NO_KW_LEN 42 -#define CALL_NO_KW_LIST_APPEND 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 -#define CALL_NO_KW_STR_1 47 -#define CALL_NO_KW_TUPLE_1 48 -#define CALL_NO_KW_TYPE_1 56 -#define COMPARE_OP_ADAPTIVE 57 -#define COMPARE_OP_FLOAT_JUMP 58 -#define COMPARE_OP_INT_JUMP 59 -#define COMPARE_OP_STR_JUMP 62 -#define EXTENDED_ARG_QUICK 63 -#define FOR_ITER_ADAPTIVE 64 -#define FOR_ITER_LIST 65 -#define FOR_ITER_RANGE 66 -#define JUMP_BACKWARD_QUICK 67 -#define LOAD_ATTR_ADAPTIVE 72 -#define LOAD_ATTR_CLASS 73 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 -#define LOAD_ATTR_INSTANCE_VALUE 77 -#define LOAD_ATTR_MODULE 78 -#define LOAD_ATTR_PROPERTY 79 -#define LOAD_ATTR_SLOT 80 -#define LOAD_ATTR_WITH_HINT 81 -#define LOAD_ATTR_METHOD_LAZY_DICT 86 -#define LOAD_ATTR_METHOD_NO_DICT 113 -#define LOAD_ATTR_METHOD_WITH_DICT 121 -#define LOAD_ATTR_METHOD_WITH_VALUES 141 -#define LOAD_CONST__LOAD_FAST 143 -#define LOAD_FAST__LOAD_CONST 153 -#define LOAD_FAST__LOAD_FAST 154 -#define LOAD_GLOBAL_ADAPTIVE 158 -#define LOAD_GLOBAL_BUILTIN 159 -#define LOAD_GLOBAL_MODULE 160 -#define RESUME_QUICK 161 -#define STORE_ATTR_ADAPTIVE 166 -#define STORE_ATTR_INSTANCE_VALUE 167 -#define STORE_ATTR_SLOT 168 -#define STORE_ATTR_WITH_HINT 169 -#define STORE_FAST__LOAD_FAST 170 -#define STORE_FAST__STORE_FAST 177 -#define STORE_SUBSCR_ADAPTIVE 178 -#define STORE_SUBSCR_DICT 179 -#define STORE_SUBSCR_LIST_INT 180 -#define UNPACK_SEQUENCE_ADAPTIVE 181 -#define UNPACK_SEQUENCE_LIST 182 -#define UNPACK_SEQUENCE_TUPLE 183 -#define UNPACK_SEQUENCE_TWO_TUPLE 184 +#define BINARY_OP_SUBTRACT_INT 17 +#define BINARY_SUBSCR_ADAPTIVE 18 +#define BINARY_SUBSCR_DICT 19 +#define BINARY_SUBSCR_GETITEM 20 +#define BINARY_SUBSCR_LIST_INT 21 +#define BINARY_SUBSCR_TUPLE_INT 22 +#define CALL_ADAPTIVE 23 +#define CALL_PY_EXACT_ARGS 24 +#define CALL_PY_WITH_DEFAULTS 28 +#define CALL_BOUND_METHOD_EXACT_ARGS 29 +#define CALL_BUILTIN_CLASS 34 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 38 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 39 +#define CALL_NO_KW_BUILTIN_FAST 40 +#define CALL_NO_KW_BUILTIN_O 41 +#define CALL_NO_KW_ISINSTANCE 42 +#define CALL_NO_KW_LEN 43 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 44 +#define CALL_NO_KW_LIST_APPEND 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 46 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 47 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 48 +#define CALL_NO_KW_STR_1 56 +#define CALL_NO_KW_TUPLE_1 57 +#define CALL_NO_KW_TYPE_1 58 +#define COMPARE_OP_ADAPTIVE 59 +#define COMPARE_OP_FLOAT_JUMP 62 +#define COMPARE_OP_INT_JUMP 63 +#define COMPARE_OP_STR_JUMP 64 +#define EXTENDED_ARG_QUICK 65 +#define FOR_ITER_ADAPTIVE 66 +#define FOR_ITER_LIST 67 +#define FOR_ITER_RANGE 72 +#define JUMP_BACKWARD_QUICK 73 +#define LOAD_ATTR_ADAPTIVE 76 +#define LOAD_ATTR_CLASS 77 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 +#define LOAD_ATTR_INSTANCE_VALUE 79 +#define LOAD_ATTR_MODULE 80 +#define LOAD_ATTR_PROPERTY 81 +#define LOAD_ATTR_SLOT 86 +#define LOAD_ATTR_WITH_HINT 113 +#define LOAD_ATTR_METHOD_LAZY_DICT 121 +#define LOAD_ATTR_METHOD_NO_DICT 141 +#define LOAD_ATTR_METHOD_WITH_DICT 143 +#define LOAD_ATTR_METHOD_WITH_VALUES 153 +#define LOAD_CONST__LOAD_FAST 154 +#define LOAD_FAST__LOAD_CONST 158 +#define LOAD_FAST__LOAD_FAST 159 +#define LOAD_GLOBAL_ADAPTIVE 160 +#define LOAD_GLOBAL_BUILTIN 161 +#define LOAD_GLOBAL_MODULE 166 +#define RESUME_QUICK 167 +#define STORE_ATTR_ADAPTIVE 168 +#define STORE_ATTR_INSTANCE_VALUE 169 +#define STORE_ATTR_SLOT 170 +#define STORE_ATTR_WITH_HINT 177 +#define STORE_FAST__LOAD_FAST 178 +#define STORE_FAST__STORE_FAST 179 +#define STORE_SUBSCR_ADAPTIVE 180 +#define STORE_SUBSCR_DICT 181 +#define STORE_SUBSCR_LIST_INT 182 +#define UNPACK_SEQUENCE_ADAPTIVE 183 +#define UNPACK_SEQUENCE_LIST 184 +#define UNPACK_SEQUENCE_TUPLE 185 +#define UNPACK_SEQUENCE_TWO_TUPLE 186 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 52c1271868e3ab..15d79b513b4254 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -84,6 +84,7 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_NOT', 12) def_op('UNARY_INVERT', 15) +def_op('EXIT_INIT_CHECK', 16) def_op('BINARY_SUBSCR', 25) def_op('BINARY_SLICE', 26) @@ -310,6 +311,7 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_BUILTIN_O", "CALL_NO_KW_ISINSTANCE", "CALL_NO_KW_LEN", + "CALL_NO_KW_ALLOC_AND_ENTER_INIT", "CALL_NO_KW_LIST_APPEND", "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 202fb30a8a7e9d..896153aa9257cf 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1526,7 +1526,7 @@ def delx(self): del self.__x '10P' # PySequenceMethods '2P' # PyBufferProcs '6P' - '1P' # Specializer cache + '2P' # Specializer cache ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 9f1aa81dbcd249..67bdd585927dc8 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1571,6 +1571,29 @@ def func(): self.run_and_compare(func, EXPECTED_EVENTS) + def test_correct_tracing_quickened_call_class_init(self): + + class C: + def __init__(self): + self + + def func(): + C() + + EXPECTED_EVENTS = [ + (0, 'call'), + (1, 'line'), + (-3, 'call'), + (-2, 'line'), + (-2, 'return'), + (1, 'return')] + + self.run_and_compare(func, EXPECTED_EVENTS) + # Quicken + for _ in range(100): + func() + self.run_and_compare(func, EXPECTED_EVENTS) + def test_very_large_function(self): # There is a separate code path when the number of lines > (1 << 15). d = {} diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst new file mode 100644 index 00000000000000..5633097f4a3fdd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-05-11-12-00.bpo-44525.4E3Pwn.rst @@ -0,0 +1,11 @@ +Specializes calls to most Python classes. Specifically, any class that +inherits from ``object``, or another Python class, and does not override +``__new__``. + +The specialized instruction does the following: + +1. Creates the object (by calling ``object.__new__``) +2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) +3. Pushes the frame for ``__init__`` to the frame stack + +Speeds up the instantiation of most Python classes. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 0cb95d52360ef1..1737c908571d17 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5307,11 +5307,10 @@ _PyDict_NewKeysForClass(void) #define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) -static int -init_inline_values(PyObject *obj, PyTypeObject *tp) +int +_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - // assert(type->tp_dictoffset > 0); -- TO DO Update this assert. assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); @@ -5343,7 +5342,7 @@ _PyObject_InitializeDict(PyObject *obj) } if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { OBJECT_STAT_INC(new_values); - return init_inline_values(obj, tp); + return _PyObject_InitInlineValues(obj, tp); } PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d2647bd122888c..a0cb0bb77975db 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1052,7 +1052,7 @@ init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) Py_INCREF(func); Py_XINCREF(locals); PyCodeObject *code = (PyCodeObject *)func->func_code; - _PyFrame_InitializeSpecials(frame, func, locals, code); + _PyFrame_InitializeSpecialsFromFunction(frame, func, locals, code); for (Py_ssize_t i = 0; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = NULL; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 40d42947dbcfd4..5f403780909bc6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1221,6 +1221,26 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) return obj; } +PyObject * +_PyType_NewManagedObject(PyTypeObject *type) +{ + assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyType_IS_GC(type)); + assert(type->tp_new == PyBaseObject_Type.tp_new); + assert(type->tp_alloc == PyType_GenericAlloc); + assert(type->tp_itemsize == 0); + PyObject *obj = PyType_GenericAlloc(type, 0); + if (obj == NULL) { + return PyErr_NoMemory(); + } + _PyObject_DictOrValuesPointer(obj)->dict = NULL; + if (_PyObject_InitInlineValues(obj, type)) { + Py_DECREF(obj); + return NULL; + } + return obj; +} + PyObject * _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) { diff --git a/Python/ceval.c b/Python/ceval.c index b3a0a3640eb97d..d912e486976c56 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -869,7 +869,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* Shared opcode macros */ #define TRACE_FUNCTION_EXIT() \ - if (cframe.use_tracing) { \ + if (cframe.use_tracing && INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) { \ if (trace_function_exit(tstate, frame, retval)) { \ Py_DECREF(retval); \ goto exit_unwind; \ @@ -1088,11 +1088,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef LLTRACE { - int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); - if (r < 0) { - goto exit_unwind; + if (PyFunction_Check(frame->f_funcobj)) { + int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); + if (r < 0) { + goto exit_unwind; + } + lltrace = r; } - lltrace = r; } if (lltrace) { lltrace_resume_frame(frame); @@ -4449,6 +4451,76 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); } + TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + assert(call_shape.kwnames == NULL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + DEOPT_IF(is_method(stack_pointer, oparg), CALL); + PyObject *callable = PEEK(oparg+1); + DEOPT_IF(!PyType_Check(callable), CALL); + PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; + DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); + PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); + STAT_INC(CALL, hit); + DEOPT_IF(code->co_argcount != oparg+1, CALL); + PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + if (self == NULL) { + goto error; + } + PEEK(oparg+1) = self; + Py_DECREF(tp); + CALL_STAT_INC(inlined_py_calls); + if (_Py_EnterRecursiveCallTstate(tstate, "")) { + tstate->recursion_remaining--; + goto exit_unwind; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, interp->callable_cache.init_cleanup, 1, 1); + assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + shim->previous = frame; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init); + /* Copy self followed by args to __init__ frame */ + STACK_SHRINK(oparg+1); + for (int i = 0; i <= oparg; i++) { + init_frame->localsplus[i] = stack_pointer[i]; + } + STACK_SHRINK(1); // Clear the NULL. + _PyFrame_SetStackPointer(frame, stack_pointer); + for (int i = oparg+1; i < code->co_nlocalsplus; i++) { + init_frame->localsplus[i] = NULL; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + init_frame->previous = shim; + frame = cframe.current_frame = init_frame; + goto start_frame; + } + + TARGET(EXIT_INIT_CHECK) { + assert(STACK_LEVEL() == 2); + PyObject *should_be_none = TOP(); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + STACK_SHRINK(1); + _Py_DECREF_NO_DEALLOC(Py_None); + NOTRACE_DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -5225,9 +5297,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Log traceback info. */ - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } } if (tstate->c_tracefunc != NULL) { @@ -5832,7 +5906,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, if (frame == NULL) { goto fail; } - _PyFrame_InitializeSpecials(frame, func, locals, code); + _PyFrame_InitializeSpecialsFromFunction(frame, func, locals, code); PyObject **localsarray = &frame->localsplus[0]; for (int i = 0; i < code->co_nlocalsplus; i++) { localsarray[i] = NULL; diff --git a/Python/compile.c b/Python/compile.c index 627f86a8ce9188..d497c729e32c10 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1163,6 +1163,9 @@ stack_effect(int opcode, int oparg, int jump) case LOAD_GLOBAL: return (oparg & 1) + 1; + case EXIT_INIT_CHECK: + return -1; + /* Exception handling pseudo-instructions */ case SETUP_FINALLY: /* 0 in the normal flow. diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 7c782d101c1b8c..da0a63e85ce287 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -15,6 +15,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_EXIT_INIT_CHECK, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_ADAPTIVE, &&TARGET_BINARY_SUBSCR_DICT, @@ -23,31 +24,30 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_ADAPTIVE, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, + &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_NO_KW_LIST_APPEND, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,37 +55,37 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_COMPARE_OP_ADAPTIVE, - &&TARGET_COMPARE_OP_FLOAT_JUMP, - &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_COMPARE_OP_FLOAT_JUMP, + &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_EXTENDED_ARG_QUICK, &&TARGET_FOR_ITER_ADAPTIVE, &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_RANGE, - &&TARGET_JUMP_BACKWARD_QUICK, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_FOR_ITER_RANGE, + &&TARGET_JUMP_BACKWARD_QUICK, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_POP_JUMP_FORWARD_IF_FALSE, &&TARGET_POP_JUMP_FORWARD_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,30 +152,32 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_RESUME_QUICK, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_RESUME_QUICK, &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_POP_JUMP_BACKWARD_IF_NOT_NONE, &&TARGET_POP_JUMP_BACKWARD_IF_NONE, &&TARGET_POP_JUMP_BACKWARD_IF_FALSE, &&TARGET_POP_JUMP_BACKWARD_IF_TRUE, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, @@ -252,7 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bb646f1a0ee2d0..ec1971258fd1d1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -787,6 +787,7 @@ pycore_init_builtins(PyThreadState *tstate) PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); assert(list_append); interp->callable_cache.list_append = list_append; + interp->callable_cache.init_cleanup = NULL; PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); assert(object__getattribute__); interp->callable_cache.object__getattribute__ = object__getattribute__; diff --git a/Python/pystate.c b/Python/pystate.c index a11f1622ecd4ab..742cc4e3ac61ee 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -450,6 +450,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + Py_CLEAR(interp->callable_cache.init_cleanup); // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's diff --git a/Python/specialize.c b/Python/specialize.c index 8a2f9054cad5c7..0e8631f192c849 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -9,6 +9,8 @@ #include "pycore_opcode.h" // _PyOpcode_Caches #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX #include "pycore_descrobject.h" +#include "pycore_function.h" // _PyFunction_FromConstructor() + #include // rand() @@ -458,6 +460,7 @@ miss_counter_start(void) { #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27 #define SPEC_FAIL_CALL_PYFUNCTION 28 #define SPEC_FAIL_CALL_PEP_523 29 +#define SPEC_FAIL_INIT_NOT_SIMPLE 30 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -1475,12 +1478,64 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins return 0; } + +static PyFunctionObject * +get_init_for_simple_managed_python_class(PyTypeObject *tp) +{ + if (tp->tp_new != PyBaseObject_Type.tp_new) { + return NULL; + } + if (tp->tp_alloc != PyType_GenericAlloc) { + return NULL; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return NULL; + } + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + return NULL; + } + PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); + if (init == NULL || !PyFunction_Check(init)) { + return NULL; + } + int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); + if (kind != SIMPLE_FUNCTION) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_INIT_NOT_SIMPLE); + return NULL; + } + Py_CLEAR(((PyHeapTypeObject *)tp)->_spec_cache.init); + ((PyHeapTypeObject *)tp)->_spec_cache.init = init; + return (PyFunctionObject *)init; +} + static int -specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, - PyObject *kwnames) +setup_init_cleanup_code(void); + +static int +specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, + int nargs, PyObject *kwnames) { assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); - PyTypeObject *tp = _PyType_CAST(callable); + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + if (setup_init_cleanup_code()) { + return -1; + } + assert(PyType_Check(callable)); + PyTypeObject *tp = (PyTypeObject *)callable; + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } if (tp->tp_new == PyBaseObject_Type.tp_new) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); return -1; @@ -1742,9 +1797,6 @@ call_fail_kind(PyObject *callable) #endif -/* TODO: - - Specialize calling classes. -*/ int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) @@ -2070,6 +2122,85 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, cache->counter = miss_counter_start(); } +/* Code init cleanup. + * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up + * the frame to execute the EXIT_INIT_CHECK + * instruction. + * Starts with an assertion error, in case it is called + * directly. + * Ends with a RESUME so that it is not traced. + * This is used as a plain code object, not a function, + * so must not access globals or builtins. + */ +char INIT_CLEANUP_CODE[10] = { + LOAD_ASSERTION_ERROR, 0, + RAISE_VARARGS, 1, + EXIT_INIT_CHECK, 0, + RETURN_VALUE, 0, + RESUME, 0 +}; + +static int +setup_init_cleanup_code(void) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->callable_cache.init_cleanup != NULL) { + return 0; + } + PyObject *name = NULL; + PyObject *code = NULL; + PyObject *lines = NULL; + PyCodeObject *codeobj = NULL; + PyObject *globals = NULL; + + name = _PyUnicode_FromASCII("type.__call__", strlen("type.__call__")); + if (name == NULL) { + goto cleanup; + } + code = PyBytes_FromStringAndSize(INIT_CLEANUP_CODE, sizeof(INIT_CLEANUP_CODE)); + if (code == NULL) { + goto cleanup; + } + const char loc[2] = { 0x80 | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3) | 3, 0 }; + lines = PyBytes_FromStringAndSize(loc, 2); + if (lines == NULL) { + goto cleanup; + } + struct _PyCodeConstructor con = { + .filename = &_Py_STR(empty), + .name = name, + .qualname = name, + .flags = CO_NEWLOCALS | CO_OPTIMIZED, + + .code = code, + .firstlineno = 1, + .linetable = lines, + + .consts = (PyObject *)&_Py_SINGLETON(tuple_empty), + .names = (PyObject *)&_Py_SINGLETON(tuple_empty), + + .localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), + .localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), + + .argcount = 0, + .posonlyargcount = 0, + .kwonlyargcount = 0, + + .stacksize = 2, + + .exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), + }; + + codeobj = _PyCode_New(&con); + interp->callable_cache.init_cleanup = codeobj; +cleanup: + PyErr_Clear(); + Py_XDECREF(globals); + Py_XDECREF(name); + Py_XDECREF(code); + Py_XDECREF(lines); + return interp->callable_cache.init_cleanup == NULL ? -1 : 0; +} + #ifdef Py_STATS static int unpack_sequence_fail_kind(PyObject *seq) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 2e8261a4755b99..c1460fcfc83dd8 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -68,7 +68,7 @@ def print_specialization_stats(name, family_stats, defines): rows.append((label, val, f"{100*val/total_attempts:0.1f}%")) emit_table(("", "Count:", "Ratio:"), rows) total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [ 0 ] * 30 + failure_kinds = [ 0 ] * 32 for key in family_stats: if not key.startswith("specialization.failure_kind"): continue