From 4b7da22b6949c6021a093c6bf62232c0ca3e7d31 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Tue, 22 Apr 2025 16:16:08 +0200 Subject: [PATCH 1/8] Pass zend_jit_ctx to zend_jit_trace_get_exit_point() --- ext/opcache/jit/zend_jit.c | 3 +- ext/opcache/jit/zend_jit_internal.h | 2 + ext/opcache/jit/zend_jit_ir.c | 124 ++++++++++++++-------------- ext/opcache/jit/zend_jit_trace.c | 32 +++---- 4 files changed, 82 insertions(+), 79 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 355178d9d51ed..a1a0eac77b896 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -102,7 +102,8 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLE static int zend_jit_trace_op_len(const zend_op *opline); static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline); -static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags); + +static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *ctx, const zend_op *to_opline, uint32_t flags); static const void *zend_jit_trace_get_exit_addr(uint32_t n); static void zend_jit_trace_add_code(const void *start, uint32_t size); static zend_string *zend_jit_func_name(const zend_op_array *op_array); diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 3ec8ae6041a53..a9f59e247a527 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -27,6 +27,8 @@ #include "Zend/Optimizer/zend_func_info.h" #include "Zend/Optimizer/zend_call_graph.h" +typedef struct _zend_jit_ctx zend_jit_ctx; + /* Address Encoding */ typedef uintptr_t zend_jit_addr; diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 60fcdff8a61f3..c77cb93aadc3d 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -4797,7 +4797,7 @@ static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t o } } - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); @@ -4831,7 +4831,7 @@ static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t o SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); } } - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { @@ -5091,17 +5091,17 @@ static int zend_jit_math_long_long(zend_jit_ctx *jit, old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } else if (opline->opcode == ZEND_SUB && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } else { - exit_point = zend_jit_trace_get_exit_point(opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); } exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -5111,7 +5111,7 @@ static int zend_jit_math_long_long(zend_jit_ctx *jit, ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); may_overflow = 0; } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -6404,7 +6404,7 @@ static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit, if (val_info & MAY_BE_UNDEF) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -8037,7 +8037,7 @@ static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit, ir_IF_FALSE_cold(if_type); SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + exit_point = zend_jit_trace_get_exit_point(jit, opline+1, flags); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; @@ -8049,7 +8049,7 @@ static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit, } SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + exit_point = zend_jit_trace_get_exit_point(jit, opline+1, flags); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; @@ -8422,7 +8422,7 @@ typedef struct _zend_closure { static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -8526,7 +8526,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL); - int32_t exit_point = zend_jit_trace_get_exit_point(opline, + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, may_be_trampoline ? (ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -8761,7 +8761,7 @@ static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const ze return 0; } - exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); + exit_point = zend_jit_trace_get_exit_point(jit, to_opline, ZEND_JIT_EXIT_POLYMORPHISM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -8786,7 +8786,7 @@ static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_ ir_ref func_ref = IR_UNUSED; if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { return 0; } } @@ -8873,7 +8873,7 @@ static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_ ZEND_UNREACHABLE(); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -8993,7 +8993,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -9104,7 +9104,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, int32_t exit_point; const void *exit_addr; - exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); + exit_point = zend_jit_trace_get_exit_point(jit, opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9274,7 +9274,7 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, int32_t exit_point; const void *exit_addr; - exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline, func ? ZEND_JIT_EXIT_INVALIDATE : 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9382,7 +9382,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -9412,7 +9412,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, func = (zend_function*)trace->func; opcodes = func->op_array.opcodes; - exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); + exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_CLOSURE_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9486,7 +9486,7 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t ir_CONST_U32(mask)); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9679,7 +9679,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen /* Don't generate code that always throws exception */ return 0; } else { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9727,7 +9727,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen ir_IF_FALSE(if_prefer_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9815,7 +9815,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen ir_IF_FALSE(if_ref); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -10100,7 +10100,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (trace) { - uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + uint32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -10423,7 +10423,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen jit_observer_fcall_begin(jit, rx, observer_handler); if (trace) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -10702,7 +10702,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (!zend_interrupt_function) { // TODO: Can we avoid checking for interrupts after each call ??? if (trace && jit->last_valid_opline != opline) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -10846,7 +10846,7 @@ static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op if (!JIT_G(current_frame) || TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -11946,7 +11946,7 @@ static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit, if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R && !exit_addr) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -11967,7 +11967,7 @@ static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit, ir_IF_TRUE(if_type); } if (op1_info & MAY_BE_PACKED_GUARD) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -12543,7 +12543,7 @@ static int zend_jit_fetch_dim_read(zend_jit_ctx *jit, if (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && !has_concrete_type(op1_info)) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -12597,7 +12597,7 @@ static int zend_jit_fetch_dim_read(zend_jit_ctx *jit, old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + exit_point = zend_jit_trace_get_exit_point(jit, opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { @@ -13376,7 +13376,7 @@ static int zend_jit_assign_dim(zend_jit_ctx *jit, } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -13603,7 +13603,7 @@ static int zend_jit_assign_dim_op(zend_jit_ctx *jit, && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { return 0; @@ -13775,7 +13775,7 @@ static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); ir_ref ref; @@ -14153,7 +14153,7 @@ static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const z !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14189,7 +14189,7 @@ static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const z static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14254,7 +14254,7 @@ static int zend_jit_fetch_obj(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14474,7 +14474,7 @@ static int zend_jit_fetch_obj(zend_jit_ctx *jit, if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14795,7 +14795,7 @@ static int zend_jit_assign_obj(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14933,7 +14933,7 @@ static int zend_jit_assign_obj(zend_jit_ctx *jit, } // Undefined property with potential magic __get()/__set() or lazy object if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15140,7 +15140,7 @@ static int zend_jit_assign_obj_op(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15254,7 +15254,7 @@ static int zend_jit_assign_obj_op(zend_jit_ctx *jit, if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15336,7 +15336,7 @@ static int zend_jit_assign_obj_op(zend_jit_ctx *jit, uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; if (use_prop_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -15575,7 +15575,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15677,7 +15677,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15758,7 +15758,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, ir_ref if_overflow = IR_UNUSED; if (use_prop_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -15932,7 +15932,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, const void *exit_addr; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16219,7 +16219,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (next_opline) { if (next_opline != default_opline) { - exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; @@ -16233,7 +16233,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; @@ -16315,7 +16315,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (target == next_opline) { ir_END_list(continue_list); } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_point = zend_jit_trace_get_exit_point(jit, target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16360,7 +16360,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; @@ -16440,7 +16440,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (target == next_opline) { ir_END_list(continue_list); } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_point = zend_jit_trace_get_exit_point(jit, target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16576,7 +16576,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (target == next_opline) { ir_END_list(continue_list); } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_point = zend_jit_trace_get_exit_point(jit, target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16853,7 +16853,7 @@ static const void *zend_jit_trace_allocate_exit_group(uint32_t n) static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); @@ -16867,7 +16867,7 @@ static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_ static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); @@ -16881,7 +16881,7 @@ static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr) { - uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -16894,7 +16894,7 @@ static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_ static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline) { - uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(jit, NULL, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -16918,7 +16918,7 @@ static bool zend_jit_guard_reference(zend_jit_ctx *jit, ir_ref ref; if (add_ref_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -16952,7 +16952,7 @@ static bool zend_jit_fetch_reference(zend_jit_ctx *jit, ir_ref ref; if (add_ref_guard || add_type_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -17013,7 +17013,7 @@ static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline ir_ref ref = IR_UNUSED; if (add_indirect_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -17036,7 +17036,7 @@ static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -17194,7 +17194,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 68c096d9d3df2..02b9f92d36b40 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -131,7 +131,7 @@ static uint32_t zend_jit_exit_point_by_addr(const void *addr) return (uint32_t)-1; } -static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags) +static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *jit, const zend_op *to_opline, uint32_t flags) { zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; uint32_t exit_point; @@ -4362,7 +4362,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { // if (ra && dzend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) { -// uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); +// uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, opline, ZEND_JIT_EXIT_TO_VM); // // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); // if (!timeout_exit_addr) { @@ -5411,7 +5411,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5459,7 +5459,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5494,7 +5494,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { bool exit_if_true = 0; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); - uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -5524,7 +5524,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5669,14 +5669,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; } } else { - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5716,7 +5716,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } else { ZEND_UNREACHABLE(); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5748,7 +5748,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { bool exit_if_true = 0; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); - uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -5778,7 +5778,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { bool exit_if_true = 0; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); - uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -5954,10 +5954,10 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info); } else { - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); } exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -6298,7 +6298,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } else { ZEND_UNREACHABLE(); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -7196,7 +7196,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par || (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) { /* Deoptimize to the first instruction of the loop */ - uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM); timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!timeout_exit_addr) { @@ -7267,7 +7267,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } - exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM); + exit_point = zend_jit_trace_get_exit_point(&ctx, zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM); timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!timeout_exit_addr) { goto jit_failure; From 9b6c36e1f4b33b0460be00d68cd0f1076f584c85 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Tue, 22 Apr 2025 16:16:28 +0200 Subject: [PATCH 2/8] JIT: Don't reuse IP register for EX(call) --- ext/opcache/jit/zend_jit.c | 35 ++-- ext/opcache/jit/zend_jit_internal.h | 16 +- ext/opcache/jit/zend_jit_ir.c | 273 ++++++++++++-------------- ext/opcache/jit/zend_jit_trace.c | 60 +++--- ext/opcache/jit/zend_jit_vm_helpers.c | 13 +- 5 files changed, 191 insertions(+), 206 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index a1a0eac77b896..4fb373ebed1c7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -803,6 +803,9 @@ static bool zend_jit_may_be_modified(const zend_function *func, const zend_op_ar # pragma clang diagnostic ignored "-Wstring-compare" #endif +static bool zend_jit_inc_call_level(uint8_t opcode); +static bool zend_jit_dec_call_level(uint8_t opcode); + #include "jit/zend_jit_ir.c" #if defined(__clang__) @@ -1609,6 +1612,18 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op call_level++; } +#if ZEND_DEBUG && 0 + { + const void *handler; + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + handler = zend_get_opcode_handler_func(opline); + } else { + handler = opline->handler; + } + ir_RSTORE(8, jit_CONST_FUNC(&ctx, (uintptr_t)handler, IR_FASTCALL_FUNC)); + } +#endif + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { switch (opline->opcode) { case ZEND_PRE_INC: @@ -1676,10 +1691,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op && zend_jit_next_is_send_result(opline)) { i++; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = -1; @@ -1730,10 +1742,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op && zend_jit_next_is_send_result(opline)) { i++; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = -1; @@ -1786,10 +1795,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if ((i + 1) <= end && zend_jit_next_is_send_result(opline)) { i++; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } if (!zend_jit_concat(&ctx, opline, op1_info, op2_info, res_addr, @@ -2040,10 +2046,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op && zend_jit_next_is_send_result(opline) && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) { i++; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } } if (!zend_jit_assign(&ctx, opline, diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index a9f59e247a527..539f75b3ae9f3 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -64,6 +64,14 @@ typedef uintptr_t zend_jit_addr; #define Z_SSA_VAR(addr) ((addr)>>_ZEND_ADDR_REG_SHIFT) #define Z_IR_REF(addr) ((addr)>>_ZEND_ADDR_REG_SHIFT) +#define ZEND_ADDR_ARG(call, var) ZEND_ADDR_REF_ZVAL(ir_ADD_OFFSET(call, var)) + +#define Z_IS_ARG_ADDR(addr) \ + (Z_MODE(addr) == IS_REF_ZVAL && ( \ + Z_IR_REF(addr) == jit->call \ + || (jit->ctx.ir_base[Z_IR_REF(addr)].op == IR_ADD \ + && jit->ctx.ir_base[Z_IR_REF(addr)].op1 == jit->call))) + #define Z_STORE(addr) \ ((jit->ra && jit->ra[Z_SSA_VAR(addr)].ref) ? \ (jit->ra[Z_SSA_VAR(addr)].flags & ZREG_STORE) : \ @@ -242,9 +250,9 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_H ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper_no_skip_recv(ZEND_OPCODE_HANDLER_ARGS); -bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D); -bool ZEND_FASTCALL zend_jit_nodiscard_helper(OPLINE_D); -bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(OPLINE_D); +bool ZEND_FASTCALL zend_jit_deprecated_helper(zend_execute_data *call); +bool ZEND_FASTCALL zend_jit_nodiscard_helper(zend_execute_data *call); +bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(zend_execute_data *call); void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D); void ZEND_FASTCALL zend_jit_undefined_long_key_ex(zend_long key EXECUTE_DATA_DC); void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D); @@ -448,6 +456,8 @@ typedef struct _zend_jit_trace_exit_info { int32_t poly_this_ref; int8_t poly_func_reg; int8_t poly_this_reg; + int32_t call_ref; + int8_t call_reg; } zend_jit_trace_exit_info; typedef struct _zend_jit_trace_stack { diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index c77cb93aadc3d..4db072be3f141 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -18,6 +18,7 @@ #include "jit/ir/ir.h" #include "jit/ir/ir_builder.h" +#include "jit/zend_jit_internal.h" #if defined(IR_TARGET_X86) # define IR_REG_SP 4 /* IR_REG_RSP */ @@ -55,8 +56,6 @@ # error "Unknown IR target" #endif -#define ZREG_RX ZREG_IP - #define OPTIMIZE_FOR_SIZE 0 /* IR builder defines */ @@ -209,9 +208,6 @@ static size_t tsrm_tls_offset = 0; #define jit_EX(_field) \ jit_CALL(jit_FP(jit), _field) -#define jit_RX(_field) \ - jit_CALL(jit_IP(jit), _field) - #define JIT_STUBS(_) \ _(exception_handler, IR_SKIP_PROLOGUE) \ _(exception_handler_undef, IR_SKIP_PROLOGUE) \ @@ -270,13 +266,13 @@ typedef struct _zend_jit_ctx { const zend_op *last_valid_opline; bool use_last_valid_opline; bool track_last_valid_opline; - bool reuse_ip; uint32_t delayed_call_level; int b; /* current basic block number or -1 */ #ifdef ZTS ir_ref tls; #endif ir_ref fp; + ir_ref call; /* EX(call) cache */ ir_ref trace_loop_ref; ir_ref return_inputs; const zend_op_array *op_array; @@ -622,6 +618,7 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) /* Check if we need snapshot entries for polymorphic method call */ zend_jit_trace_info *t = jit->trace; uint32_t exit_point = 0, n = 0; + bool save_call = false; if (addr < 0) { if (t->exit_count > 0 @@ -630,10 +627,13 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) { n = 2; } + if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + save_call = true; + } } } - if (stack_size || n) { + if (stack_size || n || save_call) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t snapshot_size, i; @@ -647,10 +647,10 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) break; } } - if (snapshot_size || n) { + if (snapshot_size || n || save_call) { ir_ref snapshot; - snapshot = ir_SNAPSHOT(snapshot_size + n); + snapshot = ir_SNAPSHOT(snapshot_size + n + save_call); for (i = 0; i < snapshot_size; i++) { ir_ref ref = STACK_REF(stack, i); @@ -663,6 +663,10 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref); ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref); } + if (save_call) { + ZEND_ASSERT(t->exit_info[exit_point].call_ref != IR_UNUSED); + ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + n + 1, t->exit_info[exit_point].call_ref); + } } } } @@ -721,6 +725,20 @@ void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snaps ZEND_ASSERT(exit_point < t->exit_count); exit_flags = t->exit_info[exit_point].flags; + if (exit_flags & ZEND_JIT_EXIT_RESTORE_CALL) { + int8_t *reg_ops = ctx->regs[snapshot_ref]; + + ZEND_ASSERT(reg_ops[n] != -1); + if ((exit_flags & ZEND_JIT_EXIT_FIXED) + && (t->exit_info[exit_point].call_reg != reg_ops[n])) { + exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); + addr = (void*)zend_jit_trace_get_exit_addr(exit_point); + exit_flags &= ~ZEND_JIT_EXIT_FIXED; + } + t->exit_info[exit_point].call_reg = reg_ops[n]; + n--; + } + if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) { int8_t *reg_ops = ctx->regs[snapshot_ref]; @@ -937,10 +955,8 @@ static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit) static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline) { - if (!jit->reuse_ip) { - jit->track_last_valid_opline = 1; - jit->last_valid_opline = opline; - } + jit->track_last_valid_opline = 1; + jit->last_valid_opline = opline; } static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit) @@ -949,28 +965,16 @@ static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit) jit->last_valid_opline = NULL; } -static void zend_jit_start_reuse_ip(zend_jit_ctx *jit) +static ir_ref jit_EX_CALL(zend_jit_ctx *jit) { - zend_jit_reset_last_valid_opline(jit); - jit->reuse_ip = 1; -} - -static int zend_jit_reuse_ip(zend_jit_ctx *jit) -{ - if (!jit->reuse_ip) { - zend_jit_start_reuse_ip(jit); - // RX = EX(call); - jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); + if (!jit->call) { + jit->call = ir_LOAD_A(jit_EX(call)); } - return 1; -} -static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit) -{ - jit->reuse_ip = 0; + return jit->call; } -static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) +static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level, int8_t call_reg) { ir_ref rx, call; @@ -982,7 +986,14 @@ static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) call = ir_LOAD_A(jit_EX(call)); } - rx = jit_IP(jit); + if (call_reg != ZREG_NONE) { + /* In deoptimization */ + rx = ir_RLOAD_A(call_reg); + jit->call = rx; + } else { + ZEND_ASSERT(jit->call != IR_UNUSED || call_level == 1); + rx = jit_EX_CALL(jit); + } // JIT: call->prev_execute_data = call; ir_STORE(jit_CALL(rx, prev_execute_data), call); @@ -1001,7 +1012,7 @@ static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) ir_ref ref; if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { return 0; } } @@ -1020,7 +1031,6 @@ static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) } else { jit_STORE_IP(jit, ir_CONST_ADDR(target)); } - jit->reuse_ip = 0; zend_jit_set_last_valid_opline(jit, target); return 1; } @@ -1043,8 +1053,6 @@ static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr) if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1072,8 +1080,6 @@ static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr) ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1098,8 +1104,6 @@ static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr) ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1124,8 +1128,6 @@ static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr) ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1148,8 +1150,6 @@ static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -2701,7 +2701,6 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) jit->last_valid_opline = NULL; jit->use_last_valid_opline = 0; jit->track_last_valid_opline = 0; - jit->reuse_ip = 0; jit->delayed_call_level = 0; delayed_call_chain = 0; jit->b = -1; @@ -2709,6 +2708,7 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) jit->tls = IR_UNUSED; #endif jit->fp = IR_UNUSED; + jit->call = IR_UNUSED; jit->trace_loop_ref = IR_UNUSED; jit->return_inputs = IR_UNUSED; jit->bb_start_ref = NULL; @@ -4180,6 +4180,15 @@ static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_th zend_jit_set_last_valid_opline(jit, opline + 1); break; } + + /* Op handler invalidates jit->call by changing EX(call) */ + if (zend_jit_inc_call_level(opline->opcode) + || zend_jit_dec_call_level(opline->opcode) + || opline->opcode == ZEND_SEND_UNPACK + || opline->opcode == ZEND_SEND_ARRAY) { + jit->call = IR_UNUSED; + } + return 1; } @@ -4254,6 +4263,7 @@ static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline) jit->b = -1; zend_jit_reset_last_valid_opline(jit); } + jit->call = IR_UNUSED; return 1; } @@ -5591,7 +5601,7 @@ static int zend_jit_math_helper(zend_jit_ctx *jit, if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); - } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { + } else if (Z_IS_ARG_ADDR(res_addr)) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); @@ -5976,7 +5986,7 @@ static int zend_jit_long_math_helper(zend_jit_ctx *jit, if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); - } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { + } else if (Z_IS_ARG_ADDR(res_addr)) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); @@ -6103,7 +6113,7 @@ static int zend_jit_concat_helper(zend_jit_ctx *jit, if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); - } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { + } else if (Z_IS_ARG_ADDR(res_addr)) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); @@ -7952,7 +7962,7 @@ static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t sm return 1; } -static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg) +static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg, uint8_t call_reg) { zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr)); @@ -7960,7 +7970,7 @@ static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, ir_IF_FALSE_cold(if_def); if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { - if (!zend_jit_save_call_chain(jit, -1)) { + if (!zend_jit_save_call_chain(jit, -1, call_reg)) { return 0; } } @@ -8459,7 +8469,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co uint32_t used_stack; ir_ref used_stack_ref = IR_UNUSED; bool stack_check = 1; - ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED; + ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED, cold_call = IR_UNUSED; ZEND_ASSERT(func_ref != IR_NULL); if (func) { @@ -8511,17 +8521,15 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref); } - zend_jit_start_reuse_ip(jit); - // JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top))); + rx = ir_LOAD_A(jit_EG(vm_stack_top)); if (stack_check) { // JIT: Check Stack Overflow ref = ir_UGE( ir_SUB_A( ir_LOAD_A(jit_EG(vm_stack_end)), - jit_IP(jit)), + rx), used_stack_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { @@ -8551,7 +8559,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif jit_SET_EX_OPLINE(jit, opline); - ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref); + cold_call = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref); } else { if (!is_closure) { ref = func_ref; @@ -8559,10 +8567,9 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)); } jit_SET_EX_OPLINE(jit, opline); - ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper), + cold_call = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper), used_stack_ref, ref); } - jit_STORE_IP(jit, ref); cold_path = ir_END(); ir_IF_TRUE(if_enough_stack); @@ -8570,7 +8577,6 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co } ref = jit_EG(vm_stack_top); - rx = jit_IP(jit); #if !OPTIMIZE_FOR_SIZE /* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack); * This vesions is longer but faster @@ -8601,7 +8607,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co #endif if (cold_path) { ir_MERGE_WITH(cold_path); - rx = jit_IP(jit); + rx = ir_PHI_2(IR_ADDR, rx, cold_call); } // JIT: call->func = func; @@ -8616,7 +8622,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co } if (cold_path) { ir_MERGE_WITH(cold_path); - rx = jit_IP(jit); + rx = ir_PHI_2(IR_ADDR, rx, cold_call); } } if (opline->opcode == ZEND_INIT_METHOD_CALL) { @@ -8714,6 +8720,8 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co // JIT: ZEND_CALL_NUM_ARGS(call) = num_args; ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value)); + jit->call = rx; + return 1; } @@ -8899,7 +8907,7 @@ jit_SET_EX_OPLINE(jit, opline); } if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level)) { + if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { return 0; } } else { @@ -8944,6 +8952,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, zend_function *func = NULL; zval *function_name; ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL; + ir_ref call_ref = IR_UNUSED, static_call_ref = IR_UNUSED; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); @@ -9023,7 +9032,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, } if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { return 0; } } @@ -9129,24 +9138,23 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { - ir_ref ret; - if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { - ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame_tmp), + static_call_ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame_tmp), this_ref, func_ref, ir_CONST_U32(opline->extended_value)); } else { - ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame), + static_call_ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame), this_ref, func_ref, ir_CONST_U32(opline->extended_value)); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { - ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); + ir_GUARD(static_call_ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } - jit_STORE_IP(jit, ret); + + jit->call = static_call_ref; } if (!func) { @@ -9158,15 +9166,16 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) { return 0; } + call_ref = jit->call; } if (!func) { ir_MERGE_WITH(cold_path); + jit->call = ir_PHI_2(IR_ADDR, call_ref, static_call_ref); } - zend_jit_start_reuse_ip(jit); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level)) { + if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { return 0; } } else { @@ -9200,8 +9209,9 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, zend_call_info *call_info = NULL; zend_class_entry *ce; zend_function *func = NULL; - ir_ref func_ref, func_ref2, scope_ref, scope_ref2, if_cached, cold_path, ref; + ir_ref func_ref, func_ref2, scope_ref, scope_ref2, if_cached, cold_path; ir_ref if_static = IR_UNUSED; + ir_ref call_ref = IR_UNUSED, this_call_ref = IR_UNUSED; if (info) { call_info = info->callee_info; @@ -9235,7 +9245,7 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, } if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { return 0; } } @@ -9300,12 +9310,13 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, } jit_SET_EX_OPLINE(jit, opline); - ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_this_method_call_frame), + this_call_ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_this_method_call_frame), scope_ref, func_ref, ir_CONST_U32(opline->extended_value)); - ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); - jit_STORE_IP(jit, ref); + ir_GUARD(this_call_ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); + + jit->call = this_call_ref; if (!func) { cold_path = ir_END(); @@ -9337,14 +9348,16 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, return 0; } + call_ref = jit->call; + if (!func) { ir_MERGE_2(cold_path, ir_END()); + jit->call = ir_PHI_2(IR_ADDR, this_call_ref, call_ref); } } - zend_jit_start_reuse_ip(jit); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level)) { + if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { return 0; } } else { @@ -9426,7 +9439,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, } if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { return 0; } } @@ -9436,7 +9449,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, } if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level)) { + if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { return 0; } } else { @@ -9463,10 +9476,6 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); - if (!zend_jit_reuse_ip(jit)) { - return 0; - } - if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); @@ -9482,8 +9491,9 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t } } else { ir_ref cond = ir_AND_U32( - ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))), + ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(jit_EX_CALL(jit), func)), offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask)); + ir_ref call = jit_EX_CALL(jit); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); @@ -9504,6 +9514,7 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF); } jit_SET_EX_OPLINE(jit, opline); + jit_STORE_IP(jit, call); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref)); ir_IF_FALSE(if_pass_by_ref); @@ -9511,7 +9522,7 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t } } - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + arg_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); @@ -9536,11 +9547,7 @@ static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zen ir_ref ref_path = IR_UNUSED; op1_addr = OP1_ADDR(); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); - - if (!zend_jit_reuse_ip(jit)) { - return 0; - } + arg_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), opline->result.var); if (opline->op1_type == IS_VAR) { if (op1_info & MAY_BE_INDIRECT) { @@ -9628,11 +9635,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || arg_num <= MAX_ARG_FLAG_NUM); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); - - if (!zend_jit_reuse_ip(jit)) { - return 0; - } + arg_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), opline->result.var); if (opline->opcode == ZEND_SEND_VAR_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE @@ -9650,7 +9653,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen // JIT: if (RX->func->quick_arg_flags & mask) ir_ref if_send_by_ref = ir_IF(ir_AND_U32( - ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))), + ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(jit_EX_CALL(jit), func)), offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask))); ir_IF_TRUE_cold(if_send_by_ref); @@ -9697,7 +9700,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen ir_ref func, if_send_by_ref, if_prefer_ref; // JIT: if (RX->func->quick_arg_flags & mask) - func = ir_LOAD_A(jit_RX(func)); + func = ir_LOAD_A(jit_CALL(jit_EX_CALL(jit), func)); if_send_by_ref = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask))); @@ -9757,7 +9760,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen } else { // JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF) ir_ref if_send_by_ref = ir_IF(ir_AND_U32( - ir_LOAD_U32(jit_RX(This.u1.type_info)), + ir_LOAD_U32(jit_CALL(jit_EX_CALL(jit), This.u1.type_info)), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF))); ir_IF_TRUE_cold(if_send_by_ref); @@ -9928,24 +9931,14 @@ static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline) if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); - if (jit->reuse_ip) { - ref = jit_IP(jit); - } else { - ref = ir_LOAD_A(jit_EX(call)); - } - ref = jit_CALL(ref, This.u1.type_info); + ref = jit_CALL(jit_EX_CALL(jit), This.u1.type_info); ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF))); } } else { if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); - if (jit->reuse_ip) { - ref = jit_IP(jit); - } else { - ref = ir_LOAD_A(jit_EX(call)); - } - ref = jit_CALL(ref, This.u1.type_info); + ref = jit_CALL(jit_EX_CALL(jit), This.u1.type_info); ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF))); } } @@ -9954,11 +9947,7 @@ static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline) uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); ir_ref rx, if_ref, cold_path; - if (!zend_jit_reuse_ip(jit)) { - return 0; - } - - rx = jit_IP(jit); + rx = jit_EX_CALL(jit); ref = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))), @@ -9987,11 +9976,7 @@ static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline) { ir_ref call, if_may_have_undef, ret; - if (jit->reuse_ip) { - call = jit_IP(jit); - } else { - call = ir_LOAD_A(jit_EX(call)); - } + call = jit_EX_CALL(jit); if_may_have_undef = ir_IF(ir_AND_U8( ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)), @@ -10018,6 +10003,8 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen const zend_op *prev_opline; ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED; + rx = jit_EX_CALL(jit); + prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { prev_opline--; @@ -10087,14 +10074,6 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); - if (!jit->reuse_ip) { - zend_jit_start_reuse_ip(jit); - // JIT: call = EX(call); - jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); - } - rx = jit_IP(jit); - zend_jit_stop_reuse_ip(jit); - jit_SET_EX_OPLINE(jit, opline); if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) { @@ -10137,7 +10116,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (!trace) { - ir_ref if_deprecated_nodiscard, ret; + ir_ref if_deprecated_nodiscard; uint32_t no_discard = RETURN_VALUE_USED(opline) ? 0 : ZEND_ACC_NODISCARD; @@ -10147,34 +10126,18 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen ir_IF_TRUE_cold(if_deprecated_nodiscard); ir_ref helper = ir_CONST_FC_FUNC(no_discard ? zend_jit_deprecated_nodiscard_helper : zend_jit_deprecated_helper); - if (GCC_GLOBAL_REGS) { - ret = ir_CALL(IR_BOOL, helper); - } else { - ret = ir_CALL_1(IR_BOOL, helper, rx); - } + ir_ref ret = ir_CALL_1(IR_BOOL, helper, rx); ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_MERGE_WITH_EMPTY_FALSE(if_deprecated_nodiscard); } } else { if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { - ir_ref ret; - - if (GCC_GLOBAL_REGS) { - ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper)); - } else { - ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx); - } + ir_ref ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx); ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } if ((func->common.fn_flags & ZEND_ACC_NODISCARD) && !RETURN_VALUE_USED(opline)) { - ir_ref ret; - - if (GCC_GLOBAL_REGS) { - ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper)); - } else { - ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper), rx); - } + ir_ref ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper), rx); ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } } @@ -10411,7 +10374,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0)) { ir_ref observer_handler; - ir_ref rx = jit_FP(jit); + ir_ref rx = jit_EX_CALL(jit); struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref); if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); @@ -10595,7 +10558,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen for (i = 0; i < call_num_args; i++ ) { if (zend_jit_needs_arg_dtor(func, i, call_info)) { uint32_t offset = EX_NUM_TO_VAR(i); - zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); + zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(ir_ADD_OFFSET(rx, offset)); jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline); } @@ -10728,6 +10691,8 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen ir_MERGE_WITH(user_path); } + jit->call = IR_UNUSED; + return 1; } @@ -17223,6 +17188,14 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr zend_jit_set_last_valid_opline(jit, trace->opline); + /* Op handler invalidates jit->call by changing EX(call) */ + if (zend_jit_inc_call_level(opline->opcode) + || zend_jit_dec_call_level(opline->opcode) + || opline->opcode == ZEND_SEND_UNPACK + || opline->opcode == ZEND_SEND_ARRAY) { + jit->call = IR_UNUSED; + } + return 1; } diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 02b9f92d36b40..2127b3fdcb4c0 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -140,7 +140,7 @@ static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *jit, const zend_op * uint32_t stack_size; zend_jit_trace_stack *stack = NULL; - if (delayed_call_chain) { + if (jit->delayed_call_level) { assert(to_opline != NULL); /* CALL and IP share the same register */ flags |= ZEND_JIT_EXIT_RESTORE_CALL; } @@ -166,7 +166,7 @@ static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *jit, const zend_op * /* Try to reuse exit points */ if (to_opline != NULL - && !(flags & ZEND_JIT_EXIT_METHOD_CALL) + && !(flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_RESTORE_CALL)) && t->exit_count > 0) { uint32_t i = t->exit_count; @@ -193,6 +193,7 @@ static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *jit, const zend_op * t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)); } + ZEND_ASSERT(!(flags & ZEND_JIT_EXIT_RESTORE_CALL) || jit->call); t->exit_count++; t->exit_info[exit_point].opline = to_opline; t->exit_info[exit_point].op_array = op_array; @@ -203,6 +204,8 @@ static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *jit, const zend_op * t->exit_info[exit_point].poly_this_ref = 0; t->exit_info[exit_point].poly_func_reg = ZREG_NONE; t->exit_info[exit_point].poly_this_reg = ZREG_NONE; + t->exit_info[exit_point].call_ref = jit->call; + t->exit_info[exit_point].call_reg = ZREG_NONE; } return exit_point; @@ -3522,7 +3525,8 @@ static int zend_jit_trace_deoptimization( zend_jit_trace_stack *stack, zend_jit_exit_const *constants, int8_t func_reg, - bool polymorphic_side_trace) + bool polymorphic_side_trace, + int8_t call_reg) { int i; int check2 = -1; @@ -3633,7 +3637,7 @@ static int zend_jit_trace_deoptimization( ZEND_ASSERT(STACK_FLAGS(parent_stack, check2) == ZREG_ZVAL_COPY); ZEND_ASSERT(reg != ZREG_NONE); - if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg)) { + if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg, call_reg)) { return 0; } if (!zend_jit_restore_zval(jit, EX_NUM_TO_VAR(check2), reg)) { @@ -3642,7 +3646,7 @@ static int zend_jit_trace_deoptimization( } if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { - if (!zend_jit_save_call_chain(jit, -1)) { + if (!zend_jit_save_call_chain(jit, -1, call_reg)) { return 0; } } @@ -4269,7 +4273,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par parent_stack, parent_vars_count, ssa, stack, zend_jit_traces[parent_trace].constants, zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg, - polymorphic_side_trace)) { + polymorphic_side_trace, + zend_jit_traces[parent_trace].exit_info[exit_num].call_reg)) { goto jit_failure; } } @@ -4388,6 +4393,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par bool gen_handler = false; opline = p->opline; + +#if ZEND_DEBUG + { + const void *handler = ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->call_handler; + ir_RSTORE(8, jit_CONST_FUNC(&ctx, (uintptr_t)handler, IR_FASTCALL_FUNC)); + } +#endif + if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { op1_type = IS_UNKNOWN; } @@ -4503,10 +4516,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = zend_jit_trace_type_to_info( STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))); @@ -4575,10 +4585,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = zend_jit_trace_type_to_info( STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))); @@ -4640,10 +4647,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_addr = RES_REG_ADDR(); if (zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } if (!zend_jit_concat(&ctx, opline, op1_info, op2_info, res_addr, @@ -5132,10 +5136,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } } op1_def_addr = op1_addr; @@ -5157,10 +5158,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } } if (!zend_jit_assign(&ctx, opline, @@ -7000,6 +6998,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par TRACE_FRAME_SET_CLOSURE_CALL(call); } } + if (init_opline) { if (init_opline->opcode != ZEND_NEW && (init_opline->opcode != ZEND_INIT_METHOD_CALL @@ -7423,7 +7422,8 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n stack, stack_size, NULL, NULL, zend_jit_traces[trace_num].constants, zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg, - 0)) { + 0, + zend_jit_traces[trace_num].exit_info[exit_num].call_reg)) { goto jit_failure; } @@ -7997,7 +7997,8 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) fprintf(stderr, "/VM"); } if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) { - fprintf(stderr, "/CALL"); + fprintf(stderr, "/CALL(%s)", + zend_reg_name(t->exit_info[i].call_reg)); } if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) { fprintf(stderr, "/POLY"); @@ -8624,7 +8625,8 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf zend_jit_trace_stack *stack = stack_size ? t->stack_map + t->exit_info[exit_num].stack_offset : NULL; if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { - zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX]; + ZEND_ASSERT(t->exit_info[exit_num].call_reg != ZREG_NONE); + zend_execute_data *call = (zend_execute_data *)regs->gpr[t->exit_info[exit_num].call_reg]; call->prev_execute_data = EX(call); EX(call) = call; } diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 0ce7f082ae15f..e98e8eefe89a8 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -176,9 +176,8 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper_no_skip_re return zend_jit_copy_extra_args_helper_ex(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX false); } -bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D) +bool ZEND_FASTCALL zend_jit_deprecated_helper(zend_execute_data *call) { - zend_execute_data *call = (zend_execute_data *) opline; zend_function *fbc = call->func; zend_deprecated_function(fbc); @@ -204,9 +203,8 @@ bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D) return 1; } -bool ZEND_FASTCALL zend_jit_nodiscard_helper(OPLINE_D) +bool ZEND_FASTCALL zend_jit_nodiscard_helper(zend_execute_data *call) { - zend_execute_data *call = (zend_execute_data *) opline; zend_function *fbc = call->func; zend_nodiscard_function(fbc); @@ -232,19 +230,18 @@ bool ZEND_FASTCALL zend_jit_nodiscard_helper(OPLINE_D) return 1; } -bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(OPLINE_D) +bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(zend_execute_data *call) { - zend_execute_data *call = (zend_execute_data *) opline; zend_function *fbc = call->func; if (fbc->common.fn_flags & ZEND_ACC_DEPRECATED) { - if (zend_jit_deprecated_helper(OPLINE_C) == 0) { + if (zend_jit_deprecated_helper(call) == 0) { return 0; } } if (fbc->common.fn_flags & ZEND_ACC_NODISCARD) { - if (zend_jit_nodiscard_helper(OPLINE_C) == 0) { + if (zend_jit_nodiscard_helper(call) == 0) { return 0; } } From 771229d5675fe8b67fa38221253b1fa09fbdea95 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Tue, 22 Apr 2025 18:16:59 +0200 Subject: [PATCH 3/8] Prevent clobbering call_reg during deopt --- ext/opcache/jit/zend_jit_ir.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 4db072be3f141..7e176b2d38cd9 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -315,6 +315,8 @@ typedef struct _zend_jit_registers_buf { static uint32_t zend_jit_exit_point_by_addr(const void *addr); int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); +static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg); + static int zend_jit_assign_to_variable(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_use_addr, @@ -988,7 +990,7 @@ static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level, int8 if (call_reg != ZREG_NONE) { /* In deoptimization */ - rx = ir_RLOAD_A(call_reg); + rx = zend_jit_deopt_rload(jit, IR_ADDR, call_reg); jit->call = rx; } else { ZEND_ASSERT(jit->call != IR_UNUSED || call_level == 1); @@ -17296,6 +17298,11 @@ static int zend_jit_trace_start(zend_jit_ctx *jit, ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg); } + if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + ZEND_ASSERT(parent->exit_info[exit_num].call_reg >= 0); + ir_RLOAD_A(parent->exit_info[exit_num].call_reg); + } + ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num)); return 1; From 19457e859ffddb2abe5bb9d55cace40b58158c2d Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Tue, 22 Apr 2025 18:17:24 +0200 Subject: [PATCH 4/8] Fix type --- ext/opcache/jit/zend_jit_ir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 7e176b2d38cd9..81bedfa785405 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -7964,7 +7964,7 @@ static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t sm return 1; } -static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg, uint8_t call_reg) +static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg, int8_t call_reg) { zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr)); From 1426d6c9292d45e33922dcd7b4ac9b33db0217f3 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Tue, 22 Apr 2025 19:12:45 +0200 Subject: [PATCH 5/8] Disable debug helper --- ext/opcache/jit/zend_jit_trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 2127b3fdcb4c0..d7da071881fb4 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -4394,7 +4394,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par opline = p->opline; -#if ZEND_DEBUG +#if ZEND_DEBUG && 1 { const void *handler = ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->call_handler; ir_RSTORE(8, jit_CONST_FUNC(&ctx, (uintptr_t)handler, IR_FASTCALL_FUNC)); From b8c908b2a95669e9395c3455d0f2a92a7bbf9a89 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Wed, 23 Apr 2025 12:12:19 +0200 Subject: [PATCH 6/8] jit_STORE_IP() should reset last valid opline by default --- ext/opcache/jit/zend_jit_ir.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 81bedfa785405..847222164f0d1 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -316,6 +316,7 @@ static uint32_t zend_jit_exit_point_by_addr(const void *addr); int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg); +static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit); static int zend_jit_assign_to_variable(zend_jit_ctx *jit, const zend_op *opline, @@ -921,11 +922,17 @@ static ir_ref jit_IP(zend_jit_ctx *jit) return ir_RLOAD_A(ZREG_IP); } -static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref) +static void jit_STORE_IP_no_reset(zend_jit_ctx *jit, ir_ref ref) { ir_RSTORE(ZREG_IP, ref); } +static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref) +{ + jit_STORE_IP_no_reset(jit, ref); + zend_jit_reset_last_valid_opline(jit); +} + static ir_ref jit_IP32(zend_jit_ctx *jit) { return ir_RLOAD_U32(ZREG_IP); @@ -1028,10 +1035,10 @@ static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) } else { ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target)); } - jit_STORE_IP(jit, ref); + jit_STORE_IP_no_reset(jit, ref); } } else { - jit_STORE_IP(jit, ir_CONST_ADDR(target)); + jit_STORE_IP_no_reset(jit, ir_CONST_ADDR(target)); } zend_jit_set_last_valid_opline(jit, target); return 1; @@ -2021,7 +2028,7 @@ static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit) if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info); - jit_STORE_IP(jit, + jit_STORE_IP_no_reset(jit, ir_LOAD_A(jit_EX(opline))); ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); } else if (GCC_GLOBAL_REGS) { @@ -9516,7 +9523,7 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF); } jit_SET_EX_OPLINE(jit, opline); - jit_STORE_IP(jit, call); + jit_STORE_IP_no_reset(jit, call); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref)); ir_IF_FALSE(if_pass_by_ref); From a86f8535397bee558f02dfd4cf3a70497d787a46 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Wed, 23 Apr 2025 13:02:57 +0200 Subject: [PATCH 7/8] Disable debug helper --- ext/opcache/jit/zend_jit_trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index d7da071881fb4..46aad6acdb1ba 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -4394,7 +4394,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par opline = p->opline; -#if ZEND_DEBUG && 1 +#if ZEND_DEBUG && 0 { const void *handler = ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->call_handler; ir_RSTORE(8, jit_CONST_FUNC(&ctx, (uintptr_t)handler, IR_FASTCALL_FUNC)); From 88fca8b89de3b306f35101f0426ae1a4d143f02f Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Wed, 23 Apr 2025 16:58:37 +0200 Subject: [PATCH 8/8] Snapshot may be spilled --- ext/opcache/jit/zend_jit_ir.c | 126 +++++++++++++++++++------------ ext/opcache/jit/zend_jit_trace.c | 102 +++++++++++++++++-------- 2 files changed, 146 insertions(+), 82 deletions(-) diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 847222164f0d1..40bb65078307b 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -272,7 +272,9 @@ typedef struct _zend_jit_ctx { ir_ref tls; #endif ir_ref fp; - ir_ref call; /* EX(call) cache */ + ir_ref call; /* EX(call) cache */ + ir_ref poly_func_ref; /* restored from parent trace snapshot */ + ir_ref poly_this_ref; /* restored from parent trace snapshot */ ir_ref trace_loop_ref; ir_ref return_inputs; const zend_op_array *op_array; @@ -624,15 +626,12 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) bool save_call = false; if (addr < 0) { - if (t->exit_count > 0 - && jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) { - exit_point = t->exit_count - 1; - if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) { - n = 2; - } - if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_RESTORE_CALL) { - save_call = true; - } + exit_point = zend_jit_exit_point_by_addr((void*)(uintptr_t) jit->ctx.ir_base[addr].val.u64); + if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) { + n = 2; + } + if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + save_call = true; } } @@ -730,31 +729,60 @@ void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snaps if (exit_flags & ZEND_JIT_EXIT_RESTORE_CALL) { int8_t *reg_ops = ctx->regs[snapshot_ref]; - ZEND_ASSERT(reg_ops[n] != -1); + + int8_t reg = reg_ops[n]; + ir_ref ref = ir_insn_op(snapshot, n); + if (IR_REG_SPILLED(reg)) { + reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD; + ref = ir_get_spill_slot_offset(ctx, ref); + } + if ((exit_flags & ZEND_JIT_EXIT_FIXED) - && (t->exit_info[exit_point].call_reg != reg_ops[n])) { + && (t->exit_info[exit_point].call_reg != reg + || (IR_REG_SPILLED(reg) && t->exit_info[exit_point].call_ref != ref))) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } - t->exit_info[exit_point].call_reg = reg_ops[n]; + t->exit_info[exit_point].call_ref = ref; + t->exit_info[exit_point].call_reg = reg; n--; } if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) { int8_t *reg_ops = ctx->regs[snapshot_ref]; - ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1); + + int8_t func_reg = reg_ops[n - 1]; + ir_ref func_ref = ir_insn_op(snapshot, n - 1); + if (IR_REG_SPILLED(func_reg)) { + func_reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD; + func_ref = ir_get_spill_slot_offset(ctx, func_ref); + } + + int8_t this_reg = reg_ops[n]; + ir_ref this_ref = ir_insn_op(snapshot, n); + if (IR_REG_SPILLED(this_reg)) { + this_reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD; + this_ref = ir_get_spill_slot_offset(ctx, this_ref); + } + if ((exit_flags & ZEND_JIT_EXIT_FIXED) - && (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1] - || t->exit_info[exit_point].poly_this_reg != reg_ops[n])) { + && ((t->exit_info[exit_point].poly_func_reg != func_reg + || (IR_REG_SPILLED(func_reg) + && t->exit_info[exit_point].poly_func_ref != func_ref)) + || (t->exit_info[exit_point].poly_this_reg != this_reg + || (IR_REG_SPILLED(this_reg) + && t->exit_info[exit_point].poly_this_ref != this_ref)))) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } - t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1]; - t->exit_info[exit_point].poly_this_reg = reg_ops[n]; + t->exit_info[exit_point].poly_func_reg = func_reg; + t->exit_info[exit_point].poly_func_ref = func_ref; + t->exit_info[exit_point].poly_this_reg = this_reg; + t->exit_info[exit_point].poly_this_ref = this_ref; n -= 2; } @@ -983,7 +1011,7 @@ static ir_ref jit_EX_CALL(zend_jit_ctx *jit) return jit->call; } -static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level, int8_t call_reg) +static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) { ir_ref rx, call; @@ -995,14 +1023,8 @@ static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level, int8 call = ir_LOAD_A(jit_EX(call)); } - if (call_reg != ZREG_NONE) { - /* In deoptimization */ - rx = zend_jit_deopt_rload(jit, IR_ADDR, call_reg); - jit->call = rx; - } else { - ZEND_ASSERT(jit->call != IR_UNUSED || call_level == 1); - rx = jit_EX_CALL(jit); - } + ZEND_ASSERT(jit->call != IR_UNUSED || call_level == 1); + rx = jit_EX_CALL(jit); // JIT: call->prev_execute_data = call; ir_STORE(jit_CALL(rx, prev_execute_data), call); @@ -1021,7 +1043,7 @@ static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) ir_ref ref; if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } @@ -2718,6 +2740,8 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) #endif jit->fp = IR_UNUSED; jit->call = IR_UNUSED; + jit->poly_func_ref = IR_UNUSED; + jit->poly_this_ref = IR_UNUSED; jit->trace_loop_ref = IR_UNUSED; jit->return_inputs = IR_UNUSED; jit->bb_start_ref = NULL; @@ -7971,7 +7995,7 @@ static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t sm return 1; } -static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg, int8_t call_reg) +static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg) { zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr)); @@ -7979,7 +8003,7 @@ static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, ir_IF_FALSE_cold(if_def); if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { - if (!zend_jit_save_call_chain(jit, -1, call_reg)) { + if (!zend_jit_save_call_chain(jit, -1)) { return 0; } } @@ -8458,10 +8482,9 @@ static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32 return 1; } -static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg) +static int zend_jit_free_trampoline(zend_jit_ctx *jit, ir_ref func) { // JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) - ir_ref func = ir_RLOAD_A(func_reg); ir_ref if_trampoline = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))), ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE))); @@ -8803,7 +8826,7 @@ static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_ ir_ref func_ref = IR_UNUSED; if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } @@ -8916,7 +8939,7 @@ jit_SET_EX_OPLINE(jit, opline); } if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { @@ -8952,15 +8975,15 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, zend_class_entry *trace_ce, zend_jit_trace_rec *trace, int checked_stack, - int8_t func_reg, - int8_t this_reg, + ir_ref func_ref, + ir_ref this_ref, bool polymorphic_side_trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; zval *function_name; - ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL; + ir_ref if_static = IR_UNUSED, cold_path; ir_ref call_ref = IR_UNUSED, static_call_ref = IR_UNUSED; ZEND_ASSERT(opline->op2_type == IS_CONST); @@ -8980,9 +9003,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, if (polymorphic_side_trace) { /* function is passed in r0 from parent_trace */ - ZEND_ASSERT(func_reg >= 0 && this_reg >= 0); - func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg); - this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg); + ZEND_ASSERT(func_ref != IR_UNUSED && this_ref != IR_UNUSED); } else { ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2; @@ -9041,7 +9062,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, } if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } @@ -9184,7 +9205,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, } if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { @@ -9254,7 +9275,7 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, } if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } @@ -9366,7 +9387,7 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, } if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { @@ -9448,7 +9469,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, } if (jit->delayed_call_level) { - if (!zend_jit_save_call_chain(jit, jit->delayed_call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } @@ -9458,7 +9479,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, } if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { - if (!zend_jit_save_call_chain(jit, call_level, ZREG_NONE)) { + if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { @@ -17211,6 +17232,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr static int zend_jit_deoptimizer_start(zend_jit_ctx *jit, zend_string *name, uint32_t trace_num, + zend_jit_trace_info *trace, uint32_t exit_num) { zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET); @@ -17301,13 +17323,19 @@ static int zend_jit_trace_start(zend_jit_ctx *jit, if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0); - ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg); - ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg); + if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_func_reg)) { + ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg); + } + if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_this_reg)) { + ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg); + } } if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { ZEND_ASSERT(parent->exit_info[exit_num].call_reg >= 0); - ir_RLOAD_A(parent->exit_info[exit_num].call_reg); + if (!IR_REG_SPILLED(parent->exit_info[exit_num].call_reg)) { + ir_RLOAD_A(parent->exit_info[exit_num].call_reg); + } } ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num)); diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 46aad6acdb1ba..198f792274707 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -3516,18 +3516,18 @@ static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t } static int zend_jit_trace_deoptimization( - zend_jit_ctx *jit, - uint32_t flags, - const zend_op *opline, - zend_jit_trace_stack *parent_stack, - int parent_vars_count, - zend_ssa *ssa, - zend_jit_trace_stack *stack, - zend_jit_exit_const *constants, - int8_t func_reg, - bool polymorphic_side_trace, - int8_t call_reg) + zend_jit_ctx *jit, + const zend_jit_trace_exit_info *exit_info, + zend_jit_trace_stack *parent_stack, + int parent_vars_count, + zend_ssa *ssa, + zend_jit_trace_stack *stack, + zend_jit_exit_const *constants, + bool polymorphic_side_trace) { + uint32_t flags = exit_info->flags; + const zend_op *opline = exit_info->opline; + int i; int check2 = -1; @@ -3632,12 +3632,23 @@ static int zend_jit_trace_deoptimization( } } + if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { + int8_t call_reg = exit_info->call_reg; + ir_ref call_ref = exit_info->call_ref; + ZEND_ASSERT(call_reg >= 0); + if (IR_REG_SPILLED(exit_info->call_reg)) { + jit->call = ir_LOAD_A(ir_ADD_OFFSET(zend_jit_deopt_rload(jit, IR_ADDR, IR_REG_NUM(call_reg)), call_ref)); + } else { + jit->call = zend_jit_deopt_rload(jit, IR_ADDR, call_reg); + } + } + if (check2 != -1) { int8_t reg = STACK_REG(parent_stack, check2); ZEND_ASSERT(STACK_FLAGS(parent_stack, check2) == ZREG_ZVAL_COPY); ZEND_ASSERT(reg != ZREG_NONE); - if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg, call_reg)) { + if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg)) { return 0; } if (!zend_jit_restore_zval(jit, EX_NUM_TO_VAR(check2), reg)) { @@ -3646,7 +3657,7 @@ static int zend_jit_trace_deoptimization( } if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { - if (!zend_jit_save_call_chain(jit, -1, call_reg)) { + if (!zend_jit_save_call_chain(jit, -1)) { return 0; } } @@ -3671,9 +3682,27 @@ static int zend_jit_trace_deoptimization( zend_jit_check_exception(jit); } - if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) { - if (!zend_jit_free_trampoline(jit, func_reg)) { - return 0; + if (flags & ZEND_JIT_EXIT_METHOD_CALL) { + int8_t func_reg = exit_info->poly_func_reg; + ir_ref func_ref = exit_info->poly_func_ref; + if (IR_REG_SPILLED(func_reg)) { + jit->poly_func_ref = ir_LOAD_A(ir_ADD_OFFSET(zend_jit_deopt_rload(jit, IR_ADDR, IR_REG_NUM(func_reg)), func_ref)); + } else { + jit->poly_func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg); + } + + int8_t this_reg = exit_info->poly_this_reg; + ir_ref this_ref = exit_info->poly_this_ref; + if (IR_REG_SPILLED(this_reg)) { + jit->poly_this_ref = ir_LOAD_A(ir_ADD_OFFSET(zend_jit_deopt_rload(jit, IR_ADDR, IR_REG_NUM(this_reg)), this_ref)); + } else { + jit->poly_this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg); + } + + if (!polymorphic_side_trace) { + if (!zend_jit_free_trampoline(jit, jit->poly_func_ref)) { + return 0; + } } } @@ -4268,13 +4297,10 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (parent_trace) { /* Deoptimization */ if (!zend_jit_trace_deoptimization(&ctx, - zend_jit_traces[parent_trace].exit_info[exit_num].flags, - zend_jit_traces[parent_trace].exit_info[exit_num].opline, + &zend_jit_traces[parent_trace].exit_info[exit_num], parent_stack, parent_vars_count, ssa, stack, zend_jit_traces[parent_trace].constants, - zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg, - polymorphic_side_trace, - zend_jit_traces[parent_trace].exit_info[exit_num].call_reg)) { + polymorphic_side_trace)) { goto jit_failure; } } @@ -6378,8 +6404,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op_array, ssa, ssa_op, frame->call_level, op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce, p + 1, peek_checked_stack - checked_stack, - polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1, - polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1, + polymorphic_side_trace ? jit->poly_func_ref : -1, + polymorphic_side_trace ? jit->poly_this_ref : -1, polymorphic_side_trace)) { goto jit_failure; } @@ -7403,7 +7429,7 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n name = zend_jit_trace_escape_name(trace_num, exit_num); - if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, exit_num)) { + if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, &zend_jit_traces[trace_num], exit_num)) { zend_string_release(name); return NULL; } @@ -7417,13 +7443,10 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n NULL; if (!zend_jit_trace_deoptimization(&ctx, - zend_jit_traces[trace_num].exit_info[exit_num].flags, - zend_jit_traces[trace_num].exit_info[exit_num].opline, + &zend_jit_traces[trace_num].exit_info[exit_num], stack, stack_size, NULL, NULL, zend_jit_traces[trace_num].constants, - zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg, - 0, - zend_jit_traces[trace_num].exit_info[exit_num].call_reg)) { + 0)) { goto jit_failure; } @@ -8625,8 +8648,15 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf zend_jit_trace_stack *stack = stack_size ? t->stack_map + t->exit_info[exit_num].stack_offset : NULL; if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { - ZEND_ASSERT(t->exit_info[exit_num].call_reg != ZREG_NONE); - zend_execute_data *call = (zend_execute_data *)regs->gpr[t->exit_info[exit_num].call_reg]; + int8_t reg = t->exit_info[exit_num].call_reg; + ZEND_ASSERT(reg != ZREG_NONE); + + zend_execute_data *call; + if (IR_REG_SPILLED(reg)) { + call = *(zend_execute_data**)(regs->gpr[IR_REG_NUM(reg)] + t->exit_info[exit_num].call_ref); + } else { + call = (zend_execute_data*)regs->gpr[reg]; + } call->prev_execute_data = EX(call); EX(call) = call; } @@ -8732,9 +8762,15 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { - ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0); - zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg]; + int8_t reg = t->exit_info[exit_num].poly_func_reg; + ZEND_ASSERT(reg >= 0); + zend_function *func; + if (IR_REG_SPILLED(reg)) { + func = *(zend_function**)(regs->gpr[IR_REG_NUM(reg)] + t->exit_info[exit_num].poly_func_ref); + } else { + func = (zend_function*)regs->gpr[reg]; + } if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) { zend_string_release_ex(func->common.function_name, 0); zend_free_trampoline(func);