diff --git a/UPGRADING.INTERNALS b/UPGRADING.INTERNALS index 71b6e1fd01ecc..3ec54f49391b8 100644 --- a/UPGRADING.INTERNALS +++ b/UPGRADING.INTERNALS @@ -49,6 +49,9 @@ PHP 8.6 INTERNALS UPGRADE NOTES removed. Call zend_wrong_param_count(); followed by RETURN_THROWS(); instead. . PHP_HAVE_STREAMS macro removed from . + . zend_function.arg_info is now always a zend_arg_info*. Before, it was a + zend_internal_arg_info on internal functions, unless the + ZEND_ACC_USER_ARG_INFO flag was set. ======================== 2. Build system changes diff --git a/Zend/tests/closures/gh19653_3.phpt b/Zend/tests/closures/gh19653_3.phpt new file mode 100644 index 0000000000000..7a9ac589182c3 --- /dev/null +++ b/Zend/tests/closures/gh19653_3.phpt @@ -0,0 +1,21 @@ +--TEST-- +GH-19653 (Closure named argument unpacking between temporary closures can cause a crash) - temporary method variation +--EXTENSIONS-- +zend_test +--FILE-- + +--EXPECTF-- +Fatal error: Uncaught Error: Unknown named parameter $tmpMethodParamName in %s:%d +Stack trace: +#0 %s(%d): usage1(Object(Closure)) +#1 {main} + thrown in %s on line %d diff --git a/Zend/tests/unified_arg_infos_001.phpt b/Zend/tests/unified_arg_infos_001.phpt new file mode 100644 index 0000000000000..d6699aa8d8cb5 --- /dev/null +++ b/Zend/tests/unified_arg_infos_001.phpt @@ -0,0 +1,21 @@ +--TEST-- +Declaring non persistent method with arg info +--EXTENSIONS-- +zend_test +--FILE-- +testTmpMethodWithArgInfo(null); + +echo new ReflectionFunction($o->testTmpMethodWithArgInfo(...)); + +?> +--EXPECT-- +Closure [ public method testTmpMethodWithArgInfo ] { + + - Parameters [2] { + Parameter #0 [ Foo|Bar|null $tmpMethodParamName = null ] + Parameter #1 [ string $tmpMethodParamWithStringDefaultValue = "tmpMethodParamWithStringDefaultValue" ] + } +} diff --git a/Zend/zend.c b/Zend/zend.c index 4d024444a4be9..c46c8e9ada86c 100644 --- a/Zend/zend.c +++ b/Zend/zend.c @@ -38,6 +38,8 @@ #include "zend_call_stack.h" #include "zend_max_execution_timer.h" #include "zend_hrtime.h" +#include "zend_enum.h" +#include "zend_closures.h" #include "Optimizer/zend_optimizer.h" #include "php.h" #include "php_globals.h" @@ -1054,6 +1056,7 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */ ZVAL_UNDEF(&EG(last_fatal_error_backtrace)); zend_interned_strings_init(); + zend_object_handlers_startup(); zend_startup_builtin_functions(); zend_register_standard_constants(); zend_register_auto_global(zend_string_init_interned("GLOBALS", sizeof("GLOBALS") - 1, 1), 1, php_auto_globals_create_globals); @@ -1077,6 +1080,9 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */ tsrm_set_new_thread_end_handler(zend_new_thread_end_handler); tsrm_set_shutdown_handler(zend_interned_strings_dtor); #endif + + zend_enum_startup(); + zend_closure_startup(); } /* }}} */ diff --git a/Zend/zend_API.c b/Zend/zend_API.c index ca001f0e0c434..0881a169dfa7d 100644 --- a/Zend/zend_API.c +++ b/Zend/zend_API.c @@ -20,6 +20,7 @@ */ #include "zend.h" +#include "zend_compile.h" #include "zend_execute.h" #include "zend_API.h" #include "zend_hash.h" @@ -2927,6 +2928,80 @@ static zend_always_inline void zend_normalize_internal_type(zend_type *type) { } ZEND_TYPE_FOREACH_END(); } +static void zend_convert_internal_arg_info_type(zend_type *type, bool persistent) +{ + if (ZEND_TYPE_HAS_LITERAL_NAME(*type)) { + // gen_stubs.php does not support codegen for compound types. As a + // temporary workaround, we support union types by splitting + // the type name on `|` characters if necessary. + const char *class_name = ZEND_TYPE_LITERAL_NAME(*type); + type->type_mask &= ~_ZEND_TYPE_LITERAL_NAME_BIT; + + size_t num_types = 1; + const char *p = class_name; + while ((p = strchr(p, '|'))) { + num_types++; + p++; + } + + if (num_types == 1) { + /* Simple class type */ + zend_string *str = zend_string_init_interned(class_name, strlen(class_name), persistent); + zend_alloc_ce_cache(str); + ZEND_TYPE_SET_PTR(*type, str); + type->type_mask |= _ZEND_TYPE_NAME_BIT; + } else { + /* Union type */ + zend_type_list *list = pemalloc(ZEND_TYPE_LIST_SIZE(num_types), persistent); + list->num_types = num_types; + ZEND_TYPE_SET_LIST(*type, list); + ZEND_TYPE_FULL_MASK(*type) |= _ZEND_TYPE_UNION_BIT; + + const char *start = class_name; + uint32_t j = 0; + while (true) { + const char *end = strchr(start, '|'); + zend_string *str = zend_string_init_interned(start, end ? end - start : strlen(start), persistent); + zend_alloc_ce_cache(str); + list->types[j] = (zend_type) ZEND_TYPE_INIT_CLASS(str, 0, 0); + if (!end) { + break; + } + start = end + 1; + j++; + } + } + } + if (ZEND_TYPE_IS_ITERABLE_FALLBACK(*type)) { + /* Warning generated an extension load warning which is emitted for every test + zend_error(E_CORE_WARNING, "iterable type is now a compile time alias for array|Traversable," + " regenerate the argument info via the php-src gen_stub build script"); + */ + zend_type legacy_iterable = ZEND_TYPE_INIT_CLASS_MASK( + ZSTR_KNOWN(ZEND_STR_TRAVERSABLE), + (type->type_mask | MAY_BE_ARRAY) + ); + *type = legacy_iterable; + } +} + +ZEND_API void zend_convert_internal_arg_info(zend_arg_info *new_arg_info, const zend_internal_arg_info *arg_info, bool is_return_info, bool persistent) +{ + if (!is_return_info) { + new_arg_info->name = zend_string_init_interned(arg_info->name, strlen(arg_info->name), persistent); + if (arg_info->default_value) { + new_arg_info->default_value = zend_string_init_interned(arg_info->default_value, strlen(arg_info->default_value), persistent); + } else { + new_arg_info->default_value = NULL; + } + } else { + new_arg_info->name = NULL; + new_arg_info->default_value = NULL; + } + new_arg_info->type = arg_info->type; + zend_convert_internal_arg_info_type(&new_arg_info->type, persistent); +} + /* registers all functions in *library_functions in the function hash */ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend_function_entry *functions, HashTable *function_table, int type) /* {{{ */ { @@ -2938,6 +3013,7 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend int error_type; zend_string *lowercase_name; size_t fname_len; + const zend_internal_arg_info *internal_arg_info; if (type==MODULE_PERSISTENT) { error_type = E_CORE_WARNING; @@ -2994,7 +3070,7 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend if (ptr->arg_info) { zend_internal_function_info *info = (zend_internal_function_info*)ptr->arg_info; - internal_function->arg_info = (zend_internal_arg_info*)ptr->arg_info+1; + internal_arg_info = ptr->arg_info+1; internal_function->num_args = ptr->num_args; /* Currently you cannot denote that the function can accept less arguments than num_args */ if (info->required_num_args == (uintptr_t)-1) { @@ -3024,7 +3100,7 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend zend_error(E_CORE_WARNING, "Missing arginfo for %s%s%s()", scope ? ZSTR_VAL(scope->name) : "", scope ? "::" : "", ptr->fname); - internal_function->arg_info = NULL; + internal_arg_info = NULL; internal_function->num_args = 0; internal_function->required_num_args = 0; } @@ -3035,13 +3111,11 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend !(internal_function->fn_flags & ZEND_ACC_HAS_RETURN_TYPE)) { zend_error(E_CORE_WARNING, "%s::__toString() implemented without string return type", ZSTR_VAL(scope->name)); - internal_function->arg_info = (zend_internal_arg_info *) arg_info_toString + 1; + internal_arg_info = (zend_internal_arg_info *) arg_info_toString + 1; internal_function->fn_flags |= ZEND_ACC_HAS_RETURN_TYPE; internal_function->num_args = internal_function->required_num_args = 0; } - - zend_set_function_arg_flags((zend_function*)internal_function); if (ptr->flags & ZEND_ACC_ABSTRACT) { if (scope) { /* This is a class that must be abstract itself. Here we set the check info. */ @@ -3106,17 +3180,17 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend } /* If types of arguments have to be checked */ - if (reg_function->arg_info && num_args) { + if (internal_arg_info && num_args) { uint32_t i; for (i = 0; i < num_args; i++) { - zend_internal_arg_info *arg_info = ®_function->arg_info[i]; + const zend_internal_arg_info *arg_info = &internal_arg_info[i]; ZEND_ASSERT(arg_info->name && "Parameter must have a name"); if (ZEND_TYPE_IS_SET(arg_info->type)) { reg_function->fn_flags |= ZEND_ACC_HAS_TYPE_HINTS; } #if ZEND_DEBUG for (uint32_t j = 0; j < i; j++) { - if (!strcmp(arg_info->name, reg_function->arg_info[j].name)) { + if (!strcmp(arg_info->name, internal_arg_info[j].name)) { zend_error_noreturn(E_CORE_ERROR, "Duplicate parameter name $%s for function %s%s%s()", arg_info->name, scope ? ZSTR_VAL(scope->name) : "", scope ? "::" : "", ptr->fname); @@ -3126,78 +3200,24 @@ ZEND_API zend_result zend_register_functions(zend_class_entry *scope, const zend } } - /* Rebuild arginfos if parameter/property types and/or a return type are used */ - if (reg_function->arg_info && - (reg_function->fn_flags & (ZEND_ACC_HAS_RETURN_TYPE|ZEND_ACC_HAS_TYPE_HINTS))) { - /* convert "const char*" class type names into "zend_string*" */ + /* Convert zend_internal_arg_info to zend_arg_info */ + if (internal_arg_info) { uint32_t i; - zend_internal_arg_info *arg_info = reg_function->arg_info - 1; - zend_internal_arg_info *new_arg_info; + const zend_internal_arg_info *arg_info = internal_arg_info - 1; + zend_arg_info *new_arg_info; /* Treat return type as an extra argument */ num_args++; - new_arg_info = malloc(sizeof(zend_internal_arg_info) * num_args); - memcpy(new_arg_info, arg_info, sizeof(zend_internal_arg_info) * num_args); + new_arg_info = malloc(sizeof(zend_arg_info) * num_args); reg_function->arg_info = new_arg_info + 1; for (i = 0; i < num_args; i++) { - if (ZEND_TYPE_HAS_LITERAL_NAME(new_arg_info[i].type)) { - // gen_stubs.php does not support codegen for DNF types in arg infos. - // As a temporary workaround, we split the type name on `|` characters, - // converting it to an union type if necessary. - const char *class_name = ZEND_TYPE_LITERAL_NAME(new_arg_info[i].type); - new_arg_info[i].type.type_mask &= ~_ZEND_TYPE_LITERAL_NAME_BIT; - - size_t num_types = 1; - const char *p = class_name; - while ((p = strchr(p, '|'))) { - num_types++; - p++; - } - - if (num_types == 1) { - /* Simple class type */ - zend_string *str = zend_string_init_interned(class_name, strlen(class_name), 1); - zend_alloc_ce_cache(str); - ZEND_TYPE_SET_PTR(new_arg_info[i].type, str); - new_arg_info[i].type.type_mask |= _ZEND_TYPE_NAME_BIT; - } else { - /* Union type */ - zend_type_list *list = malloc(ZEND_TYPE_LIST_SIZE(num_types)); - list->num_types = num_types; - ZEND_TYPE_SET_LIST(new_arg_info[i].type, list); - ZEND_TYPE_FULL_MASK(new_arg_info[i].type) |= _ZEND_TYPE_UNION_BIT; - - const char *start = class_name; - uint32_t j = 0; - while (true) { - const char *end = strchr(start, '|'); - zend_string *str = zend_string_init_interned(start, end ? end - start : strlen(start), 1); - zend_alloc_ce_cache(str); - list->types[j] = (zend_type) ZEND_TYPE_INIT_CLASS(str, 0, 0); - if (!end) { - break; - } - start = end + 1; - j++; - } - } - } - if (ZEND_TYPE_IS_ITERABLE_FALLBACK(new_arg_info[i].type)) { - /* Warning generated an extension load warning which is emitted for every test - zend_error(E_CORE_WARNING, "iterable type is now a compile time alias for array|Traversable," - " regenerate the argument info via the php-src gen_stub build script"); - */ - zend_type legacy_iterable = ZEND_TYPE_INIT_CLASS_MASK( - ZSTR_KNOWN(ZEND_STR_TRAVERSABLE), - (new_arg_info[i].type.type_mask | MAY_BE_ARRAY) - ); - new_arg_info[i].type = legacy_iterable; - } - - zend_normalize_internal_type(&new_arg_info[i].type); + zend_convert_internal_arg_info(&new_arg_info[i], &arg_info[i], + i == 0, true); } } + zend_set_function_arg_flags((zend_function*)reg_function); + if (scope) { zend_check_magic_method_implementation( scope, (zend_function *)reg_function, lowercase_name, E_CORE_ERROR); @@ -5242,48 +5262,43 @@ static zend_string *try_parse_string(const char *str, size_t len, char quote) { } ZEND_API zend_result zend_get_default_from_internal_arg_info( - zval *default_value_zval, const zend_internal_arg_info *arg_info) + zval *default_value_zval, const zend_arg_info *arg_info) { - const char *default_value = arg_info->default_value; + const zend_string *default_value = arg_info->default_value; if (!default_value) { return FAILURE; } /* Avoid going through the full AST machinery for some simple and common cases. */ - size_t default_value_len = strlen(default_value); zend_ulong lval; - if (default_value_len == sizeof("null")-1 - && !memcmp(default_value, "null", sizeof("null")-1)) { + if (zend_string_equals_literal(default_value, "null")) { ZVAL_NULL(default_value_zval); return SUCCESS; - } else if (default_value_len == sizeof("true")-1 - && !memcmp(default_value, "true", sizeof("true")-1)) { + } else if (zend_string_equals_literal(default_value, "true")) { ZVAL_TRUE(default_value_zval); return SUCCESS; - } else if (default_value_len == sizeof("false")-1 - && !memcmp(default_value, "false", sizeof("false")-1)) { + } else if (zend_string_equals_literal(default_value, "false")) { ZVAL_FALSE(default_value_zval); return SUCCESS; - } else if (default_value_len >= 2 - && (default_value[0] == '\'' || default_value[0] == '"') - && default_value[default_value_len - 1] == default_value[0]) { + } else if (ZSTR_LEN(default_value) >= 2 + && (ZSTR_VAL(default_value)[0] == '\'' || ZSTR_VAL(default_value)[0] == '"') + && ZSTR_VAL(default_value)[ZSTR_LEN(default_value) - 1] == ZSTR_VAL(default_value)[0]) { zend_string *str = try_parse_string( - default_value + 1, default_value_len - 2, default_value[0]); + ZSTR_VAL(default_value) + 1, ZSTR_LEN(default_value) - 2, ZSTR_VAL(default_value)[0]); if (str) { ZVAL_STR(default_value_zval, str); return SUCCESS; } - } else if (default_value_len == sizeof("[]")-1 - && !memcmp(default_value, "[]", sizeof("[]")-1)) { + } else if (zend_string_equals_literal(default_value, "[]")) { ZVAL_EMPTY_ARRAY(default_value_zval); return SUCCESS; - } else if (ZEND_HANDLE_NUMERIC_STR(default_value, default_value_len, lval)) { + } else if (ZEND_HANDLE_NUMERIC(default_value, lval)) { ZVAL_LONG(default_value_zval, lval); return SUCCESS; } #if 0 - fprintf(stderr, "Evaluating %s via AST\n", default_value); + fprintf(stderr, "Evaluating %s via AST\n", ZSTR_VAL(default_value)); #endif - return get_default_via_ast(default_value_zval, default_value); + return get_default_via_ast(default_value_zval, ZSTR_VAL(default_value)); } diff --git a/Zend/zend_API.h b/Zend/zend_API.h index 1bb47a2e8703d..c1ccbf13666a5 100644 --- a/Zend/zend_API.h +++ b/Zend/zend_API.h @@ -922,8 +922,12 @@ ZEND_API bool zend_is_iterable(const zval *iterable); ZEND_API bool zend_is_countable(const zval *countable); +ZEND_API void zend_convert_internal_arg_info(zend_arg_info *new_arg_info, + const zend_internal_arg_info *arg_info, bool is_return_info, + bool permanent); + ZEND_API zend_result zend_get_default_from_internal_arg_info( - zval *default_value_zval, const zend_internal_arg_info *arg_info); + zval *default_value_zval, const zend_arg_info *arg_info); END_EXTERN_C() diff --git a/Zend/zend_closures.c b/Zend/zend_closures.c index 4ecb6b2c493b9..05b6862044816 100644 --- a/Zend/zend_closures.c +++ b/Zend/zend_closures.c @@ -510,7 +510,7 @@ ZEND_API zend_function *zend_get_closure_invoke_method(zend_object *object) /* { * ZEND_ACC_USER_ARG_INFO flag to prevent invalid usage by Reflection */ invoke->type = ZEND_INTERNAL_FUNCTION; invoke->internal_function.fn_flags = - ZEND_ACC_PUBLIC | ZEND_ACC_CALL_VIA_HANDLER | (closure->func.common.fn_flags & keep_flags); + ZEND_ACC_PUBLIC | ZEND_ACC_CALL_VIA_HANDLER | ZEND_ACC_NEVER_CACHE | (closure->func.common.fn_flags & keep_flags); if (closure->func.type != ZEND_INTERNAL_FUNCTION || (closure->func.common.fn_flags & ZEND_ACC_USER_ARG_INFO)) { invoke->internal_function.fn_flags |= ZEND_ACC_USER_ARG_INFO; @@ -618,7 +618,6 @@ static HashTable *zend_closure_get_debug_info(zend_object *object, int *is_temp) zval val; struct _zend_arg_info *arg_info = closure->func.common.arg_info; HashTable *debug_info; - bool zstr_args = (closure->func.type == ZEND_USER_FUNCTION) || (closure->func.common.fn_flags & ZEND_ACC_USER_ARG_INFO); *is_temp = 1; @@ -694,15 +693,9 @@ static HashTable *zend_closure_get_debug_info(zend_object *object, int *is_temp) zend_string *name; zval info; ZEND_ASSERT(arg_info->name && "Argument should have name"); - if (zstr_args) { - name = zend_strpprintf(0, "%s$%s", - ZEND_ARG_SEND_MODE(arg_info) ? "&" : "", - ZSTR_VAL(arg_info->name)); - } else { - name = zend_strpprintf(0, "%s$%s", - ZEND_ARG_SEND_MODE(arg_info) ? "&" : "", - ((zend_internal_arg_info*)arg_info)->name); - } + name = zend_strpprintf(0, "%s$%s", + ZEND_ARG_SEND_MODE(arg_info) ? "&" : "", + ZSTR_VAL(arg_info->name)); ZVAL_NEW_STR(&info, zend_strpprintf(0, "%s", i >= required ? "" : "")); zend_hash_update(Z_ARRVAL(val), name, &info); zend_string_release_ex(name, 0); @@ -885,8 +878,7 @@ ZEND_API void zend_create_fake_closure(zval *res, zend_function *func, zend_clas } /* }}} */ -/* __call and __callStatic name the arguments "$arguments" in the docs. */ -static zend_internal_arg_info trampoline_arg_info[] = {ZEND_ARG_VARIADIC_TYPE_INFO(false, arguments, IS_MIXED, false)}; +static zend_arg_info trampoline_arg_info[1]; void zend_closure_from_frame(zval *return_value, const zend_execute_data *call) { /* {{{ */ zval instance; @@ -951,3 +943,11 @@ void zend_closure_bind_var_ex(zval *closure_zv, uint32_t offset, zval *val) /* { ZVAL_COPY_VALUE(var, val); } /* }}} */ + +void zend_closure_startup(void) +{ + /* __call and __callStatic name the arguments "$arguments" in the docs. */ + trampoline_arg_info[0].name = zend_string_init_interned("arguments", strlen("arguments"), true); + trampoline_arg_info[0].type = (zend_type)ZEND_TYPE_INIT_CODE(IS_MIXED, false, _ZEND_ARG_INFO_FLAGS(false, 1, 0)); + trampoline_arg_info[0].default_value = NULL; +} diff --git a/Zend/zend_closures.h b/Zend/zend_closures.h index 8bea4ffb051e8..a118044c6e248 100644 --- a/Zend/zend_closures.h +++ b/Zend/zend_closures.h @@ -28,6 +28,7 @@ BEGIN_EXTERN_C() #define ZEND_CLOSURE_OBJECT(op_array) \ ((zend_object*)((char*)(op_array) - sizeof(zend_object))) +void zend_closure_startup(void); void zend_register_closure_ce(void); void zend_closure_bind_var(zval *closure_zv, zend_string *var_name, zval *var); void zend_closure_bind_var_ex(zval *closure_zv, uint32_t offset, zval *val); diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index 50ba8029873ad..5eba2ec1366fa 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -3688,21 +3688,10 @@ static void zend_compile_compound_assign(znode *result, zend_ast *ast) /* {{{ */ static uint32_t zend_get_arg_num(const zend_function *fn, const zend_string *arg_name) { // TODO: Caching? - if (fn->type == ZEND_USER_FUNCTION) { - for (uint32_t i = 0; i < fn->common.num_args; i++) { - const zend_arg_info *arg_info = &fn->op_array.arg_info[i]; - if (zend_string_equals(arg_info->name, arg_name)) { - return i + 1; - } - } - } else { - ZEND_ASSERT(fn->common.num_args == 0 || fn->internal_function.arg_info); - for (uint32_t i = 0; i < fn->common.num_args; i++) { - const zend_internal_arg_info *arg_info = &fn->internal_function.arg_info[i]; - size_t len = strlen(arg_info->name); - if (zend_string_equals_cstr(arg_name, arg_info->name, len)) { - return i + 1; - } + for (uint32_t i = 0; i < fn->common.num_args; i++) { + zend_arg_info *arg_info = &fn->op_array.arg_info[i]; + if (zend_string_equals(arg_info->name, arg_name)) { + return i + 1; } } @@ -4702,7 +4691,7 @@ static uint32_t zend_compile_frameless_icall_ex(znode *result, const zend_ast_li if (i < args->children) { zend_compile_expr(&arg_zvs[i], args->child[i]); } else { - const zend_internal_arg_info *arg_info = (const zend_internal_arg_info *)&fbc->common.arg_info[i]; + const zend_arg_info *arg_info = &fbc->common.arg_info[i]; arg_zvs[i].op_type = IS_CONST; if (zend_get_default_from_internal_arg_info(&arg_zvs[i].u.constant, arg_info) == FAILURE) { ZEND_UNREACHABLE(); @@ -7934,6 +7923,7 @@ static void zend_compile_params(zend_ast *ast, zend_ast *return_type_ast, uint32 arg_info = &arg_infos[i]; arg_info->name = zend_string_copy(name); arg_info->type = (zend_type) ZEND_TYPE_INIT_NONE(0); + arg_info->default_value = NULL; if (attributes_ast) { zend_compile_attributes( diff --git a/Zend/zend_compile.h b/Zend/zend_compile.h index 86fab4b57ded6..d2a3b47bf92f4 100644 --- a/Zend/zend_compile.h +++ b/Zend/zend_compile.h @@ -592,7 +592,7 @@ typedef struct _zend_internal_function { zend_function *prototype; uint32_t num_args; uint32_t required_num_args; - zend_internal_arg_info *arg_info; + zend_arg_info *arg_info; HashTable *attributes; ZEND_MAP_PTR_DEF(void **, run_time_cache); zend_string *doc_comment; @@ -976,7 +976,8 @@ ZEND_API ZEND_COLD void zend_user_exception_handler(void); } \ } while (0) -void zend_free_internal_arg_info(zend_internal_function *function); +ZEND_API void zend_free_internal_arg_info(zend_internal_function *function, + bool permanent); ZEND_API void destroy_zend_function(zend_function *function); ZEND_API void zend_function_dtor(zval *zv); ZEND_API void destroy_zend_class(zval *zv); diff --git a/Zend/zend_enum.c b/Zend/zend_enum.c index ae9e7b701213f..00fbab22a056e 100644 --- a/Zend/zend_enum.c +++ b/Zend/zend_enum.c @@ -36,6 +36,10 @@ ZEND_API zend_class_entry *zend_ce_unit_enum; ZEND_API zend_class_entry *zend_ce_backed_enum; ZEND_API zend_object_handlers zend_enum_object_handlers; +static zend_arg_info zarginfo_class_UnitEnum_cases[sizeof(arginfo_class_UnitEnum_cases)/sizeof(zend_internal_arg_info)]; +static zend_arg_info zarginfo_class_BackedEnum_from[sizeof(arginfo_class_BackedEnum_from)/sizeof(zend_internal_arg_info)]; +static zend_arg_info zarginfo_class_BackedEnum_tryFrom[sizeof(arginfo_class_BackedEnum_tryFrom)/sizeof(zend_internal_arg_info)]; + zend_object *zend_enum_new(zval *result, zend_class_entry *ce, zend_string *case_name, zval *backing_value_zv) { zend_object *zobj = zend_objects_new(ce); @@ -446,7 +450,7 @@ void zend_enum_register_funcs(zend_class_entry *ce) cases_function->function_name = ZSTR_KNOWN(ZEND_STR_CASES); cases_function->fn_flags = fn_flags; cases_function->doc_comment = NULL; - cases_function->arg_info = (zend_internal_arg_info *) (arginfo_class_UnitEnum_cases + 1); + cases_function->arg_info = zarginfo_class_UnitEnum_cases + 1; zend_enum_register_func(ce, ZEND_STR_CASES, cases_function); if (ce->enum_backing_type != IS_UNDEF) { @@ -457,7 +461,7 @@ void zend_enum_register_funcs(zend_class_entry *ce) from_function->doc_comment = NULL; from_function->num_args = 1; from_function->required_num_args = 1; - from_function->arg_info = (zend_internal_arg_info *) (arginfo_class_BackedEnum_from + 1); + from_function->arg_info = zarginfo_class_BackedEnum_from + 1; zend_enum_register_func(ce, ZEND_STR_FROM, from_function); zend_internal_function *try_from_function = zend_arena_calloc(&CG(arena), sizeof(zend_internal_function), 1); @@ -467,7 +471,7 @@ void zend_enum_register_funcs(zend_class_entry *ce) try_from_function->doc_comment = NULL; try_from_function->num_args = 1; try_from_function->required_num_args = 1; - try_from_function->arg_info = (zend_internal_arg_info *) (arginfo_class_BackedEnum_tryFrom + 1); + try_from_function->arg_info = zarginfo_class_BackedEnum_tryFrom + 1; zend_enum_register_func(ce, ZEND_STR_TRYFROM_LOWERCASE, try_from_function); } } @@ -633,3 +637,16 @@ ZEND_API zend_object *zend_enum_get_case_cstr(zend_class_entry *ce, const char * zend_class_constant *c = zend_hash_str_find_ptr(CE_CONSTANTS_TABLE(ce), name, strlen(name)); return zend_enum_case_from_class_constant(c); } + +void zend_enum_startup(void) +{ + for (size_t i = 0; i < sizeof(zarginfo_class_UnitEnum_cases)/sizeof(zend_arg_info); i++) { + zend_convert_internal_arg_info(&zarginfo_class_UnitEnum_cases[i], &arginfo_class_UnitEnum_cases[i], i == 0, true); + } + for (size_t i = 0; i < sizeof(zarginfo_class_BackedEnum_from)/sizeof(zend_arg_info); i++) { + zend_convert_internal_arg_info(&zarginfo_class_BackedEnum_from[i], &arginfo_class_BackedEnum_from[i], i == 0, true); + } + for (size_t i = 0; i < sizeof(zarginfo_class_BackedEnum_tryFrom)/sizeof(zend_arg_info); i++) { + zend_convert_internal_arg_info(&zarginfo_class_BackedEnum_tryFrom[i], &arginfo_class_BackedEnum_tryFrom[i], i == 0, true); + } +} diff --git a/Zend/zend_enum.h b/Zend/zend_enum.h index 7b3b0184b4eb5..d6c820189475a 100644 --- a/Zend/zend_enum.h +++ b/Zend/zend_enum.h @@ -30,6 +30,7 @@ extern ZEND_API zend_class_entry *zend_ce_unit_enum; extern ZEND_API zend_class_entry *zend_ce_backed_enum; extern ZEND_API zend_object_handlers zend_enum_object_handlers; +void zend_enum_startup(void); void zend_register_enum_ce(void); void zend_enum_add_interfaces(zend_class_entry *ce); zend_result zend_enum_build_backed_enum_table(zend_class_entry *ce); diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index d411dcbc3b953..518cbb98fc0f8 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -136,8 +136,7 @@ static ZEND_FUNCTION(pass) { } -ZEND_BEGIN_ARG_INFO_EX(zend_pass_function_arg_info, 0, 0, 0) -ZEND_END_ARG_INFO() +static zend_arg_info zend_pass_function_arg_info[1] = {0}; ZEND_API const zend_internal_function zend_pass_function = { ZEND_INTERNAL_FUNCTION, /* type */ @@ -148,7 +147,7 @@ ZEND_API const zend_internal_function zend_pass_function = { NULL, /* prototype */ 0, /* num_args */ 0, /* required_num_args */ - (zend_internal_arg_info *) zend_pass_function_arg_info + 1, /* arg_info */ + zend_pass_function_arg_info + 1, /* arg_info */ NULL, /* attributes */ NULL, /* run_time_cache */ NULL, /* doc_comment */ @@ -1480,7 +1479,7 @@ static ZEND_COLD void zend_verify_void_return_error(const zend_function *zf, con ZEND_API bool zend_verify_internal_return_type(const zend_function *zf, zval *ret) { - const zend_internal_arg_info *ret_info = zf->internal_function.arg_info - 1; + const zend_arg_info *ret_info = zf->internal_function.arg_info - 1; if (ZEND_TYPE_FULL_MASK(ret_info->type) & MAY_BE_VOID) { if (UNEXPECTED(Z_TYPE_P(ret) != IS_NULL)) { @@ -5474,28 +5473,17 @@ static zend_always_inline uint32_t zend_get_arg_offset_by_name( // TODO: Use a hash table? uint32_t num_args = fbc->common.num_args; - if (EXPECTED(fbc->type == ZEND_USER_FUNCTION) - || EXPECTED(fbc->common.fn_flags & ZEND_ACC_USER_ARG_INFO)) { - for (uint32_t i = 0; i < num_args; i++) { - const zend_arg_info *arg_info = &fbc->common.arg_info[i]; - if (zend_string_equals(arg_name, arg_info->name)) { - if (fbc->type == ZEND_USER_FUNCTION && (!fbc->op_array.refcount || !(fbc->op_array.fn_flags & ZEND_ACC_CLOSURE))) { - *cache_slot = unique_id; - *(uintptr_t *)(cache_slot + 1) = i; - } - return i; - } - } - } else { - ZEND_ASSERT(num_args == 0 || fbc->internal_function.arg_info); - for (uint32_t i = 0; i < num_args; i++) { - const zend_internal_arg_info *arg_info = &fbc->internal_function.arg_info[i]; - size_t len = strlen(arg_info->name); - if (zend_string_equals_cstr(arg_name, arg_info->name, len)) { + for (uint32_t i = 0; i < num_args; i++) { + const zend_arg_info *arg_info = &fbc->common.arg_info[i]; + if (zend_string_equals(arg_name, arg_info->name)) { + if ((fbc->type == ZEND_USER_FUNCTION + && (!fbc->op_array.refcount || !(fbc->op_array.fn_flags & ZEND_ACC_CLOSURE))) + || (fbc->type == ZEND_INTERNAL_FUNCTION + && !(fbc->common.fn_flags & ZEND_ACC_NEVER_CACHE))) { *cache_slot = unique_id; *(uintptr_t *)(cache_slot + 1) = i; - return i; } + return i; } } @@ -5503,7 +5491,7 @@ static zend_always_inline uint32_t zend_get_arg_offset_by_name( if ((fbc->type == ZEND_USER_FUNCTION && (!fbc->op_array.refcount || !(fbc->op_array.fn_flags & ZEND_ACC_CLOSURE))) || (fbc->type == ZEND_INTERNAL_FUNCTION - && !(fbc->common.fn_flags & ZEND_ACC_USER_ARG_INFO))) { + && !(fbc->common.fn_flags & ZEND_ACC_NEVER_CACHE))) { *cache_slot = unique_id; *(uintptr_t *)(cache_slot + 1) = fbc->common.num_args; } @@ -5661,7 +5649,7 @@ ZEND_API zend_result ZEND_FASTCALL zend_handle_undef_args(zend_execute_data *cal continue; } - zend_internal_arg_info *arg_info = &fbc->internal_function.arg_info[i]; + zend_arg_info *arg_info = &fbc->internal_function.arg_info[i]; if (i < fbc->common.required_num_args) { zend_execute_data *old = start_fake_frame(call, NULL); zend_argument_error(zend_ce_argument_count_error, i + 1, "not passed"); diff --git a/Zend/zend_execute_API.c b/Zend/zend_execute_API.c index 660975f9bc1b5..e134d3d496b6d 100644 --- a/Zend/zend_execute_API.c +++ b/Zend/zend_execute_API.c @@ -652,11 +652,7 @@ ZEND_API const char *get_function_arg_name(const zend_function *func, uint32_t a return NULL; } - if (func->type == ZEND_USER_FUNCTION || (func->common.fn_flags & ZEND_ACC_USER_ARG_INFO)) { - return ZSTR_VAL(func->common.arg_info[arg_num - 1].name); - } else { - return ((zend_internal_arg_info*) func->common.arg_info)[arg_num - 1].name; - } + return ZSTR_VAL(func->common.arg_info[arg_num - 1].name); } /* }}} */ diff --git a/Zend/zend_inheritance.c b/Zend/zend_inheritance.c index 64448bc535590..3c3931cdca164 100644 --- a/Zend/zend_inheritance.c +++ b/Zend/zend_inheritance.c @@ -953,18 +953,14 @@ static ZEND_COLD zend_string *zend_get_function_declaration( } smart_str_appendc(&str, '$'); - if (fptr->type == ZEND_INTERNAL_FUNCTION) { - smart_str_appends(&str, ((zend_internal_arg_info*)arg_info)->name); - } else { - smart_str_appendl(&str, ZSTR_VAL(arg_info->name), ZSTR_LEN(arg_info->name)); - } + smart_str_append(&str, arg_info->name); if (i >= required && !ZEND_ARG_IS_VARIADIC(arg_info)) { smart_str_appends(&str, " = "); if (fptr->type == ZEND_INTERNAL_FUNCTION) { - if (((zend_internal_arg_info*)arg_info)->default_value) { - smart_str_appends(&str, ((zend_internal_arg_info*)arg_info)->default_value); + if (arg_info->default_value) { + smart_str_append(&str, arg_info->default_value); } else { smart_str_appends(&str, ""); } diff --git a/Zend/zend_object_handlers.c b/Zend/zend_object_handlers.c index 470fb76ec14e1..88b7b1112d7b1 100644 --- a/Zend/zend_object_handlers.c +++ b/Zend/zend_object_handlers.c @@ -46,6 +46,9 @@ #define IN_ISSET ZEND_GUARD_PROPERTY_ISSET #define IN_HOOK ZEND_GUARD_PROPERTY_HOOK +static zend_arg_info zend_call_trampoline_arginfo[1] = {{0}}; +static zend_arg_info zend_property_hook_arginfo[1] = {{0}}; + static zend_always_inline bool zend_objects_check_stack_limit(void) { #ifdef ZEND_CHECK_STACK_LIMIT @@ -1782,9 +1785,6 @@ ZEND_API zend_function *zend_get_property_hook_trampoline( const zend_property_info *prop_info, zend_property_hook_kind kind, zend_string *prop_name) { - static const zend_internal_arg_info arg_info[2] = { - { .name = "value" } - }; zend_function *func; if (EXPECTED(EG(trampoline).common.function_name == NULL)) { func = &EG(trampoline); @@ -1810,7 +1810,7 @@ ZEND_API zend_function *zend_get_property_hook_trampoline( func->common.scope = prop_info->ce; func->common.prototype = NULL; func->common.prop_info = prop_info; - func->common.arg_info = (zend_arg_info *) arg_info; + func->common.arg_info = zend_property_hook_arginfo; func->internal_function.handler = kind == ZEND_PROPERTY_HOOK_GET ? ZEND_FN(zend_parent_hook_get_trampoline) : ZEND_FN(zend_parent_hook_set_trampoline); @@ -2574,3 +2574,8 @@ ZEND_API const zend_object_handlers std_object_handlers = { zend_std_compare_objects, /* compare */ NULL, /* get_properties_for */ }; + +void zend_object_handlers_startup(void) { + zend_call_trampoline_arginfo[0].name = ZSTR_KNOWN(ZEND_STR_ARGS); + zend_property_hook_arginfo[0].name = ZSTR_KNOWN(ZEND_STR_VALUE); +} diff --git a/Zend/zend_object_handlers.h b/Zend/zend_object_handlers.h index 59277c09d8024..3e922343eb15a 100644 --- a/Zend/zend_object_handlers.h +++ b/Zend/zend_object_handlers.h @@ -334,6 +334,8 @@ ZEND_API zend_function *zend_get_property_hook_trampoline( ZEND_API bool ZEND_FASTCALL zend_asymmetric_property_has_set_access(const zend_property_info *prop_info); +void zend_object_handlers_startup(void); + #define zend_release_properties(ht) do { \ if (ht) { \ zend_array_release(ht); \ diff --git a/Zend/zend_opcode.c b/Zend/zend_opcode.c index 1962c7b5a56d1..5e9e7b20d869b 100644 --- a/Zend/zend_opcode.c +++ b/Zend/zend_opcode.c @@ -124,21 +124,32 @@ ZEND_API void zend_type_release(zend_type type, bool persistent) { } } -void zend_free_internal_arg_info(zend_internal_function *function) { - if ((function->fn_flags & (ZEND_ACC_HAS_RETURN_TYPE|ZEND_ACC_HAS_TYPE_HINTS)) && - function->arg_info) { +ZEND_API void zend_free_internal_arg_info(zend_internal_function *function, + bool persistent) { + if (function->arg_info) { + ZEND_ASSERT((persistent || (function->fn_flags & ZEND_ACC_NEVER_CACHE)) + && "Functions with non-persistent arg_info must be flagged ZEND_ACC_NEVER_CACHE"); uint32_t i; uint32_t num_args = function->num_args + 1; - zend_internal_arg_info *arg_info = function->arg_info - 1; + zend_arg_info *arg_info = function->arg_info - 1; if (function->fn_flags & ZEND_ACC_VARIADIC) { num_args++; } for (i = 0 ; i < num_args; i++) { - zend_type_release(arg_info[i].type, /* persistent */ true); + bool is_return_info = i == 0; + if (!is_return_info) { + zend_string_release_ex(arg_info[i].name, persistent); + if (arg_info[i].default_value) { + zend_string_release_ex(arg_info[i].default_value, + persistent); + } + } + zend_type_release(arg_info[i].type, persistent); } - free(arg_info); + + pefree(arg_info, persistent); } } @@ -157,7 +168,7 @@ ZEND_API void zend_function_dtor(zval *zv) /* For methods this will be called explicitly. */ if (!function->common.scope) { - zend_free_internal_arg_info(&function->internal_function); + zend_free_internal_arg_info(&function->internal_function, true); if (function->common.attributes) { zend_hash_release(function->common.attributes); @@ -474,12 +485,9 @@ ZEND_API void destroy_zend_class(zval *zv) zend_hash_destroy(&ce->properties_info); zend_string_release_ex(ce->name, 1); - /* TODO: eliminate this loop for classes without functions with arg_info / attributes */ ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, fn) { if (fn->common.scope == ce) { - if (fn->common.fn_flags & (ZEND_ACC_HAS_RETURN_TYPE|ZEND_ACC_HAS_TYPE_HINTS)) { - zend_free_internal_arg_info(&fn->internal_function); - } + zend_free_internal_arg_info(&fn->internal_function, true); if (fn->common.attributes) { zend_hash_release(fn->common.attributes); diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index d1d49ca391a69..7acb14b778f8d 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -663,8 +663,7 @@ static void accel_copy_permanent_strings(zend_new_interned_string_func_t new_int if (Z_FUNC(p->val)->common.function_name) { Z_FUNC(p->val)->common.function_name = new_interned_string(Z_FUNC(p->val)->common.function_name); } - if (Z_FUNC(p->val)->common.arg_info && - (Z_FUNC(p->val)->common.fn_flags & (ZEND_ACC_HAS_RETURN_TYPE|ZEND_ACC_HAS_TYPE_HINTS))) { + if (Z_FUNC(p->val)->common.arg_info) { uint32_t i; uint32_t num_args = Z_FUNC(p->val)->common.num_args + 1; zend_arg_info *arg_info = Z_FUNC(p->val)->common.arg_info - 1; @@ -673,6 +672,12 @@ static void accel_copy_permanent_strings(zend_new_interned_string_func_t new_int num_args++; } for (i = 0 ; i < num_args; i++) { + if (i > 0) { + arg_info[i].name = new_interned_string(arg_info[i].name); + if (arg_info[i].default_value) { + arg_info[i].default_value = new_interned_string(arg_info[i].default_value); + } + } accel_copy_permanent_list_types(new_interned_string, arg_info[i].type); } } @@ -714,6 +719,24 @@ static void accel_copy_permanent_strings(zend_new_interned_string_func_t new_int if (Z_FUNC(q->val)->common.function_name) { Z_FUNC(q->val)->common.function_name = new_interned_string(Z_FUNC(q->val)->common.function_name); } + if (Z_FUNC(q->val)->common.scope == ce) { + uint32_t i; + uint32_t num_args = Z_FUNC(q->val)->common.num_args + 1; + zend_arg_info *arg_info = Z_FUNC(q->val)->common.arg_info - 1; + + if (Z_FUNC(q->val)->common.fn_flags & ZEND_ACC_VARIADIC) { + num_args++; + } + for (i = 0 ; i < num_args; i++) { + if (i > 0) { + arg_info[i].name = new_interned_string(arg_info[i].name); + if (arg_info[i].default_value) { + arg_info[i].default_value = new_interned_string(arg_info[i].default_value); + } + } + accel_copy_permanent_list_types(new_interned_string, arg_info[i].type); + } + } } ZEND_HASH_FOREACH_END(); ZEND_HASH_MAP_FOREACH_BUCKET(&ce->constants_table, q) { diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 81621ce11bd36..745a66b2163ae 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -118,7 +118,7 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted { char buf[128]; - if (insn->op == IR_FUNC || insn->op == IR_SYM) { + if (insn->op == IR_FUNC || insn->op == IR_SYM || insn->op == IR_LABEL) { fprintf(f, "%s", ir_get_str(ctx, insn->val.name)); return; } else if (insn->op == IR_STR) { @@ -290,6 +290,7 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted #define ir_op_kind_prb IR_OPND_PROB #define ir_op_kind_opt IR_OPND_PROB #define ir_op_kind_pro IR_OPND_PROTO +#define ir_op_kind_lbl IR_OPND_LABEL_REF #define _IR_OP_FLAGS(name, flags, op1, op2, op3) \ IR_OP_FLAGS(ir_op_flag_ ## flags, ir_op_kind_ ## op1, ir_op_kind_ ## op2, ir_op_kind_ ## op3), @@ -689,6 +690,13 @@ ir_ref ir_const_str(ir_ctx *ctx, ir_ref str) return ir_const_ex(ctx, val, IR_ADDR, IR_OPTX(IR_STR, IR_ADDR, 0)); } +ir_ref ir_const_label(ir_ctx *ctx, ir_ref str) +{ + ir_val val; + val.u64 = str; + return ir_const_ex(ctx, val, IR_ADDR, IR_OPTX(IR_LABEL, IR_ADDR, 0)); +} + ir_ref ir_str(ir_ctx *ctx, const char *s) { size_t len; @@ -879,6 +887,17 @@ static ir_ref _ir_fold_cse(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir return IR_UNUSED; } +IR_ALWAYS_INLINE ir_ref _ir_fold_cast(ir_ctx *ctx, ir_ref ref, ir_type type) +{ + if (ctx->ir_base[ref].type == type) { + return ref; + } else if (IR_IS_CONST_REF(ref) && !IR_IS_SYM_CONST(ctx->ir_base[ref].op)) { + return ir_const(ctx, ctx->ir_base[ref].val, type); + } else { + return ir_emit1(ctx, IR_OPT(IR_BITCAST, type), ref); + } +} + #define IR_FOLD(X) IR_FOLD1(X, __LINE__) #define IR_FOLD1(X, Y) IR_FOLD2(X, Y) #define IR_FOLD2(X, Y) case IR_RULE_ ## Y: @@ -1158,7 +1177,7 @@ ir_ref ir_bind(ir_ctx *ctx, ir_ref var, ir_ref def) IR_ASSERT(var < 0); if (!ir_hashtab_add(ctx->binding, def, var)) { /* Add a copy with different binding */ - def = ir_emit2(ctx, IR_OPT(IR_COPY, ctx->ir_base[def].type), def, 1); + def = ir_emit2(ctx, IR_OPT(IR_COPY, ctx->ir_base[def].type), def, IR_COPY_HARD); ir_hashtab_add(ctx->binding, def, var); } return def; @@ -1836,8 +1855,49 @@ int ir_mem_flush(void *ptr, size_t size) return 1; } #else + +#if defined(__linux__) && defined(__x86_64__) && defined(PKEY_DISABLE_WRITE) +# define HAVE_PKEY_MPROTECT 1 +#endif + +#ifdef HAVE_PKEY_MPROTECT + +#ifndef PKEY_DISABLE_EXECUTE +# define PKEY_DISABLE_EXECUTE 0 +#endif + +int pkey_mprotect(void* addr, size_t len, int prot, int pkey) __attribute__((weak)); +int pkey_alloc(unsigned int, unsigned int) __attribute__((weak)); +int pkey_free(int) __attribute__((weak)); +int pkey_set(int, unsigned) __attribute__((weak)); + +static int ir_pkey = 0; +#endif + void *ir_mem_mmap(size_t size) { +#ifdef HAVE_PKEY_MPROTECT + if (!ir_pkey && pkey_mprotect) { + int key = pkey_alloc(0, PKEY_DISABLE_WRITE); + if (key > 0) { + ir_pkey = key; + } + } + if (ir_pkey > 0) { + void *ret = mmap(NULL, size, PROT_EXEC|PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ret == MAP_FAILED) { + return NULL; + } + if (pkey_mprotect(ret, size, PROT_EXEC|PROT_READ|PROT_WRITE, ir_pkey) != 0) { +#ifdef IR_DEBUG + fprintf(stderr, "pkey_mprotect() failed\n"); +#endif + munmap(ret, size); + return NULL; + } + return ret; + } +#endif int prot_flags = PROT_EXEC; #if defined(__NetBSD__) prot_flags |= PROT_MPROTECT(PROT_READ|PROT_WRITE); @@ -1852,11 +1912,28 @@ void *ir_mem_mmap(size_t size) int ir_mem_unmap(void *ptr, size_t size) { munmap(ptr, size); +#ifdef HAVE_PKEY_MPROTECT +// if (ir_pkey > 0) { +// pkey_free(ir_pkey); +// ir_pkey = 0; +// } +#endif return 1; } int ir_mem_protect(void *ptr, size_t size) { +#ifdef HAVE_PKEY_MPROTECT + if (ir_pkey > 0) { + if (pkey_set(ir_pkey, PKEY_DISABLE_WRITE)) { +#ifdef IR_DEBUG + fprintf(stderr, "mprotect() failed\n"); +#endif + return 0; + } + return 1; + } +#endif if (mprotect(ptr, size, PROT_READ | PROT_EXEC) != 0) { #ifdef IR_DEBUG fprintf(stderr, "mprotect() failed\n"); @@ -1868,6 +1945,17 @@ int ir_mem_protect(void *ptr, size_t size) int ir_mem_unprotect(void *ptr, size_t size) { +#ifdef HAVE_PKEY_MPROTECT + if (ir_pkey > 0) { + if (pkey_set(ir_pkey, PKEY_DISABLE_EXECUTE)) { +#ifdef IR_DEBUG + fprintf(stderr, "mprotect() failed\n"); +#endif + return 0; + } + return 1; + } +#endif if (mprotect(ptr, size, PROT_READ | PROT_WRITE) != 0) { #ifdef IR_DEBUG fprintf(stderr, "mprotect() failed\n"); @@ -2070,7 +2158,26 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_load_i(ir_ctx *ctx, ir_ref ref, ir_type } } else if (insn->op == IR_RSTORE) { modified_regset |= (1 << insn->op3); - } else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_CALL || insn->op == IR_VSTORE) { + } else if (insn->op == IR_CALL) { + ir_insn *func = &ctx->ir_base[insn->op2]; + ir_ref func_proto; + const ir_proto_t *proto; + + if (func->op == IR_FUNC || func->op == IR_FUNC_ADDR) { + func_proto = func->proto; + } else if (func->op == IR_PROTO) { + func_proto = func->op2; + } else { + break; + } + if (!func_proto) { + break; + } + proto = (const ir_proto_t *)ir_get_str(ctx, func_proto); + if (!(proto->flags & (IR_CONST_FUNC|IR_PURE_FUNC))) { + break; + } + } else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_VSTORE) { return IR_UNUSED; } ref = insn->op1; @@ -2116,7 +2223,26 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vload_i(ir_ctx *ctx, ir_ref ref, ir_typ break; } } - } else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_CALL || insn->op == IR_STORE) { + } else if (insn->op == IR_CALL) { + ir_insn *func = &ctx->ir_base[insn->op2]; + ir_ref func_proto; + const ir_proto_t *proto; + + if (func->op == IR_FUNC || func->op == IR_FUNC_ADDR) { + func_proto = func->proto; + } else if (func->op == IR_PROTO) { + func_proto = func->op2; + } else { + break; + } + if (!func_proto) { + break; + } + proto = (const ir_proto_t *)ir_get_str(ctx, func_proto); + if (!(proto->flags & (IR_CONST_FUNC|IR_PURE_FUNC))) { + break; + } + } else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_STORE) { break; } ref = insn->op1; @@ -3013,6 +3139,16 @@ void _ir_IJMP(ir_ctx *ctx, ir_ref addr) ctx->control = IR_UNUSED; } +ir_ref _ir_IGOTO(ir_ctx *ctx, ir_ref addr) +{ + ir_ref ref; + + IR_ASSERT(ctx->control); + ctx->control = ref = ir_emit2(ctx, IR_IGOTO, ctx->control, addr); + ctx->control = IR_UNUSED; + return ref; +} + ir_ref _ir_ADD_OFFSET(ir_ctx *ctx, ir_ref addr, uintptr_t offset) { if (offset) { @@ -3135,6 +3271,18 @@ void _ir_VSTORE(ir_ctx *ctx, ir_ref var, ir_ref val) ctx->control = ir_emit3(ctx, IR_VSTORE, ctx->control, var, val); } +ir_ref _ir_VLOAD_v(ir_ctx *ctx, ir_type type, ir_ref var) +{ + IR_ASSERT(ctx->control); + return ctx->control = ir_emit2(ctx, IR_OPT(IR_VLOAD_v, type), ctx->control, var); +} + +void _ir_VSTORE_v(ir_ctx *ctx, ir_ref var, ir_ref val) +{ + IR_ASSERT(ctx->control); + ctx->control = ir_emit3(ctx, IR_VSTORE_v, ctx->control, var, val); +} + ir_ref _ir_TLS(ir_ctx *ctx, ir_ref index, ir_ref offset) { IR_ASSERT(ctx->control); @@ -3193,6 +3341,18 @@ void _ir_STORE(ir_ctx *ctx, ir_ref addr, ir_ref val) ctx->control = ir_emit3(ctx, IR_STORE, ctx->control, addr, val); } +ir_ref _ir_LOAD_v(ir_ctx *ctx, ir_type type, ir_ref addr) +{ + IR_ASSERT(ctx->control); + return ctx->control = ir_emit2(ctx, IR_OPT(IR_LOAD_v, type), ctx->control, addr); +} + +void _ir_STORE_v(ir_ctx *ctx, ir_ref addr, ir_ref val) +{ + IR_ASSERT(ctx->control); + ctx->control = ir_emit3(ctx, IR_STORE_v, ctx->control, addr, val); +} + void _ir_VA_START(ir_ctx *ctx, ir_ref list) { IR_ASSERT(ctx->control); @@ -3217,11 +3377,13 @@ ir_ref _ir_VA_ARG(ir_ctx *ctx, ir_type type, ir_ref list) return ctx->control = ir_emit2(ctx, IR_OPT(IR_VA_ARG, type), ctx->control, list); } -ir_ref _ir_VA_ARG_EX(ir_ctx *ctx, ir_type type, ir_ref list, size_t size) +ir_ref _ir_VA_ARG_EX(ir_ctx *ctx, ir_type type, ir_ref list, size_t size, size_t align) { IR_ASSERT(ctx->control); - IR_ASSERT(size <= 0x7fffffff); - return ctx->control = ir_emit3(ctx, IR_OPT(IR_VA_ARG, type), ctx->control, list, (ir_ref)size); + IR_ASSERT(size <= 0x0fffffff); + IR_ASSERT(align != 0 && ((align & (align - 1)) == 0) && align <= 128); + return ctx->control = ir_emit3(ctx, IR_OPT(IR_VA_ARG, type), ctx->control, list, + (ir_ref)IR_VA_ARG_OP3(size, align)); } ir_ref _ir_BLOCK_BEGIN(ir_ctx *ctx) diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 8fcfbffa7d6bc..a96650597055a 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -216,6 +216,7 @@ typedef enum _ir_type { * prb - branch probability 1-99 (0 - unspecified): (IF_TRUE, IF_FALSE, CASE_VAL, CASE_DEFAULT) * opt - optional number * pro - function prototype + * lbl - label used as value (a reference to constant): (BEGIN) * * The order of IR opcodes is carefully selected for efficient folding. * - foldable instruction go first @@ -322,6 +323,7 @@ typedef enum _ir_type { _(FUNC_ADDR, r0, ___, ___, ___) /* constant func ref */ \ _(FUNC, r0, ___, ___, ___) /* constant func ref */ \ _(SYM, r0, ___, ___, ___) /* constant symbol ref */ \ + _(LABEL, r0, ___, ___, ___) /* label address ref */ \ _(STR, r0, ___, ___, ___) /* constant str ref */ \ \ /* call ops */ \ @@ -334,11 +336,15 @@ typedef enum _ir_type { _(BLOCK_BEGIN, a1, src, ___, ___) /* stacksave */ \ _(BLOCK_END, a2, src, def, ___) /* stackrestore */ \ _(VLOAD, l2, src, var, ___) /* load value of local var */ \ + _(VLOAD_v, l2, src, var, ___) /* volatile variant of VLOAD */ \ _(VSTORE, s3, src, var, def) /* store value to local var */ \ + _(VSTORE_v, s3, src, var, def) /* volatile variant of VSTORE */ \ _(RLOAD, l1X2, src, num, opt) /* load value from register */ \ _(RSTORE, s2X1, src, def, num) /* store value into register */ \ _(LOAD, l2, src, ref, ___) /* load from memory */ \ + _(LOAD_v, l2, src, ref, ___) /* volatile variant of VLOAD */ \ _(STORE, s3, src, ref, def) /* store to memory */ \ + _(STORE_v, s3, src, ref, def) /* volatile variant of VSTORE */ \ _(TLS, l1X2, src, num, num) /* thread local variable */ \ _(TRAP, x1, src, ___, ___) /* DebugBreak */ \ /* memory reference ops (A, H, U, S, TMP, STR, NEW, X, V) ??? */ \ @@ -360,7 +366,7 @@ typedef enum _ir_type { /* control-flow nodes */ \ _(START, S0X1, ret, ___, ___) /* function start */ \ _(ENTRY, S1X1, src, num, ___) /* entry with a fake src edge */ \ - _(BEGIN, S1, src, ___, ___) /* block start */ \ + _(BEGIN, S1X1, src, lbl, ___) /* block start, optional &&lbl */ \ _(IF_TRUE, S1X1, src, prb, ___) /* IF TRUE proj. */ \ _(IF_FALSE, S1X1, src, prb, ___) /* IF FALSE proj. */ \ _(CASE_VAL, S2X1, src, def, prb) /* switch proj. */ \ @@ -372,8 +378,9 @@ typedef enum _ir_type { _(LOOP_END, E1, src, ___, ___) /* loop end */ \ _(IF, E2, src, def, ___) /* conditional control split */ \ _(SWITCH, E2, src, def, ___) /* multi-way control split */ \ + _(IGOTO, E2, src, def, ___) /* computed goto (internal) */ \ + _(IJMP, T2X1, src, def, ret) /* computed goto (terminating) */ \ _(RETURN, T2X1, src, def, ret) /* function return */ \ - _(IJMP, T2X1, src, def, ret) /* computed goto */ \ _(UNREACHABLE, T1X2, src, ___, ret) /* unreachable (tailcall, etc) */ \ \ /* deoptimization helper */ \ @@ -400,6 +407,13 @@ typedef enum _ir_op { #define IR_OPTX(op, type, n) ((uint32_t)(op) | ((uint32_t)(type) << IR_OPT_TYPE_SHIFT) | ((uint32_t)(n) << IR_OPT_INPUTS_SHIFT)) #define IR_OPT_TYPE(opt) (((opt) & IR_OPT_TYPE_MASK) >> IR_OPT_TYPE_SHIFT) +/* "opt" modifiers */ +#define IR_COPY_HARD (1<<0) + +#define IR_VA_ARG_SIZE(op3) (((uint32_t)(op3) >> 3)) +#define IR_VA_ARG_ALIGN(op3) (1U << ((uint32_t)(op3) & 0x7)) +#define IR_VA_ARG_OP3(s, a) (((s) << 3) | ir_ntzl(a)) + /* IR References */ typedef int32_t ir_ref; @@ -533,6 +547,9 @@ void ir_strtab_free(ir_strtab *strtab); #define IR_EXTERN (1<<5) #define IR_CONST (1<<6) +#define IR_CONST_FUNC (1<<6) +#define IR_PURE_FUNC (1<<7) + #define IR_INITIALIZED (1<<7) /* sym data flag: constant or an initialized variable */ #define IR_CONST_STRING (1<<8) /* sym data flag: constant string */ @@ -648,7 +665,6 @@ struct _ir_ctx { ir_ref vars; /* list of VARs (used by register allocator) */ }; ir_snapshot_create_t snapshot_create; - int32_t stack_frame_alignment; int32_t stack_frame_size; /* spill stack frame size (used by register allocator and code generator) */ int32_t call_stack_size; /* stack for parameter passing (used by register allocator and code generator) */ uint64_t used_preserved_regs; @@ -698,6 +714,7 @@ ir_ref ir_const_func_addr(ir_ctx *ctx, uintptr_t c, ir_ref proto); ir_ref ir_const_func(ir_ctx *ctx, ir_ref str, ir_ref proto); ir_ref ir_const_sym(ir_ctx *ctx, ir_ref str); ir_ref ir_const_str(ir_ctx *ctx, ir_ref str); +ir_ref ir_const_label(ir_ctx *ctx, ir_ref str); ir_ref ir_unique_const_addr(ir_ctx *ctx, uintptr_t c); @@ -893,6 +910,7 @@ struct _ir_loader { void*(*resolve_sym_name) (ir_loader *loader, const char *name, uint32_t flags); bool (*has_sym) (ir_loader *loader, const char *name); bool (*add_sym) (ir_loader *loader, const char *name, void *addr); + bool (*add_label) (ir_loader *loader, const char *name, void *addr); }; void ir_loader_init(void); diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 12c3694d469f0..b553243309f54 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -218,6 +218,7 @@ typedef struct _ir_backend_data { dasm_State *dasm_state; ir_bitset emit_constants; int rodata_label, jmp_table_label; + bool resolved_label_syms; } ir_backend_data; #define IR_GP_REG_NAME(code, name64, name32) \ @@ -315,6 +316,7 @@ const char *ir_reg_name(int8_t reg, ir_type type) _(RETURN_VOID) \ _(RETURN_INT) \ _(RETURN_FP) \ + _(IGOTO_DUP) \ #define IR_RULE_ENUM(name) IR_ ## name, @@ -385,7 +387,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co n++; break; } - } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + } else if (!IR_IS_CONST_REF(insn->op2) && ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { constraints->tmp_regs[n] = IR_TMP_REG(2, IR_ADDR, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; } @@ -478,10 +480,16 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co if (IR_IS_CONST_REF(insn->op1)) { constraints->tmp_regs[n] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; + } else if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) { constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n++; + } else if (!IR_IS_CONST_REF(insn->op2) && ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) { + constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); + n++; } break; case IR_CMP_INT: @@ -520,6 +528,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co } break; case IR_VSTORE: + case IR_VSTORE_v: insn = &ctx->ir_base[ref]; if (IR_IS_CONST_REF(insn->op3)) { insn = &ctx->ir_base[insn->op3]; @@ -596,6 +605,19 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co } flags = IR_USE_SHOULD_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_OP3_SHOULD_BE_IN_REG; break; + case IR_IGOTO: + insn = &ctx->ir_base[ref]; + if (ctx->ir_base[insn->op1].op == IR_MERGE || ctx->ir_base[insn->op1].op == IR_LOOP_BEGIN) { + ir_insn *merge = &ctx->ir_base[insn->op1]; + ir_ref *p, n = merge->inputs_count; + + for (p = merge->ops + 1; n > 0; p++, n--) { + ir_ref input = *p; + IR_ASSERT(ctx->ir_base[input].op == IR_END || ctx->ir_base[input].op == IR_LOOP_END); + ctx->rules[input] = IR_IGOTO_DUP; + } + } + return insn->op; case IR_COND: insn = &ctx->ir_base[ref]; n = 0; @@ -665,7 +687,7 @@ int ir_get_target_constraints(ir_ctx *ctx, ir_ref ref, ir_target_constraints *co } break; case IR_VA_ARG: - flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; + flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS; constraints->tmp_regs[0] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n = 1; insn = &ctx->ir_base[ref]; @@ -714,7 +736,8 @@ static void ir_match_fuse_addr(ir_ctx *ctx, ir_ref addr_ref, ir_type type) do { ir_insn *insn = &ctx->ir_base[*p]; - if (insn->op != IR_LOAD && (insn->op != IR_STORE || insn->op3 == addr_ref)) { + if (insn->op != IR_LOAD && insn->op != IR_LOAD_v + && ((insn->op != IR_STORE && insn->op != IR_STORE_v) || insn->op3 == addr_ref)) { return; } p++; @@ -961,7 +984,7 @@ binop_fp: ctx->flags2 |= IR_HAS_CALLS; return IR_CALL; case IR_VAR: - return IR_SKIPPED | IR_VAR; + return IR_STATIC_ALLOCA; case IR_PARAM: return ctx->use_lists[ref].count > 0 ? IR_PARAM : IR_SKIPPED | IR_PARAM; case IR_ALLOCA: @@ -978,6 +1001,7 @@ binop_fp: } return IR_ALLOCA; case IR_LOAD: + case IR_LOAD_v: ir_match_fuse_addr(ctx, insn->op2, insn->type); if (IR_IS_TYPE_INT(insn->type)) { return IR_LOAD_INT; @@ -986,6 +1010,7 @@ binop_fp: } break; case IR_STORE: + case IR_STORE_v: ir_match_fuse_addr(ctx, insn->op2, ctx->ir_base[insn->op3].type); if (IR_IS_TYPE_INT(ctx->ir_base[insn->op3].type)) { return IR_STORE_INT; @@ -1364,7 +1389,7 @@ static void ir_emit_load_imm_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref sr } else if (type == IR_DOUBLE && insn->val.u64 == 0) { | fmov Rd(reg-IR_REG_FP_FIRST), xzr } else { - label = ir_const_label(ctx, src); + label = ir_get_const_label(ctx, src); if (type == IR_DOUBLE) { | ldr Rd(reg-IR_REG_FP_FIRST), =>label } else { @@ -1441,10 +1466,41 @@ static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) | add Rx(reg), Rx(base), #offset } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); - | add Rx(reg), sp, Rx(IR_REG_INT_TMP) + | add Rx(reg), Rx(base), Rx(IR_REG_INT_TMP) } } +static void ir_resolve_label_syms(ir_ctx *ctx) +{ + uint32_t b; + ir_block *bb; + + for (b = 1, bb = &ctx->cfg_blocks[b]; b <= ctx->cfg_blocks_count; bb++, b++) { + ir_insn *insn = &ctx->ir_base[bb->start]; + + if (insn->op == IR_BEGIN && insn->op2) { + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_LABEL); + ctx->ir_base[insn->op2].val.u32_hi = b; + } + } +} + +static void ir_emit_load_label_addr(ir_ctx *ctx, ir_reg reg, ir_insn *label) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + + if (!data->resolved_label_syms) { + data->resolved_label_syms = 1; + ir_resolve_label_syms(ctx); + } + + IR_ASSERT(label->op == IR_LABEL); + int b = label->val.u32_hi; + + b = ir_skip_empty_target_blocks(ctx, b); + | adr Rx(reg), =>b +} static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) { @@ -1459,9 +1515,11 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else if (insn->op == IR_STR) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; - int label = ir_const_label(ctx, src); + int label = ir_get_const_label(ctx, src); | adr Rx(reg), =>label + } else if (insn->op == IR_LABEL) { + ir_emit_load_label_addr(ctx, reg, insn); } else { ir_emit_load_imm_int(ctx, type, reg, insn->val.i64); } @@ -1697,6 +1755,7 @@ static void ir_emit_prologue(ir_ctx *ctx) | str Rd(i-IR_REG_FP_FIRST), [Rx(fp), #offset] } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + offset -= sizeof(void*); | str Rx(prev), [Rx(fp), Rx(IR_REG_INT_TMP)] | sub Rx(IR_REG_INT_TMP), Rx(IR_REG_INT_TMP), #8 | str Rd(i-IR_REG_FP_FIRST), [Rx(fp), Rx(IR_REG_INT_TMP)] @@ -1795,7 +1854,12 @@ static void ir_emit_prologue(ir_ctx *ctx) offset += 16 * ctx->fp_reg_params; for (i = ctx->fp_reg_params; i < IR_REG_FP_ARGS; i++) { // TODO: Rd->Rq stur->str ??? - | str Rd(fp_reg_params[i]-IR_REG_FP_FIRST), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, 8)) { + | str Rd(fp_reg_params[i]-IR_REG_FP_FIRST), [Rx(fp), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | str Rd(fp_reg_params[i]-IR_REG_FP_FIRST), [Rx(fp), Rx(IR_REG_INT_TMP)] + } offset += 16; } } @@ -1828,26 +1892,44 @@ static void ir_emit_epilogue(ir_ctx *ctx) offset -= sizeof(void*) * 2; if (aarch64_may_encode_imm7_addr_offset(offset, 8)) { | ldp Rx(prev), Rx(i), [Rx(fp), #offset] - } else { - IR_ASSERT(aarch64_may_encode_addr_offset(offset, 8)); + } else if (aarch64_may_encode_addr_offset(offset + 8, 8)) { | ldr Rx(prev), [Rx(fp), #offset] | ldr Rx(i), [Rx(fp), #(offset+8)] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | ldr Rx(prev), [Rx(fp), Rx(IR_REG_INT_TMP)] + | add Rx(IR_REG_INT_TMP), Rx(IR_REG_INT_TMP), #8 + | ldr Rx(i), [Rx(fp), Rx(IR_REG_INT_TMP)] } prev = IR_REG_NONE; } else { if (prev < IR_REG_FP_FIRST) { offset -= sizeof(void*); - | ldr Rx(prev), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, 8)) { + | ldr Rx(prev), [Rx(fp), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | ldr Rx(prev), [Rx(fp), Rx(IR_REG_INT_TMP)] + } offset -= sizeof(void*); - | ldr Rd(i-IR_REG_FP_FIRST), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, 8)) { + | ldr Rd(i-IR_REG_FP_FIRST), [Rx(fp), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | ldr Rd(i-IR_REG_FP_FIRST), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } else { offset -= sizeof(void*) * 2; if (aarch64_may_encode_imm7_addr_offset(offset, 8)) { | ldp Rd(prev-IR_REG_FP_FIRST), Rd(i-IR_REG_FP_FIRST), [Rx(fp), #offset] - } else { - IR_ASSERT(aarch64_may_encode_addr_offset(offset, 8)); + } else if (aarch64_may_encode_addr_offset(offset + 8, 8)) { | ldr Rd(prev-IR_REG_FP_FIRST), [Rx(fp), #offset] | ldr Rd(i-IR_REG_FP_FIRST), [Rx(fp), #(offset+8)] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | ldr Rx(prev-IR_REG_FP_FIRST), [Rx(fp), Rx(IR_REG_INT_TMP)] + | add Rx(IR_REG_INT_TMP), Rx(IR_REG_INT_TMP), #8 + | ldr Rx(i-IR_REG_FP_FIRST), [Rx(fp), Rx(IR_REG_INT_TMP)] } } prev = IR_REG_NONE; @@ -1857,10 +1939,20 @@ static void ir_emit_epilogue(ir_ctx *ctx) if (prev != IR_REG_NONE) { if (prev < IR_REG_FP_FIRST) { offset -= sizeof(void*); - | ldr Rx(prev), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, 8)) { + | ldr Rx(prev), [Rx(fp), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | ldr Rx(prev), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } else { offset -= sizeof(void*); - | ldr Rd(prev-IR_REG_FP_FIRST), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, 8)) { + | ldr Rd(prev-IR_REG_FP_FIRST), [Rx(fp), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | ldr Rd(prev-IR_REG_FP_FIRST), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } } } @@ -1909,6 +2001,9 @@ static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) op1_reg = IR_REG_NUM(op1_reg); ir_emit_load(ctx, type, op1_reg, op1); } + if (op2_reg == IR_REG_NONE && op1 == op2) { + op2_reg = op1_reg; + } if (op2_reg != IR_REG_NONE) { if (IR_REG_SPILLED(op2_reg)) { op2_reg = IR_REG_NUM(op2_reg); @@ -3415,25 +3510,52 @@ static void ir_emit_sext(ir_ctx *ctx, ir_ref def, ir_insn *insn) int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op1, &fp); if (ir_type_size[src_type] == 1) { - if (ir_type_size[dst_type] == 2) { - | ldrsb Rw(def_reg), [Rx(fp), #offset] - } else if (ir_type_size[dst_type] == 4) { - | ldrsb Rw(def_reg), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, ir_type_size[src_type])) { + if (ir_type_size[dst_type] == 2) { + | ldrsb Rw(def_reg), [Rx(fp), #offset] + } else if (ir_type_size[dst_type] == 4) { + | ldrsb Rw(def_reg), [Rx(fp), #offset] + } else { + IR_ASSERT(ir_type_size[dst_type] == 8); + | ldrsb Rx(def_reg), [Rx(fp), #offset] + } } else { - IR_ASSERT(ir_type_size[dst_type] == 8); - | ldrsb Rx(def_reg), [Rx(fp), #offset] + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + if (ir_type_size[dst_type] == 2) { + | ldrsb Rw(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } else if (ir_type_size[dst_type] == 4) { + | ldrsb Rw(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } else { + IR_ASSERT(ir_type_size[dst_type] == 8); + | ldrsb Rx(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } } else if (ir_type_size[src_type] == 2) { - if (ir_type_size[dst_type] == 4) { - | ldrsh Rw(def_reg), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, ir_type_size[src_type])) { + if (ir_type_size[dst_type] == 4) { + | ldrsh Rw(def_reg), [Rx(fp), #offset] + } else { + IR_ASSERT(ir_type_size[dst_type] == 8); + | ldrsh Rx(def_reg), [Rx(fp), #offset] + } } else { - IR_ASSERT(ir_type_size[dst_type] == 8); - | ldrsh Rx(def_reg), [Rx(fp), #offset] + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + if (ir_type_size[dst_type] == 4) { + | ldrsh Rw(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } else { + IR_ASSERT(ir_type_size[dst_type] == 8); + | ldrsh Rx(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } } else { IR_ASSERT(ir_type_size[src_type] == 4); IR_ASSERT(ir_type_size[dst_type] == 8); - | ldrsw Rx(def_reg), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, ir_type_size[src_type])) { + | ldrsw Rx(def_reg), [Rx(fp), #offset] + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | ldrsw Rx(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -3473,14 +3595,27 @@ static void ir_emit_zext(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg fp; int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op1, &fp); - if (ir_type_size[src_type] == 1) { - | ldrb Rw(def_reg), [Rx(fp), #offset] - } else if (ir_type_size[src_type] == 2) { - | ldrh Rw(def_reg), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, ir_type_size[src_type])) { + if (ir_type_size[src_type] == 1) { + | ldrb Rw(def_reg), [Rx(fp), #offset] + } else if (ir_type_size[src_type] == 2) { + | ldrh Rw(def_reg), [Rx(fp), #offset] + } else { + IR_ASSERT(ir_type_size[src_type] == 4); + IR_ASSERT(ir_type_size[dst_type] == 8); + | ldr Rw(def_reg), [Rx(fp), #offset] + } } else { - IR_ASSERT(ir_type_size[src_type] == 4); - IR_ASSERT(ir_type_size[dst_type] == 8); - | ldr Rw(def_reg), [Rx(fp), #offset] + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + if (ir_type_size[src_type] == 1) { + | ldrb Rw(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } else if (ir_type_size[src_type] == 2) { + | ldrh Rw(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } else { + IR_ASSERT(ir_type_size[src_type] == 4); + IR_ASSERT(ir_type_size[dst_type] == 8); + | ldr Rw(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -3579,11 +3714,21 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg fp; int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op1, &fp); - if (src_type == IR_DOUBLE) { - | ldr Rx(def_reg), [Rx(fp), #offset] + if (aarch64_may_encode_addr_offset(offset, ir_type_size[src_type])) { + if (src_type == IR_DOUBLE) { + | ldr Rx(def_reg), [Rx(fp), #offset] + } else { + IR_ASSERT(src_type == IR_FLOAT); + | ldr Rw(def_reg), [Rx(fp), #offset] + } } else { - IR_ASSERT(src_type == IR_FLOAT); - | ldr Rw(def_reg), [Rx(fp), #offset] + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + if (src_type == IR_DOUBLE) { + | ldr Rx(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } else { + IR_ASSERT(src_type == IR_FLOAT); + | ldr Rw(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } } } } else if (IR_IS_TYPE_FP(dst_type)) { @@ -3605,12 +3750,22 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg fp; int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op1, &fp); - if (dst_type == IR_DOUBLE) { - | ldr Rd(def_reg), [Rx(fp), #offset] - } else { - IR_ASSERT(src_type == IR_FLOAT); - | ldr Rs(def_reg), [Rx(fp), #offset] - } + if (aarch64_may_encode_addr_offset(offset, ir_type_size[src_type])) { + if (dst_type == IR_DOUBLE) { + | ldr Rd(def_reg), [Rx(fp), #offset] + } else { + IR_ASSERT(dst_type == IR_FLOAT); + | ldr Rs(def_reg), [Rx(fp), #offset] + } + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + if (dst_type == IR_DOUBLE) { + | ldr Rd(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } else { + IR_ASSERT(dst_type == IR_FLOAT); + | ldr Rs(def_reg), [Rx(fp), Rx(IR_REG_INT_TMP)] + } + } } } if (IR_REG_SPILLED(ctx->regs[def][0])) { @@ -3833,7 +3988,12 @@ static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(def_reg != IR_REG_NONE); offset = ir_var_spill_slot(ctx, insn->op1, &fp); - | add Rx(def_reg), Rx(fp), #offset + if (aarch64_may_encode_imm12(offset)) { + | add Rx(def_reg), Rx(fp), #offset + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, offset); + | add Rx(def_reg), Rx(fp), Rx(IR_REG_INT_TMP) + } if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); } @@ -4221,7 +4381,12 @@ static void ir_emit_afree(ir_ctx *ctx, ir_ref def, ir_insn *insn) /* Stack must be 16 byte aligned */ size = IR_ALIGNED_SIZE(size, 16); - | add sp, sp, #size + if (aarch64_may_encode_imm12(size)) { + | add sp, sp, #size + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, size); + | add sp, sp, Rx(IR_REG_INT_TMP) + } if (!(ctx->flags & IR_USE_FRAME_POINTER)) { ctx->call_stack_size -= size; } @@ -4283,8 +4448,11 @@ static void ir_emit_frame_addr(ir_ctx *ctx, ir_ref def) if (ctx->flags & IR_USE_FRAME_POINTER) { | mov Rx(def_reg), Rx(IR_REG_X29) - } else { + } else if (aarch64_may_encode_imm12(ctx->stack_frame_size + ctx->call_stack_size)) { | add Rx(def_reg), Rx(IR_REG_X31), #(ctx->stack_frame_size + ctx->call_stack_size) + } else { + ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, ctx->stack_frame_size + ctx->call_stack_size); + | add Rx(def_reg), Rx(IR_REG_X31), Rx(IR_REG_INT_TMP) } if (IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, IR_ADDR, def, def_reg); @@ -4377,7 +4545,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) if ((ctx->flags2 & (IR_HAS_VA_ARG_FP|IR_HAS_VA_COPY)) && ctx->fp_reg_params < IR_REG_FP_ARGS) { reg_save_area_offset += 16 * IR_REG_FP_ARGS; /* Set va_list.vr_top */ - if (overflow_arg_area_offset != reg_save_area_offset) { + if (overflow_arg_area_offset != reg_save_area_offset || ctx->gp_reg_params < IR_REG_INT_ARGS) { | add Rx(tmp_reg), Rx(fp), #reg_save_area_offset } | str Rx(tmp_reg), [Rx(op2_reg), #(offset+offsetof(ir_va_list, vr_top))] @@ -5246,6 +5414,19 @@ static void ir_emit_ijmp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } | br Rx(op2_reg) } else if (IR_IS_CONST_REF(insn->op2)) { + if (ctx->ir_base[insn->op2].op == IR_LABEL) { + if (!data->resolved_label_syms) { + data->resolved_label_syms = 1; + ir_resolve_label_syms(ctx); + } + + uint32_t target = ctx->ir_base[insn->op2].val.u32_hi; + target = ir_skip_empty_target_blocks(ctx, target); + + | b =>target + return; + } + void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op2]); if (aarch64_may_use_b(ctx->code_buffer, addr)) { @@ -5636,6 +5817,7 @@ static void ir_emit_param_move(ir_ctx *ctx, uint8_t type, ir_reg from_reg, ir_re { ir_reg fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(offset); IR_ASSERT(from_reg != IR_REG_NONE || to_reg != IR_REG_NONE); if (IR_IS_TYPE_INT(type)) { @@ -5676,13 +5858,8 @@ static void ir_emit_load_params(ir_ctx *ctx) const int8_t *int_reg_params = _ir_int_reg_params; const int8_t *fp_reg_params = _ir_fp_reg_params; int32_t stack_offset = 0; + int32_t stack_start = ctx->stack_frame_size; - if (ctx->flags & IR_USE_FRAME_POINTER) { - /* skip old frame pointer and return address */ - stack_offset = sizeof(void*) * 2 + ctx->stack_frame_size + ctx->call_stack_size; - } else { - stack_offset = ctx->stack_frame_size + ctx->call_stack_size; - } n = use_list->count; for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { use = *p; @@ -5706,12 +5883,9 @@ static void ir_emit_load_params(ir_ctx *ctx) if (ctx->vregs[use]) { dst_reg = IR_REG_NUM(ctx->regs[use][0]); IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE || - stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos + - ((ctx->flags & IR_USE_FRAME_POINTER) ? - -(ctx->stack_frame_size - ctx->stack_frame_alignment) : - ctx->call_stack_size)); + stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos); if (src_reg != dst_reg) { - ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_offset); + ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_start + stack_offset); } if (dst_reg != IR_REG_NONE && IR_REG_SPILLED(ctx->regs[use][0])) { ir_emit_store(ctx, insn->type, use, dst_reg); @@ -5785,14 +5959,8 @@ static void ir_fix_param_spills(ir_ctx *ctx) const int8_t *int_reg_params = _ir_int_reg_params; const int8_t *fp_reg_params = _ir_fp_reg_params; int32_t stack_offset = 0; - int32_t param_stack_size = 0; + int32_t stack_start = ctx->stack_frame_size; - if (ctx->flags & IR_USE_FRAME_POINTER) { - /* skip old frame pointer and return address */ - stack_offset = sizeof(void*) * 2 + (ctx->stack_frame_size - ctx->stack_frame_alignment); - } else { - stack_offset = ctx->stack_frame_size; - } n = use_list->count; for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { use = *p; @@ -5819,15 +5987,13 @@ static void ir_fix_param_spills(ir_ctx *ctx) if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) && ival->stack_spill_pos == -1 && (ival->next || ival->reg == IR_REG_NONE)) { - ival->stack_spill_pos = stack_offset; + ival->stack_spill_pos = stack_start + stack_offset; } } if (sizeof(void*) == 8) { stack_offset += sizeof(void*); - param_stack_size += sizeof(void*); } else { stack_offset += IR_MAX(sizeof(void*), ir_type_size[insn->type]); - param_stack_size += IR_MAX(sizeof(void*), ir_type_size[insn->type]); } } } @@ -5835,7 +6001,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) ctx->gp_reg_params = IR_MIN(int_param_num, int_reg_params_count); ctx->fp_reg_params = IR_MIN(fp_param_num, fp_reg_params_count); - ctx->param_stack_size = param_stack_size; + ctx->param_stack_size = stack_offset; } static void ir_allocate_unique_spill_slots(ir_ctx *ctx) @@ -5876,6 +6042,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) case IR_MERGE: case IR_LOOP_BEGIN: case IR_LOOP_END: + case IR_IGOTO_DUP: break; default: def_flags = ir_get_target_constraints(ctx, i, &constraints); @@ -5892,7 +6059,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) IR_REGSET_EXCL(available, reg); ctx->regs[i][0] = reg | IR_REG_SPILL_STORE; } else if (def_flags & IR_USE_MUST_BE_IN_REG) { - if (insn->op == IR_VLOAD + if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) && ctx->live_intervals[ctx->vregs[i]] && ctx->live_intervals[ctx->vregs[i]]->stack_spill_pos != -1 && ir_is_same_mem_var(ctx, i, ctx->ir_base[insn->op2].op3)) { @@ -5932,7 +6099,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { use = *p; use_insn = &ctx->ir_base[use]; - if (use_insn->op == IR_VLOAD) { + if (use_insn->op == IR_VLOAD || use_insn->op == IR_VLOAD_v) { if (ctx->vregs[use] && !ctx->live_intervals[ctx->vregs[use]]) { ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); @@ -5943,7 +6110,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) ival->vreg = ctx->vregs[use]; ival->stack_spill_pos = stack_spill_pos; } - } else if (use_insn->op == IR_VSTORE) { + } else if (use_insn->op == IR_VSTORE || use_insn->op == IR_STORE_v) { if (!IR_IS_CONST_REF(use_insn->op3) && ctx->vregs[use_insn->op3] && !ctx->live_intervals[ctx->vregs[use_insn->op3]]) { @@ -6080,25 +6247,21 @@ void ir_fix_stack_frame(ir_ctx *ctx) ctx->stack_frame_size = IR_ALIGNED_SIZE(ctx->stack_frame_size, sizeof(void*)); ctx->stack_frame_size += additional_size; - ctx->stack_frame_alignment = 0; ctx->call_stack_size = 0; if (!(ctx->flags & IR_FUNCTION)) { while (IR_ALIGNED_SIZE(ctx->stack_frame_size, 16) != ctx->stack_frame_size) { ctx->stack_frame_size += sizeof(void*); - ctx->stack_frame_alignment += sizeof(void*); } } else { /* Stack must be 16 byte aligned */ if (!(ctx->flags & IR_FUNCTION)) { while (IR_ALIGNED_SIZE(ctx->stack_frame_size, 16) != ctx->stack_frame_size) { ctx->stack_frame_size += sizeof(void*); - ctx->stack_frame_alignment += sizeof(void*); } } else if (ctx->flags & IR_USE_FRAME_POINTER) { while (IR_ALIGNED_SIZE(ctx->stack_frame_size + sizeof(void*) * 2, 16) != ctx->stack_frame_size + sizeof(void*) * 2) { ctx->stack_frame_size += sizeof(void*); - ctx->stack_frame_alignment += sizeof(void*); } } else { if (!(ctx->flags & IR_NO_STACK_COMBINE)) { @@ -6107,7 +6270,6 @@ void ir_fix_stack_frame(ir_ctx *ctx) while (IR_ALIGNED_SIZE(ctx->stack_frame_size + ctx->call_stack_size, 16) != ctx->stack_frame_size + ctx->call_stack_size) { ctx->stack_frame_size += sizeof(void*); - ctx->stack_frame_alignment += sizeof(void*); } } } @@ -6143,6 +6305,8 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) int ret; void *entry; size_t size; + ir_ref igoto_dup_ref = IR_UNUSED; + uint32_t igoto_dup_block = 0; data.ra_data.unused_slot_4 = 0; data.ra_data.unused_slot_2 = 0; @@ -6150,11 +6314,11 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) data.ra_data.handled = NULL; data.rodata_label = 0; data.jmp_table_label = 0; + data.resolved_label_syms = 0; ctx->data = &data; if (!ctx->live_intervals) { ctx->stack_frame_size = 0; - ctx->stack_frame_alignment = 0; ctx->call_stack_size = 0; ctx->used_preserved_regs = 0; ir_allocate_unique_spill_slots(ctx); @@ -6176,7 +6340,6 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) } ctx->stack_frame_size = ctx->fixed_stack_frame_size; ctx->call_stack_size = ctx->fixed_call_stack_size; - ctx->stack_frame_alignment = 0; } Dst = &data.dasm_state; @@ -6386,6 +6549,35 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_TAILCALL: ir_emit_tailcall(ctx, i, insn); break; + case IR_IGOTO_DUP: + if (bb->flags & IR_BB_DESSA_MOVES) { + ir_emit_dessa_moves(ctx, b, bb); + } + IR_ASSERT(!igoto_dup_ref && !igoto_dup_block); + igoto_dup_ref = i; + igoto_dup_block = b; + b = ctx->cfg_edges[bb->successors]; + bb = &ctx->cfg_blocks[b]; + i = bb->start; + insn = &ctx->ir_base[i]; + rule = &ctx->rules[i]; + break; + case IR_IGOTO: + if ((ctx->ir_base[insn->op1].op == IR_MERGE || ctx->ir_base[insn->op1].op == IR_LOOP_BEGIN) + && (ctx->rules[ctx->ir_base[insn->op1].op1] & IR_RULE_MASK) == IR_IGOTO_DUP + && igoto_dup_ref) { + ir_emit_ijmp(ctx, i, insn); + b = igoto_dup_block; + bb = &ctx->cfg_blocks[b]; + i = igoto_dup_ref; + insn = &ctx->ir_base[i]; + rule = &ctx->rules[i]; + igoto_dup_block= 0; + igoto_dup_ref = 0; + break; + } + IR_ASSERT(!igoto_dup_ref && !igoto_dup_block); + IR_FALLTHROUGH; case IR_IJMP: ir_emit_ijmp(ctx, i, insn); break; @@ -6396,9 +6588,11 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) ir_emit_vaddr(ctx, i, insn); break; case IR_VLOAD: + case IR_VLOAD_v: ir_emit_vload(ctx, i, insn); break; case IR_VSTORE: + case IR_VSTORE_v: ir_emit_vstore(ctx, i, insn); break; case IR_RLOAD: @@ -6645,6 +6839,28 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) } while (i != 0); } + if ((ctx->flags2 & IR_HAS_BLOCK_ADDR) && ctx->loader && ctx->loader->add_label) { + for (b = 1, bb = &ctx->cfg_blocks[b]; b <= ctx->cfg_blocks_count; bb++, b++) { + ir_insn *insn = &ctx->ir_base[bb->start]; + + if (insn->op == IR_BEGIN && insn->op2) { + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_LABEL); + ctx->ir_base[insn->op2].val.u32_hi = 0; + ctx->loader->add_label(ctx->loader, ir_get_str(ctx, ctx->ir_base[insn->op2].val.str), + (char*)entry + dasm_getpclabel(&data.dasm_state, ir_skip_empty_target_blocks(ctx, b))); + } + } + } else if (data.resolved_label_syms) { + for (b = 1, bb = &ctx->cfg_blocks[b]; b <= ctx->cfg_blocks_count; bb++, b++) { + ir_insn *insn = &ctx->ir_base[bb->start]; + + if (insn->op == IR_BEGIN && insn->op2) { + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_LABEL); + ctx->ir_base[insn->op2].val.u32_hi = 0; + } + } + } + dasm_free(&data.dasm_state); if (ctx->code_buffer) { diff --git a/ext/opcache/jit/ir/ir_builder.h b/ext/opcache/jit/ir/ir_builder.h index c1dcffdbaa084..03add75906553 100644 --- a/ext/opcache/jit/ir/ir_builder.h +++ b/ext/opcache/jit/ir/ir_builder.h @@ -490,7 +490,7 @@ extern "C" { #define ir_ADD_OFFSET(_addr, _offset) _ir_ADD_OFFSET(_ir_CTX, (_addr), (_offset)) /* Unfoldable variant of COPY */ -#define ir_HARD_COPY(_type, _op1) ir_emit2(_ir_CTX, IR_OPT(IR_COPY, (_type)), (_op1), 1) +#define ir_HARD_COPY(_type, _op1) ir_emit2(_ir_CTX, IR_OPT(IR_COPY, (_type)), (_op1), IR_COPY_HARD) #define ir_HARD_COPY_B(_op1) ir_HARD_COPY(IR_BOOL, _op1) #define ir_HARD_COPY_U8(_op1) ir_HARD_COPY(IR_U8, _op1) #define ir_HARD_COPY_U16(_op1) ir_HARD_COPY(IR_U16, _op1) @@ -544,6 +544,8 @@ extern "C" { #define ir_VLOAD_D(_var) _ir_VLOAD(_ir_CTX, IR_DOUBLE, (_var)) #define ir_VLOAD_F(_var) _ir_VLOAD(_ir_CTX, IR_FLOAT, (_var)) #define ir_VSTORE(_var, _val) _ir_VSTORE(_ir_CTX, (_var), (_val)) +#define ir_VLOAD_v(_type, _var) _ir_VLOAD_v(_ir_CTX, (_type), (_var)) +#define ir_VSTORE_v(_var, _val) _ir_VSTORE_v(_ir_CTX, (_var), (_val)) #define ir_RLOAD(_type, _reg) _ir_RLOAD(_ir_CTX, (_type), (_reg)) #define ir_RLOAD_B(_reg) _ir_RLOAD(_ir_CTX, IR_BOOL, (_reg)) #define ir_RLOAD_U8(_reg) _ir_RLOAD(_ir_CTX, IR_U8, (_reg)) @@ -574,6 +576,8 @@ extern "C" { #define ir_LOAD_D(_addr) _ir_LOAD(_ir_CTX, IR_DOUBLE, (_addr)) #define ir_LOAD_F(_addr) _ir_LOAD(_ir_CTX, IR_FLOAT, (_addr)) #define ir_STORE(_addr, _val) _ir_STORE(_ir_CTX, (_addr), (_val)) +#define ir_LOAD_v(_type, _addr) _ir_LOAD_v(_ir_CTX, (_type), (_addr)) +#define ir_STORE_v(_addr, _val) _ir_STORE_v(_ir_CTX, (_addr), (_val)) #define ir_TLS(_index, _offset) _ir_TLS(_ir_CTX, (_index), (_offset)) #define ir_TRAP() do {_ir_CTX->control = ir_emit1(_ir_CTX, IR_TRAP, _ir_CTX->control);} while (0) @@ -586,7 +590,7 @@ extern "C" { #define ir_VA_END(_list) _ir_VA_END(_ir_CTX, _list) #define ir_VA_COPY(_dst, _src) _ir_VA_COPY(_ir_CTX, _dst, _src) #define ir_VA_ARG(_list, _type) _ir_VA_ARG(_ir_CTX, _type, _list) -#define ir_VA_ARG_EX(_list, _type, size) _ir_VA_ARG_EX(_ir_CTX, _type, _list, size) +#define ir_VA_ARG_EX(_list, _type, s, a) _ir_VA_ARG_EX(_ir_CTX, _type, _list, s, a) #define ir_START() _ir_START(_ir_CTX) #define ir_ENTRY(_src, _num) _ir_ENTRY(_ir_CTX, (_src), (_num)) @@ -607,6 +611,7 @@ extern "C" { #define ir_CASE_RANGE(_switch, _v1, _v2) _ir_CASE_RANGE(_ir_CTX, (_switch), (_v1), (_v2)) #define ir_CASE_DEFAULT(_switch) _ir_CASE_DEFAULT(_ir_CTX, (_switch)) #define ir_RETURN(_val) _ir_RETURN(_ir_CTX, (_val)) +#define ir_IGOTO(_addr) _ir_IGOTO(_ir_CTX, (_addr)) #define ir_IJMP(_addr) _ir_IJMP(_ir_CTX, (_addr)) #define ir_UNREACHABLE() _ir_UNREACHABLE(_ir_CTX) @@ -654,15 +659,19 @@ ir_ref _ir_ALLOCA(ir_ctx *ctx, ir_ref size); void _ir_AFREE(ir_ctx *ctx, ir_ref size); ir_ref _ir_VLOAD(ir_ctx *ctx, ir_type type, ir_ref var); void _ir_VSTORE(ir_ctx *ctx, ir_ref var, ir_ref val); +ir_ref _ir_VLOAD_v(ir_ctx *ctx, ir_type type, ir_ref var); +void _ir_VSTORE_v(ir_ctx *ctx, ir_ref var, ir_ref val); ir_ref _ir_RLOAD(ir_ctx *ctx, ir_type type, ir_ref reg); void _ir_RSTORE(ir_ctx *ctx, ir_ref reg, ir_ref val); ir_ref _ir_LOAD(ir_ctx *ctx, ir_type type, ir_ref addr); void _ir_STORE(ir_ctx *ctx, ir_ref addr, ir_ref val); +ir_ref _ir_LOAD_v(ir_ctx *ctx, ir_type type, ir_ref addr); +void _ir_STORE_v(ir_ctx *ctx, ir_ref addr, ir_ref val); void _ir_VA_START(ir_ctx *ctx, ir_ref list); void _ir_VA_END(ir_ctx *ctx, ir_ref list); void _ir_VA_COPY(ir_ctx *ctx, ir_ref dst, ir_ref src); ir_ref _ir_VA_ARG(ir_ctx *ctx, ir_type type, ir_ref list); -ir_ref _ir_VA_ARG_EX(ir_ctx *ctx, ir_type type, ir_ref list, size_t size); +ir_ref _ir_VA_ARG_EX(ir_ctx *ctx, ir_type type, ir_ref list, size_t size, size_t align); void _ir_START(ir_ctx *ctx); void _ir_ENTRY(ir_ctx *ctx, ir_ref src, ir_ref num); void _ir_BEGIN(ir_ctx *ctx, ir_ref src); @@ -688,6 +697,7 @@ void _ir_CASE_VAL(ir_ctx *ctx, ir_ref switch_ref, ir_ref val); void _ir_CASE_RANGE(ir_ctx *ctx, ir_ref switch_ref, ir_ref v1, ir_ref v2); void _ir_CASE_DEFAULT(ir_ctx *ctx, ir_ref switch_ref); void _ir_RETURN(ir_ctx *ctx, ir_ref val); +ir_ref _ir_IGOTO(ir_ctx *ctx, ir_ref addr); void _ir_IJMP(ir_ctx *ctx, ir_ref addr); void _ir_GUARD(ir_ctx *ctx, ir_ref condition, ir_ref addr); void _ir_GUARD_NOT(ir_ctx *ctx, ir_ref condition, ir_ref addr); diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c index 00923387bb21c..46755067b2444 100644 --- a/ext/opcache/jit/ir/ir_cfg.c +++ b/ext/opcache/jit/ir/ir_cfg.c @@ -820,11 +820,14 @@ int ir_build_dominators_tree(ir_ctx *ctx) succ_b = ctx->cfg_edges[bb->successors]; if (bb->successors_count != 1) { /* LOOP_END/END may be linked with the following ENTRY by a fake edge */ - IR_ASSERT(bb->successors_count == 2); - if (blocks[succ_b].flags & IR_BB_ENTRY) { + if (bb->successors_count != 2) { + complete = 0; + break; + } else if (blocks[succ_b].flags & IR_BB_ENTRY) { succ_b = ctx->cfg_edges[bb->successors + 1]; - } else { - IR_ASSERT(blocks[ctx->cfg_edges[bb->successors + 1]].flags & IR_BB_ENTRY); + } else if (!(blocks[ctx->cfg_edges[bb->successors + 1]].flags & IR_BB_ENTRY)) { + complete = 0; + break; } } dom_depth = blocks[succ_b].dom_depth;; diff --git a/ext/opcache/jit/ir/ir_check.c b/ext/opcache/jit/ir/ir_check.c index c25a984aefc1d..ee951291b1b05 100644 --- a/ext/opcache/jit/ir/ir_check.c +++ b/ext/opcache/jit/ir/ir_check.c @@ -328,7 +328,9 @@ bool ir_check(const ir_ctx *ctx) } break; case IR_LOAD: + case IR_LOAD_v: case IR_STORE: + case IR_STORE_v: type = ctx->ir_base[insn->op2].type; if (type != IR_ADDR && (!IR_IS_TYPE_INT(type) || ir_type_size[type] != ir_type_size[IR_ADDR])) { @@ -338,7 +340,9 @@ bool ir_check(const ir_ctx *ctx) } break; case IR_VLOAD: + case IR_VLOAD_v: case IR_VSTORE: + case IR_VSTORE_v: if (ctx->ir_base[insn->op2].op != IR_VAR) { fprintf(stderr, "ir_base[%d].op2 must be 'VAR' (%s)\n", i, ir_op_name[ctx->ir_base[insn->op2].op]); @@ -408,6 +412,8 @@ bool ir_check(const ir_ctx *ctx) ok = 0; } break; + case IR_IGOTO: + break; default: /* skip data references */ count = n = use_list->count; diff --git a/ext/opcache/jit/ir/ir_dump.c b/ext/opcache/jit/ir/ir_dump.c index a501d261f30a7..5cc732927d412 100644 --- a/ext/opcache/jit/ir/ir_dump.c +++ b/ext/opcache/jit/ir/ir_dump.c @@ -129,6 +129,11 @@ void ir_dump_dot(const ir_ctx *ctx, const char *name, FILE *f) case IR_OPND_CONTROL_REF: fprintf(f, "\tn%d -> n%d [style=dashed,dir=back,weight=%d];\n", ref, i, REF_WEIGHT); break; + case IR_OPND_LABEL_REF: + if (ref) { + fprintf(f, "\tc%d -> n%d [color=blue,weight=%d];\n", -ref, i, REF_WEIGHT); + } + break; } } } @@ -491,6 +496,8 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) ir_print_proto(ctx, insn->proto, f); } else if (insn->op == IR_SYM) { fprintf(f, "sym(%s)", ir_get_str(ctx, insn->val.name)); + } else if (insn->op == IR_LABEL) { + fprintf(f, "label(%s)", ir_get_str(ctx, insn->val.name)); } else if (insn->op == IR_FUNC_ADDR) { fprintf(f, "func *"); ir_print_const(ctx, insn, f, true); @@ -648,6 +655,12 @@ void ir_dump_codegen(const ir_ctx *ctx, FILE *f) fprintf(f, "%s%d", first ? "(" : ", ", ref); first = 0; break; + case IR_OPND_LABEL_REF: + if (ref) { + IR_ASSERT(IR_IS_CONST_REF(ref)); + fprintf(f, "%sc_%d", first ? "(" : ", ", -ref); + } + break; } } else if (opnd_kind == IR_OPND_NUM) { fprintf(f, "%s%d", first ? "(" : ", ", ref); diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index 7a10da1322a72..847ca375b5bd0 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -244,32 +244,30 @@ static int ir_get_args_regs(const ir_ctx *ctx, const ir_insn *insn, int8_t *regs ir_insn *arg = &ctx->ir_base[ir_insn_op(insn, j)]; type = arg->type; if (IR_IS_TYPE_INT(type)) { - if (arg->op == IR_ARGVAL) { - continue; - } else if (int_param < int_reg_params_count) { + if (int_param < int_reg_params_count && arg->op != IR_ARGVAL) { regs[j] = int_reg_params[int_param]; count = j + 1; + int_param++; +#ifdef _WIN64 + /* WIN64 calling convention use common couter for int and fp registers */ + fp_param++; +#endif } else { regs[j] = IR_REG_NONE; } - int_param++; -#ifdef _WIN64 - /* WIN64 calling convention use common couter for int and fp registers */ - fp_param++; -#endif } else { IR_ASSERT(IR_IS_TYPE_FP(type)); if (fp_param < fp_reg_params_count) { regs[j] = fp_reg_params[fp_param]; count = j + 1; + fp_param++; +#ifdef _WIN64 + /* WIN64 calling convention use common couter for int and fp registers */ + int_param++; +#endif } else { regs[j] = IR_REG_NONE; } - fp_param++; -#ifdef _WIN64 - /* WIN64 calling convention use common couter for int and fp registers */ - int_param++; -#endif } } return count; @@ -426,7 +424,7 @@ typedef struct _ir_common_backend_data { ir_bitset emit_constants; } ir_common_backend_data; -static int ir_const_label(ir_ctx *ctx, ir_ref ref) +static int ir_get_const_label(ir_ctx *ctx, ir_ref ref) { ir_common_backend_data *data = ctx->data; int label = ctx->cfg_blocks_count - ref; @@ -1015,11 +1013,16 @@ int ir_match(ir_ctx *ctx) entries_count++; } ctx->rules[start] = IR_SKIPPED | IR_NOP; + if (ctx->ir_base[start].op == IR_BEGIN && ctx->ir_base[start].op2) { + ctx->flags2 |= IR_HAS_BLOCK_ADDR; + } ref = bb->end; if (bb->successors_count == 1) { insn = &ctx->ir_base[ref]; if (insn->op == IR_END || insn->op == IR_LOOP_END) { - ctx->rules[ref] = insn->op; + if (!ctx->rules[ref]) { + ctx->rules[ref] = insn->op; + } ref = prev_ref[ref]; if (ref == start && ctx->cfg_edges[bb->successors] != b) { if (EXPECTED(!(bb->flags & IR_BB_ENTRY))) { diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 74f7818d747c4..bab6b2916075f 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -755,8 +755,35 @@ IR_FOLD(NEG(C_FLOAT)) } IR_FOLD(ABS(C_I8)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op1_insn->val.i64 >= 0) { + IR_FOLD_COPY(op1); + } else { + IR_FOLD_CONST_I(-op1_insn->val.i8); + } +} + IR_FOLD(ABS(C_I16)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op1_insn->val.i64 >= 0) { + IR_FOLD_COPY(op1); + } else { + IR_FOLD_CONST_I(-op1_insn->val.i16); + } +} + IR_FOLD(ABS(C_I32)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op1_insn->val.i64 >= 0) { + IR_FOLD_COPY(op1); + } else { + IR_FOLD_CONST_I((int32_t)-op1_insn->val.u32); + } +} + IR_FOLD(ABS(C_I64)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); @@ -847,7 +874,7 @@ IR_FOLD(MUL_OV(C_U64, C_U64)) uint64_t res; IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); res = op1_insn->val.u64 * op2_insn->val.u64; - if (op1_insn->val.u64 != 0 && res / op1_insn->val.u64 != op2_insn->val.u64 && res <= max) { + if ((op1_insn->val.u64 != 0 && res / op1_insn->val.u64 != op2_insn->val.u64) || res > max) { IR_FOLD_NEXT; } IR_FOLD_CONST_U(res); @@ -864,7 +891,7 @@ IR_FOLD(MUL_OV(C_I64, C_I64)) int64_t res; IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); res = op1_insn->val.u64 * op2_insn->val.u64; - if (op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64 && res >= min && res <= max) { + if ((op1_insn->val.i64 != 0 && res / op1_insn->val.i64 != op2_insn->val.i64) || res < min || res > max) { IR_FOLD_NEXT; } IR_FOLD_CONST_U(res); @@ -1037,220 +1064,220 @@ IR_FOLD(SHL(C_U8, C_U8)) IR_FOLD(SHL(C_CHAR, C_CHAR)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u8 << op2_insn->val.u8); + IR_FOLD_CONST_U(op1_insn->val.u8 << (op2_insn->val.u8 & 0x7)); } IR_FOLD(SHL(C_I8, C_I8)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int8_t)(op1_insn->val.u8 << op2_insn->val.u8)); + IR_FOLD_CONST_I((int8_t)(op1_insn->val.u8 << (op2_insn->val.u8 & 0x7))); } IR_FOLD(SHL(C_U16, C_U16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u16 << op2_insn->val.u16); + IR_FOLD_CONST_U(op1_insn->val.u16 << (op2_insn->val.u16 & 0xf)); } IR_FOLD(SHL(C_I16, C_I16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int16_t)(op1_insn->val.u16 << op2_insn->val.u16)); + IR_FOLD_CONST_I((int16_t)(op1_insn->val.u16 << (op2_insn->val.u16 & 0xf))); } IR_FOLD(SHL(C_U32, C_U32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u32 << op2_insn->val.u32); + IR_FOLD_CONST_U(op1_insn->val.u32 << (op2_insn->val.u32 & 0x1f)); } IR_FOLD(SHL(C_I32, C_I32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int32_t)(op1_insn->val.u32 << op2_insn->val.u32)); + IR_FOLD_CONST_I((int32_t)(op1_insn->val.u32 << (op2_insn->val.u32 & 0x1f))); } IR_FOLD(SHL(C_U64, C_U64)) IR_FOLD(SHL(C_I64, C_I64)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u64 << op2_insn->val.u64); + IR_FOLD_CONST_U(op1_insn->val.u64 << (op2_insn->val.u64 & 0x3f)); } IR_FOLD(SHR(C_U8, C_U8)) IR_FOLD(SHR(C_CHAR, C_CHAR)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u8 >> op2_insn->val.u8); + IR_FOLD_CONST_U(op1_insn->val.u8 >> (op2_insn->val.u8 & 0x7)); } IR_FOLD(SHR(C_I8, C_I8)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int8_t)(op1_insn->val.u8 >> op2_insn->val.u8)); + IR_FOLD_CONST_I((int8_t)(op1_insn->val.u8 >> (op2_insn->val.u8 & 0x7))); } IR_FOLD(SHR(C_U16, C_U16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u16 >> op2_insn->val.u16); + IR_FOLD_CONST_U(op1_insn->val.u16 >> (op2_insn->val.u16 & 0xf)); } IR_FOLD(SHR(C_I16, C_I16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int16_t)(op1_insn->val.u16 >> op2_insn->val.u16)); + IR_FOLD_CONST_I((int16_t)(op1_insn->val.u16 >> (op2_insn->val.u16 & 0xf))); } IR_FOLD(SHR(C_U32, C_U32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u32 >> op2_insn->val.u32); + IR_FOLD_CONST_U(op1_insn->val.u32 >> (op2_insn->val.u32 & 0x1f)); } IR_FOLD(SHR(C_I32, C_I32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int32_t)(op1_insn->val.u32 >> op2_insn->val.u32)); + IR_FOLD_CONST_I((int32_t)(op1_insn->val.u32 >> (op2_insn->val.u32 & 0x1f))); } IR_FOLD(SHR(C_U64, C_U64)) IR_FOLD(SHR(C_I64, C_I64)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(op1_insn->val.u64 >> op2_insn->val.u64); + IR_FOLD_CONST_U(op1_insn->val.u64 >> (op2_insn->val.u64 & 0x3f)); } IR_FOLD(SAR(C_U8, C_U8)) IR_FOLD(SAR(C_CHAR, C_CHAR)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U((uint8_t)(op1_insn->val.i8 >> op2_insn->val.i8)); + IR_FOLD_CONST_U((uint8_t)(op1_insn->val.i8 >> (op2_insn->val.i8 & 0x7))); } IR_FOLD(SAR(C_I8, C_I8)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I(op1_insn->val.i8 >> op2_insn->val.i8); + IR_FOLD_CONST_I(op1_insn->val.i8 >> (op2_insn->val.i8 & 0x7)); } IR_FOLD(SAR(C_U16, C_U16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U((uint16_t)(op1_insn->val.i16 >> op2_insn->val.i16)); + IR_FOLD_CONST_U((uint16_t)(op1_insn->val.i16 >> (op2_insn->val.i16 & 0xf))); } IR_FOLD(SAR(C_I16, C_I16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I(op1_insn->val.i16 >> op2_insn->val.i16); + IR_FOLD_CONST_I(op1_insn->val.i16 >> (op2_insn->val.i16 & 0xf)); } IR_FOLD(SAR(C_U32, C_U32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U((uint32_t)(op1_insn->val.i32 >> op2_insn->val.i32)); + IR_FOLD_CONST_U((uint32_t)(op1_insn->val.i32 >> (op2_insn->val.i32 & 0x1f))); } IR_FOLD(SAR(C_I32, C_I32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I(op1_insn->val.i32 >> op2_insn->val.i32); + IR_FOLD_CONST_I(op1_insn->val.i32 >> (op2_insn->val.i32 & 0x1f)); } IR_FOLD(SAR(C_U64, C_U64)) IR_FOLD(SAR(C_I64, C_I64)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I(op1_insn->val.i64 >> op2_insn->val.i64); + IR_FOLD_CONST_I(op1_insn->val.i64 >> (op2_insn->val.i64 & 0x3f)); } IR_FOLD(ROL(C_U8, C_U8)) IR_FOLD(ROL(C_CHAR, C_CHAR)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_rol8(op1_insn->val.u8, op2_insn->val.u8)); + IR_FOLD_CONST_U(ir_rol8(op1_insn->val.u8, (op2_insn->val.u8 & 0x7))); } IR_FOLD(ROL(C_I8, C_I8)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int8_t)ir_rol8(op1_insn->val.u8, op2_insn->val.u8)); + IR_FOLD_CONST_I((int8_t)ir_rol8(op1_insn->val.u8, (op2_insn->val.u8 & 0x7))); } IR_FOLD(ROL(C_U16, C_U16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_rol16(op1_insn->val.u16, op2_insn->val.u16)); + IR_FOLD_CONST_U(ir_rol16(op1_insn->val.u16, (op2_insn->val.u16 & 0xf))); } IR_FOLD(ROL(C_I16, C_I16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int16_t)ir_rol16(op1_insn->val.u16, op2_insn->val.u16)); + IR_FOLD_CONST_I((int16_t)ir_rol16(op1_insn->val.u16, (op2_insn->val.u16 & 0xf))); } IR_FOLD(ROL(C_U32, C_U32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_rol32(op1_insn->val.u32, op2_insn->val.u32)); + IR_FOLD_CONST_U(ir_rol32(op1_insn->val.u32, (op2_insn->val.u32 & 0x1f))); } IR_FOLD(ROL(C_I32, C_I32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int32_t)ir_rol32(op1_insn->val.u32, op2_insn->val.u32)); + IR_FOLD_CONST_I((int32_t)ir_rol32(op1_insn->val.u32, (op2_insn->val.u32 & 0x1f))); } IR_FOLD(ROL(C_U64, C_U64)) IR_FOLD(ROL(C_I64, C_I64)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_rol64(op1_insn->val.u64, op2_insn->val.u64)); + IR_FOLD_CONST_U(ir_rol64(op1_insn->val.u64, (op2_insn->val.u64 & 0x3f))); } IR_FOLD(ROR(C_U8, C_U8)) IR_FOLD(ROR(C_CHAR, C_CHAR)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_ror8(op1_insn->val.u8, op2_insn->val.u8)); + IR_FOLD_CONST_U(ir_ror8(op1_insn->val.u8, (op2_insn->val.u8 & 0x7))); } IR_FOLD(ROR(C_I8, C_I8)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int8_t)ir_ror8(op1_insn->val.u8, op2_insn->val.u8)); + IR_FOLD_CONST_I((int8_t)ir_ror8(op1_insn->val.u8, (op2_insn->val.u8 & 0x7))); } IR_FOLD(ROR(C_U16, C_U16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_ror16(op1_insn->val.u16, op2_insn->val.u16)); + IR_FOLD_CONST_U(ir_ror16(op1_insn->val.u16, (op2_insn->val.u16 & 0xf))); } IR_FOLD(ROR(C_I16, C_I16)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int16_t)ir_ror16(op1_insn->val.u16, op2_insn->val.u16)); + IR_FOLD_CONST_I((int16_t)ir_ror16(op1_insn->val.u16, (op2_insn->val.u16 & 0xf))); } IR_FOLD(ROR(C_U32, C_U32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_ror32(op1_insn->val.u32, op2_insn->val.u32)); + IR_FOLD_CONST_U(ir_ror32(op1_insn->val.u32, (op2_insn->val.u32 & 0x1f))); } IR_FOLD(ROR(C_I32, C_I32)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_I((int32_t)ir_ror32(op1_insn->val.u32, op2_insn->val.u32)); + IR_FOLD_CONST_I((int32_t)ir_ror32(op1_insn->val.u32, (op2_insn->val.u32 & 0x1f))); } IR_FOLD(ROR(C_U64, C_U64)) IR_FOLD(ROR(C_I64, C_I64)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - IR_FOLD_CONST_U(ir_ror64(op1_insn->val.u64, op2_insn->val.u64)); + IR_FOLD_CONST_U(ir_ror64(op1_insn->val.u64, (op2_insn->val.u64 & 0x3f))); } //IR_FOLD(BSWAP(CONST)) @@ -1392,6 +1419,9 @@ IR_FOLD(TRUNC(C_U64)) IR_FOLD_CONST_U(op1_insn->val.u16); case IR_U32: IR_FOLD_CONST_U(op1_insn->val.u32); + case IR_ADDR: + IR_ASSERT(sizeof(void*) == 4); + IR_FOLD_CONST_U(op1_insn->val.u32); } } @@ -1545,7 +1575,7 @@ IR_FOLD(FP2FP(C_DOUBLE)) IR_FOLD(COPY(_)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); - if (!op2) { + if (!(op2 & IR_COPY_HARD)) { IR_FOLD_COPY(op1); } /* skip CSE */ @@ -2075,23 +2105,23 @@ IR_FOLD(SUB(ADD, ADD)) IR_FOLD_CONST_U(0); } else if (op1_insn->op1 == op2_insn->op1) { /* (a + b) - (a + c) => b - c */ - op1 = op1_insn->op2; - op2 = op2_insn->op2; + op1 = _ir_fold_cast(ctx, op1_insn->op2, IR_OPT_TYPE(opt)); + op2 = _ir_fold_cast(ctx, op2_insn->op2, IR_OPT_TYPE(opt)); IR_FOLD_RESTART; } else if (op1_insn->op1 == op2_insn->op2) { /* (a + b) - (c + a) => b - c */ - op1 = op1_insn->op2; - op2 = op2_insn->op1; + op1 = _ir_fold_cast(ctx, op1_insn->op2, IR_OPT_TYPE(opt)); + op2 = _ir_fold_cast(ctx, op2_insn->op1, IR_OPT_TYPE(opt)); IR_FOLD_RESTART; } else if (op1_insn->op2 == op2_insn->op1) { /* (a + b) - (b + c) => a - c */ - op1 = op1_insn->op1; - op2 = op2_insn->op2; + op1 = _ir_fold_cast(ctx, op1_insn->op1, IR_OPT_TYPE(opt)); + op2 = _ir_fold_cast(ctx, op2_insn->op2, IR_OPT_TYPE(opt)); IR_FOLD_RESTART; } else if (op1_insn->op2 == op2_insn->op2) { /* (a + b) - (c + b) => a - c */ - op1 = op1_insn->op1; - op2 = op2_insn->op1; + op1 = _ir_fold_cast(ctx, op1_insn->op1, IR_OPT_TYPE(opt)); + op2 = _ir_fold_cast(ctx, op2_insn->op1, IR_OPT_TYPE(opt)); IR_FOLD_RESTART; } } diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 043e1e7bdd853..e6486ba64a1c5 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -361,20 +361,20 @@ static bool ir_split_partially_dead_node(ir_ctx *ctx, ir_ref ref, uint32_t b) while (ir_sparse_set_in(&data->totally_useful, ctx->cfg_blocks[j].idom)) { j = ctx->cfg_blocks[j].idom; } + clone = ir_hashtab_find(&hash, j); + if (clone == IR_INVALID_VAL) { + clone = clones_count++; + ir_hashtab_add(&hash, j, clone); + clones[clone].block = j; + clones[clone].use_count = 0; + clones[clone].use = -1; + } + uses[uses_count].ref = use; + uses[uses_count].block = i; + uses[uses_count].next = clones[clone].use; + clones[clone].use_count++; + clones[clone].use = uses_count++; } - clone = ir_hashtab_find(&hash, j); - if (clone == IR_INVALID_VAL) { - clone = clones_count++; - ir_hashtab_add(&hash, j, clone); - clones[clone].block = j; - clones[clone].use_count = 0; - clones[clone].use = -1; - } - uses[uses_count].ref = use; - uses[uses_count].block = i; - uses[uses_count].next = clones[clone].use; - clones[clone].use_count++; - clones[clone].use = uses_count++; } } @@ -1007,7 +1007,11 @@ int ir_schedule(ir_ctx *ctx) start = i = bb->start; _xlat[i] = bb->start = insns_count; insn = &ctx->ir_base[i]; - if (insn->op == IR_CASE_VAL) { + if (insn->op == IR_BEGIN) { + if (insn->op2) { + consts_count += ir_count_constant(_xlat, insn->op2); + } + } else if (insn->op == IR_CASE_VAL) { IR_ASSERT(insn->op2 < IR_TRUE); consts_count += ir_count_constant(_xlat, insn->op2); } else if (insn->op == IR_CASE_RANGE) { @@ -1255,7 +1259,7 @@ int ir_schedule(ir_ctx *ctx) const char *proto = ir_get_strl(ctx, new_insn->proto, &len); new_insn->proto = ir_strl(&new_ctx, proto, len); } - } else if (new_insn->op == IR_SYM || new_insn->op == IR_STR) { + } else if (new_insn->op == IR_SYM || new_insn->op == IR_STR || new_insn->op == IR_LABEL) { size_t len; const char *str = ir_get_strl(ctx, new_insn->val.name, &len); new_insn->val.u64 = ir_strl(&new_ctx, str, len); @@ -1292,7 +1296,7 @@ int ir_schedule(ir_ctx *ctx) } else { new_insn->proto = 0; } - } else if (insn->op == IR_SYM || insn->op == IR_STR) { + } else if (insn->op == IR_SYM || insn->op == IR_STR || insn->op == IR_LABEL) { size_t len; const char *str = ir_get_strl(ctx, insn->val.name, &len); new_insn->val.u64 = ir_strl(&new_ctx, str, len); @@ -1364,6 +1368,8 @@ int ir_schedule(ir_ctx *ctx) size_t len; const char *str = ir_get_strl(ctx, insn->op2, &len); new_insn->op2 = ir_strl(&new_ctx, str, len); + } else if (new_insn->op == IR_BEGIN && insn->op2) { + new_insn->op2 = _xlat[insn->op2]; } else { new_insn->op2 = insn->op2; } diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 2f457cbc99333..dbacc3967d0f7 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -887,7 +887,7 @@ void ir_print_escaped_str(const char *s, size_t len, FILE *f); #define IR_IS_CONST_OP(op) ((op) > IR_NOP && (op) <= IR_C_FLOAT) #define IR_IS_FOLDABLE_OP(op) ((op) <= IR_LAST_FOLDABLE_OP) -#define IR_IS_SYM_CONST(op) ((op) == IR_STR || (op) == IR_SYM || (op) == IR_FUNC) +#define IR_IS_SYM_CONST(op) ((op) == IR_STR || (op) == IR_SYM || (op) == IR_FUNC || (op) == IR_LABEL) ir_ref ir_const_ex(ir_ctx *ctx, ir_val val, uint8_t type, uint32_t optx); @@ -946,12 +946,13 @@ IR_ALWAYS_INLINE bool ir_ref_is_true(ir_ctx *ctx, ir_ref ref) #define IR_OPND_UNUSED 0x0 #define IR_OPND_DATA 0x1 #define IR_OPND_CONTROL 0x2 -#define IR_OPND_CONTROL_DEP 0x3 -#define IR_OPND_CONTROL_REF 0x4 -#define IR_OPND_STR 0x5 -#define IR_OPND_NUM 0x6 -#define IR_OPND_PROB 0x7 -#define IR_OPND_PROTO 0x8 +#define IR_OPND_LABEL_REF 0x3 +#define IR_OPND_CONTROL_DEP 0x4 +#define IR_OPND_CONTROL_REF 0x5 +#define IR_OPND_STR 0x6 +#define IR_OPND_NUM 0x7 +#define IR_OPND_PROB 0x8 +#define IR_OPND_PROTO 0x9 #define IR_OP_FLAGS(op_flags, op1_flags, op2_flags, op3_flags) \ ((op_flags) | ((op1_flags) << 20) | ((op2_flags) << 24) | ((op3_flags) << 28)) @@ -1013,6 +1014,7 @@ IR_ALWAYS_INLINE uint32_t ir_insn_len(const ir_insn *insn) #define IR_HAS_VA_ARG_FP (1<<9) #define IR_HAS_FP_RET_SLOT (1<<10) #define IR_16B_FRAME_ALIGNMENT (1<<11) +#define IR_HAS_BLOCK_ADDR (1<<12) /* Temporary: MEM2SSA -> SCCP */ #define IR_MEM2SSA_VARS (1<<25) @@ -1248,11 +1250,10 @@ struct _ir_live_range { #define IR_LIVE_INTERVAL_HAS_HINT_REGS (1<<2) #define IR_LIVE_INTERVAL_HAS_HINT_REFS (1<<3) #define IR_LIVE_INTERVAL_MEM_PARAM (1<<4) -#define IR_LIVE_INTERVAL_MEM_LOAD (1<<5) -#define IR_LIVE_INTERVAL_COALESCED (1<<6) -#define IR_LIVE_INTERVAL_SPILL_SPECIAL (1<<7) /* spill slot is pre-allocated in a special area (see ir_ctx.spill_reserved_base) */ -#define IR_LIVE_INTERVAL_SPILLED (1<<8) -#define IR_LIVE_INTERVAL_SPLIT_CHILD (1<<9) +#define IR_LIVE_INTERVAL_COALESCED (1<<5) +#define IR_LIVE_INTERVAL_SPILL_SPECIAL (1<<6) /* spill slot is pre-allocated in a special area (see ir_ctx.spill_reserved_base) */ +#define IR_LIVE_INTERVAL_SPILLED (1<<7) +#define IR_LIVE_INTERVAL_SPLIT_CHILD (1<<8) struct _ir_live_interval { uint8_t type; diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index 21c7ee3ac64e5..2e8a8e3f34f3f 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -776,9 +776,6 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PARAM) { /* We may reuse parameter stack slot for spilling */ ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - } else if (insn->op == IR_VLOAD) { - /* Load may be fused into the usage instruction */ - ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_LOAD; } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -845,11 +842,17 @@ int ir_compute_live_ranges(ir_ctx *ctx) ival = ctx->live_intervals[v]; } ir_add_use(ctx, ival, j, use_pos, reg, IR_USE_FLAGS(def_flags, j), hint_ref); - } else if (ctx->rules) { - if (ctx->rules[input] & IR_FUSED) { - ir_add_fusion_ranges(ctx, ref, input, bb, live); - } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { - ir_set_alocated_reg(ctx, ref, j, ctx->ir_base[input].op2); + } else { + if (ctx->rules) { + if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { + ir_add_fusion_ranges(ctx, ref, input, bb, live); + } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { + ir_set_alocated_reg(ctx, ref, j, ctx->ir_base[input].op2); + } + } + if (reg != IR_REG_NONE) { + use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); + ir_add_fixed_live_range(ctx, reg, use_pos, use_pos + IR_USE_SUB_REF); } } } else if (reg != IR_REG_NONE) { @@ -1396,9 +1399,6 @@ int ir_compute_live_ranges(ir_ctx *ctx) if (insn->op == IR_PARAM) { /* We may reuse parameter stack slot for spilling */ ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; - } else if (insn->op == IR_VLOAD) { - /* Load may be fused into the usage instruction */ - ctx->live_intervals[v]->flags |= IR_LIVE_INTERVAL_MEM_LOAD; } def_pos = IR_DEF_LIVE_POS_FROM_REF(ref); } @@ -1465,17 +1465,17 @@ int ir_compute_live_ranges(ir_ctx *ctx) ival = ctx->live_intervals[v]; } ir_add_use(ctx, ival, j, use_pos, reg, IR_USE_FLAGS(def_flags, j), hint_ref); - } else if (ctx->rules) { - if (ctx->rules[input] & IR_FUSED) { - ir_add_fusion_ranges(ctx, ref, input, bb, live_in_block, b); - } else { - if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { + } else { + if (ctx->rules) { + if ((ctx->rules[input] & (IR_FUSED|IR_SKIPPED)) == IR_FUSED) { + ir_add_fusion_ranges(ctx, ref, input, bb, live_in_block, b); + } else if (ctx->rules[input] == (IR_SKIPPED|IR_RLOAD)) { ir_set_alocated_reg(ctx, ref, j, ctx->ir_base[input].op2); } - if (reg != IR_REG_NONE) { - use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); - ir_add_fixed_live_range(ctx, reg, use_pos, use_pos + IR_USE_SUB_REF); - } + } + if (reg != IR_REG_NONE) { + use_pos = IR_LOAD_LIVE_POS_FROM_REF(ref); + ir_add_fixed_live_range(ctx, reg, use_pos, use_pos + IR_USE_SUB_REF); } } } else if (reg != IR_REG_NONE) { @@ -1605,7 +1605,7 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2) } while (*prev && ((*prev)->pos < use_pos->pos || ((*prev)->pos == use_pos->pos && - (use_pos->op_num == 0 || (*prev)->op_num < use_pos->op_num)))) { + (use_pos->op_num == 0 || ((*prev)->op_num != 0 && (*prev)->op_num < use_pos->op_num))))) { if ((*prev)->hint_ref > 0 && ctx->vregs[(*prev)->hint_ref] == r2) { (*prev)->hint_ref = 0; } @@ -1627,9 +1627,6 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2) ctx->live_intervals[r1]->flags |= IR_LIVE_INTERVAL_COALESCED | (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS)); - if (ctx->ir_base[IR_LIVE_POS_TO_REF(ctx->live_intervals[r1]->use_pos->pos)].op != IR_VLOAD) { - ctx->live_intervals[r1]->flags &= ~IR_LIVE_INTERVAL_MEM_LOAD; - } if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(!(ctx->live_intervals[r1]->flags & IR_LIVE_INTERVAL_MEM_PARAM)); ctx->live_intervals[r1]->flags |= IR_LIVE_INTERVAL_MEM_PARAM; @@ -2343,16 +2340,6 @@ static ir_live_pos ir_first_use_pos_after(ir_live_interval *ival, ir_live_pos po return p ? p->pos : 0x7fffffff; } -static ir_live_pos ir_first_use_pos(ir_live_interval *ival, uint8_t flags) -{ - ir_use_pos *p = ival->use_pos; - - while (p && !(p->flags & flags)) { - p = p->next; - } - return p ? p->pos : 0x7fffffff; -} - static ir_block *ir_block_from_live_pos(ir_ctx *ctx, ir_live_pos pos) { ir_ref ref = IR_LIVE_POS_TO_REF(pos); @@ -3194,7 +3181,6 @@ static ir_reg ir_allocate_blocked_reg(ir_ctx *ctx, ir_live_interval *ival, ir_li /* split current before its first use position that requires a register */ ir_live_pos split_pos; -spill_current: if (next_use_pos == ival->range.start) { IR_ASSERT(ival->use_pos && ival->use_pos->op_num == 0); /* split right after definition */ @@ -3228,7 +3214,6 @@ static ir_reg ir_allocate_blocked_reg(ir_ctx *ctx, ir_live_interval *ival, ir_li return IR_REG_NONE; } if (split_pos >= blockPos[reg]) { -try_next_available_register: IR_REGSET_EXCL(available, reg); if (IR_REGSET_IS_EMPTY(available)) { fprintf(stderr, "LSRA Internal Error: Unsolvable conflict. Allocation is not possible\n"); @@ -3274,23 +3259,6 @@ static ir_reg ir_allocate_blocked_reg(ir_ctx *ctx, ir_live_interval *ival, ir_li } IR_LOG_LSRA(" ---- Finish", other, ""); } else { - if (ir_first_use_pos(other, IR_USE_MUST_BE_IN_REG) <= other->end) { - if (!(ival->flags & IR_LIVE_INTERVAL_TEMP)) { - next_use_pos = ir_first_use_pos(ival, IR_USE_MUST_BE_IN_REG); - if (next_use_pos == ival->range.start) { - IR_ASSERT(ival->use_pos && ival->use_pos->op_num == 0); - /* split right after definition */ - split_pos = next_use_pos + 1; - } else { - split_pos = ir_find_optimal_split_position(ctx, ival, ival->range.start, next_use_pos - 1, 1); - } - - if (split_pos > ival->range.start) { - goto spill_current; - } - } - goto try_next_available_register; - } child = other; other->reg = IR_REG_NONE; if (prev) { @@ -3400,12 +3368,13 @@ static int ir_fix_dessa_tmps(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to) static bool ir_ival_spill_for_fuse_load(ir_ctx *ctx, ir_live_interval *ival, ir_reg_alloc_data *data) { ir_use_pos *use_pos = ival->use_pos; - ir_insn *insn; if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { IR_ASSERT(!ival->next && use_pos && use_pos->op_num == 0); - insn = &ctx->ir_base[IR_LIVE_POS_TO_REF(use_pos->pos)]; +#if IR_DEBUG + ir_insn *insn = &ctx->ir_base[IR_LIVE_POS_TO_REF(use_pos->pos)]; IR_ASSERT(insn->op == IR_PARAM); +#endif use_pos = use_pos->next; if (use_pos && (use_pos->next || (use_pos->flags & IR_USE_MUST_BE_IN_REG))) { return 0; @@ -3418,38 +3387,6 @@ static bool ir_ival_spill_for_fuse_load(ir_ctx *ctx, ir_live_interval *ival, ir_ } } - return 1; - } else if (ival->flags & IR_LIVE_INTERVAL_MEM_LOAD) { - insn = &ctx->ir_base[IR_LIVE_POS_TO_REF(use_pos->pos)]; - IR_ASSERT(insn->op == IR_VLOAD); - IR_ASSERT(ctx->ir_base[insn->op2].op == IR_VAR); - use_pos = use_pos->next; - if (use_pos && (use_pos->next || (use_pos->flags & IR_USE_MUST_BE_IN_REG))) { - return 0; - } - - if (use_pos) { - ir_block *bb = ir_block_from_live_pos(ctx, use_pos->pos); - if (bb->loop_depth && bb != ir_block_from_live_pos(ctx, ival->use_pos->pos)) { - return 0; - } - /* check if VAR may be clobbered between VLOAD and use */ - ir_use_list *use_list = &ctx->use_lists[insn->op2]; - ir_ref n = use_list->count; - ir_ref *p = &ctx->use_edges[use_list->refs]; - for (; n > 0; p++, n--) { - ir_ref use = *p; - if (ctx->ir_base[use].op == IR_VSTORE) { - if (use > IR_LIVE_POS_TO_REF(ival->use_pos->pos) && use < IR_LIVE_POS_TO_REF(use_pos->pos)) { - return 0; - } - } else if (ctx->ir_base[use].op == IR_VADDR) { - return 0; - } - } - } - ival->stack_spill_pos = ctx->ir_base[insn->op2].op3; - return 1; } return 0; @@ -3554,7 +3491,7 @@ static int ir_linear_scan(ir_ctx *ctx) for (j = ctx->vregs_count; j != 0; j--) { ival = ctx->live_intervals[j]; if (ival) { - if (!(ival->flags & (IR_LIVE_INTERVAL_MEM_PARAM|IR_LIVE_INTERVAL_MEM_LOAD)) + if (!(ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) || !ir_ival_spill_for_fuse_load(ctx, ival, &data)) { ir_add_to_unhandled(&unhandled, ival); } diff --git a/ext/opcache/jit/ir/ir_save.c b/ext/opcache/jit/ir/ir_save.c index 5ba986fadd481..dd955172950c8 100644 --- a/ext/opcache/jit/ir/ir_save.c +++ b/ext/opcache/jit/ir/ir_save.c @@ -40,6 +40,11 @@ void ir_print_proto_ex(uint8_t flags, ir_type ret_type, uint32_t params_count, c } else if (flags & IR_BUILTIN_FUNC) { fprintf(f, " __builtin"); } + if (flags & IR_CONST_FUNC) { + fprintf(f, " __const"); + } else if (flags & IR_PURE_FUNC) { + fprintf(f, " __pure"); + } } static void ir_save_dessa_moves(const ir_ctx *ctx, int b, ir_block *bb, FILE *f) @@ -109,6 +114,10 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) fprintf(f, "sym(%s%s)", (save_flags & IR_SAVE_SAFE_NAMES) ? "@" : "", ir_get_str(ctx, insn->val.name)); + } else if (insn->op == IR_LABEL) { + fprintf(f, "label(%s%s)", + (save_flags & IR_SAVE_SAFE_NAMES) ? "@" : "", + ir_get_str(ctx, insn->val.name)); } else if (insn->op == IR_FUNC_ADDR) { fprintf(f, "func *"); ir_print_const(ctx, insn, f, true); @@ -272,6 +281,13 @@ void ir_save(const ir_ctx *ctx, uint32_t save_flags, FILE *f) fprintf(f, "%s%d", first ? "(" : ", ", ref); first = 0; break; + case IR_OPND_LABEL_REF: + if (ref) { + IR_ASSERT(IR_IS_CONST_REF(ref)); + fprintf(f, "%sc_%d", first ? "(" : ", ", -ref); + first = 0; + } + break; } } else if (opnd_kind == IR_OPND_NUM) { fprintf(f, "%s%d", first ? "(" : ", ", ref); diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 45df92ec2be6f..e2f38a058aeba 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -1508,8 +1508,8 @@ static bool ir_may_promote_f2d(ir_ctx *ctx, ir_ref ref) switch (insn->op) { case IR_FP2FP: return 1; - case IR_INT2FP: - return ctx->use_lists[ref].count == 1; +// case IR_INT2FP: +// return ctx->use_lists[ref].count == 1; case IR_NEG: case IR_ABS: return ctx->use_lists[ref].count == 1 && @@ -2110,7 +2110,9 @@ static bool ir_try_promote_induction_var_ext(ir_ctx *ctx, ir_ref ext_ref, ir_ref && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op1].op)) { ctx->ir_base[use].op1 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op1], op, type); } else { - ctx->ir_base[use].op1 = ir_ext_ref(ctx, use, use_insn->op1, op, type, worklist); + ir_ref tmp = ir_ext_ref(ctx, use, use_insn->op1, op, type, worklist); + use_insn = &ctx->ir_base[use]; + use_insn->op1 = tmp; } ir_bitqueue_add(worklist, use); } @@ -2119,7 +2121,9 @@ static bool ir_try_promote_induction_var_ext(ir_ctx *ctx, ir_ref ext_ref, ir_ref && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op2].op)) { ctx->ir_base[use].op2 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op2], op, type); } else { - ctx->ir_base[use].op2 = ir_ext_ref(ctx, use, use_insn->op2, op, type, worklist); + ir_ref tmp = ir_ext_ref(ctx, use, use_insn->op2, op, type, worklist); + use_insn = &ctx->ir_base[use]; + use_insn->op2 = tmp; } ir_bitqueue_add(worklist, use); } @@ -2147,7 +2151,9 @@ static bool ir_try_promote_induction_var_ext(ir_ctx *ctx, ir_ref ext_ref, ir_ref && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op1].op)) { ctx->ir_base[use].op1 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op1], op, type); } else { - ctx->ir_base[use].op1 = ir_ext_ref(ctx, use, use_insn->op1, op, type, worklist); + ir_ref tmp = ir_ext_ref(ctx, use, use_insn->op1, op, type, worklist); + use_insn = &ctx->ir_base[use]; + use_insn->op1 = tmp; } ir_bitqueue_add(worklist, use); } @@ -2156,7 +2162,9 @@ static bool ir_try_promote_induction_var_ext(ir_ctx *ctx, ir_ref ext_ref, ir_ref && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op2].op)) { ctx->ir_base[use].op2 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op2], op, type); } else { - ctx->ir_base[use].op2 = ir_ext_ref(ctx, use, use_insn->op2, op, type, worklist); + ir_ref tmp = ir_ext_ref(ctx, use, use_insn->op2, op, type, worklist); + use_insn = &ctx->ir_base[use]; + use_insn->op2 = tmp; } ir_bitqueue_add(worklist, use); } @@ -2178,7 +2186,8 @@ static bool ir_try_promote_induction_var_ext(ir_ctx *ctx, ir_ref ext_ref, ir_ref && !IR_IS_SYM_CONST(ctx->ir_base[phi_insn->op2].op)) { ctx->ir_base[phi_ref].op2 = ir_ext_const(ctx, &ctx->ir_base[phi_insn->op2], op, type); } else { - ctx->ir_base[phi_ref].op2 = ir_ext_ref(ctx, phi_ref, phi_insn->op2, op, type, worklist); + ir_ref tmp = ir_ext_ref(ctx, phi_ref, phi_insn->op2, op, type, worklist); + ctx->ir_base[phi_ref].op2 = tmp; } return 1; @@ -2251,42 +2260,6 @@ static void ir_merge_blocks(ir_ctx *ctx, ir_ref end, ir_ref begin, ir_bitqueue * ir_ref prev, next; ir_use_list *use_list; - if (ctx->use_lists[begin].count > 1) { - ir_ref *p, n, i, use; - ir_insn *use_insn; - ir_ref region = end; - ir_ref next = IR_UNUSED; - - while (!IR_IS_BB_START(ctx->ir_base[region].op)) { - region = ctx->ir_base[region].op1; - } - - use_list = &ctx->use_lists[begin]; - n = use_list->count; - for (p = &ctx->use_edges[use_list->refs], i = 0; i < n; p++, i++) { - use = *p; - use_insn = &ctx->ir_base[use]; - if (ir_op_flags[use_insn->op] & IR_OP_FLAG_CONTROL) { - IR_ASSERT(!next); - next = use; - } else { - IR_ASSERT(use_insn->op == IR_VAR); - IR_ASSERT(use_insn->op1 == begin); - use_insn->op1 = region; - if (ir_use_list_add(ctx, region, use)) { - /* restore after reallocation */ - use_list = &ctx->use_lists[begin]; - n = use_list->count; - p = &ctx->use_edges[use_list->refs + i]; - } - } - } - - IR_ASSERT(next); - ctx->use_edges[use_list->refs] = next; - use_list->count = 1; - } - IR_ASSERT(ctx->ir_base[begin].op == IR_BEGIN); IR_ASSERT(ctx->ir_base[end].op == IR_END); IR_ASSERT(ctx->ir_base[begin].op1 == end); @@ -3595,7 +3568,10 @@ void ir_iter_opt(ir_ctx *ctx, ir_bitqueue *worklist) if (!(ctx->flags & IR_OPT_CFG)) { /* pass */ } else if (insn->op == IR_BEGIN) { - if (insn->op1 && ctx->ir_base[insn->op1].op == IR_END) { + if (insn->op1 + && !insn->op2 /* no computed goto label */ + && ctx->use_lists[i].count == 1 + && ctx->ir_base[insn->op1].op == IR_END) { ir_merge_blocks(ctx, insn->op1, i, worklist); } } else if (insn->op == IR_MERGE) { diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 42e4eee7da0fc..7f714dd11d27c 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -66,7 +66,7 @@ IR_ALWAYS_INLINE ir_mem IR_MEM(ir_reg base, int32_t offset, ir_reg index, int32_ #define IR_SPILL_POS_TO_OFFSET(offset) \ ((ctx->flags & IR_USE_FRAME_POINTER) ? \ - ((offset) - (ctx->stack_frame_size - ctx->stack_frame_alignment)) : \ + ((offset) - ctx->stack_frame_size) : \ ((offset) + ctx->call_stack_size)) |.macro ASM_EXPAND_OP_MEM, MACRO, op, type, op1 @@ -892,6 +892,9 @@ typedef struct _ir_backend_data { bool double_abs_const; bool float_abs_const; bool double_zero_const; + bool u2d_const; + bool u2f_const; + bool resolved_label_syms; } ir_backend_data; #define IR_GP_REG_NAME(code, name64, name32, name16, name8, name8h) \ @@ -1087,6 +1090,7 @@ const char *ir_reg_name(int8_t reg, ir_type type) _(SSE_TRUNC) \ _(SSE_NEARBYINT) \ _(BIT_OP) \ + _(IGOTO_DUP) \ #define IR_LEA_FIRST IR_LEA_OB #define IR_LEA_LAST IR_LEA_O_SYM @@ -1110,35 +1114,24 @@ const char *ir_rule_name[IR_LAST_OP] = { static bool ir_may_fuse_addr(ir_ctx *ctx, const ir_insn *addr_insn) { - if (sizeof(void*) == 4) { - return 1; + if (addr_insn->op == IR_LABEL) { + return 0; } else if (IR_IS_SYM_CONST(addr_insn->op)) { void *addr = ir_sym_addr(ctx, addr_insn); if (!addr) { return 0; } - return IR_IS_SIGNED_32BIT((int64_t)(intptr_t)addr); + return (sizeof(void*) == 4) || IR_IS_SIGNED_32BIT((int64_t)(intptr_t)addr); } else { - return IR_IS_SIGNED_32BIT(addr_insn->val.i64); + return (sizeof(void*) == 4) || IR_IS_SIGNED_32BIT(addr_insn->val.i64); } } static bool ir_may_fuse_imm(ir_ctx *ctx, const ir_insn *val_insn) { if (val_insn->type == IR_ADDR) { - if (sizeof(void*) == 4) { - return 1; - } else if (IR_IS_SYM_CONST(val_insn->op)) { - void *addr = ir_sym_addr(ctx, val_insn); - - if (!addr) { - return 0; - } - return IR_IS_SIGNED_32BIT((intptr_t)addr); - } else { - return IR_IS_SIGNED_32BIT(val_insn->val.i64); - } + return ir_may_fuse_addr(ctx, val_insn); } else { return (ir_type_size[val_insn->type] <= 4 || IR_IS_SIGNED_32BIT(val_insn->val.i64)); } @@ -1517,6 +1510,11 @@ op2_const: constraints->tmp_regs[0] = IR_TMP_REG(1, ctx->ir_base[insn->op1].type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF); n = 1; } + if (IR_IS_TYPE_UNSIGNED(ctx->ir_base[insn->op1].type) + && ir_type_size[ctx->ir_base[insn->op1].type] >= sizeof(void*)) { + constraints->tmp_regs[n] = IR_TMP_REG(2, ctx->ir_base[insn->op1].type, IR_USE_SUB_REF, IR_DEF_SUB_REF); + n++; + } break; case IR_ABS_INT: flags = IR_DEF_CONFLICTS_WITH_INPUT_REGS | IR_USE_MUST_BE_IN_REG | IR_OP1_MUST_BE_IN_REG; @@ -1542,6 +1540,7 @@ op2_const: case IR_GUARD_NOT: flags = IR_OP2_SHOULD_BE_IN_REG; break; + case IR_IGOTO: case IR_IJMP: flags = IR_OP2_SHOULD_BE_IN_REG; break; @@ -1574,7 +1573,7 @@ op2_const: } break; case IR_VA_ARG: - flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG; + flags = IR_USE_MUST_BE_IN_REG | IR_OP2_MUST_BE_IN_REG | IR_DEF_CONFLICTS_WITH_INPUT_REGS; constraints->tmp_regs[0] = IR_TMP_REG(3, IR_ADDR, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF); n = 1; insn = &ctx->ir_base[ref]; @@ -1669,7 +1668,9 @@ static void ir_match_fuse_addr(ir_ctx *ctx, ir_ref addr_ref) do { ir_insn *insn = &ctx->ir_base[*p]; - if (insn->op != IR_LOAD && (insn->op != IR_STORE || insn->op3 == addr_ref)) { + if (insn->op != IR_LOAD + && insn->op != IR_LOAD_v + && ((insn->op != IR_STORE && insn->op != IR_STORE_v) || insn->op3 == addr_ref)) { return; } p++; @@ -1752,7 +1753,7 @@ static bool ir_match_has_mem_deps(ir_ctx *ctx, ir_ref ref, ir_ref root) do { ir_insn *insn = &ctx->ir_base[pos]; - if (insn->op == IR_STORE) { + if (insn->op == IR_STORE || insn->op == IR_STORE_v || insn->op == IR_VSTORE || insn->op == IR_VSTORE_v) { // TODO: check if LOAD and STORE addresses may alias return 1; } else if (insn->op == IR_CALL) { @@ -1766,8 +1767,9 @@ static bool ir_match_has_mem_deps(ir_ctx *ctx, ir_ref ref, ir_ref root) static void ir_match_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root) { - if (ir_in_same_block(ctx, ref) - && ctx->ir_base[ref].op == IR_LOAD) { + if (ir_in_same_block(ctx, ref) && + (ctx->ir_base[ref].op == IR_LOAD || ctx->ir_base[ref].op == IR_LOAD_v || + ctx->ir_base[ref].op == IR_VLOAD || ctx->ir_base[ref].op == IR_VLOAD_v)) { if (ctx->use_lists[ref].count == 2 && !ir_match_has_mem_deps(ctx, ref, root)) { ir_ref addr_ref = ctx->ir_base[ref].op2; @@ -1792,7 +1794,7 @@ static bool ir_match_try_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root) ir_insn *insn = &ctx->ir_base[ref]; if (ir_in_same_block(ctx, ref) - && insn->op == IR_LOAD) { + && (insn->op == IR_LOAD || insn->op == IR_LOAD_v || insn->op == IR_VLOAD || insn->op == IR_VLOAD_v)) { if (ctx->use_lists[ref].count == 2 && !ir_match_has_mem_deps(ctx, ref, root)) { ir_ref addr_ref = ctx->ir_base[ref].op2; @@ -1814,8 +1816,6 @@ static bool ir_match_try_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root) && ir_get_param_reg(ctx, ref) == IR_REG_NONE) { return 1; } - } else if (ctx->ir_base[ref].op == IR_VLOAD) { - return 1; } return 0; } @@ -2462,8 +2462,21 @@ binop_fp: case IR_IJMP: ir_match_fuse_load(ctx, insn->op2, ref); return insn->op; + case IR_IGOTO: + if (ctx->ir_base[insn->op1].op == IR_MERGE || ctx->ir_base[insn->op1].op == IR_LOOP_BEGIN) { + ir_insn *merge = &ctx->ir_base[insn->op1]; + ir_ref *p, n = merge->inputs_count; + + for (p = merge->ops + 1; n > 0; p++, n--) { + ir_ref input = *p; + IR_ASSERT(ctx->ir_base[input].op == IR_END || ctx->ir_base[input].op == IR_LOOP_END); + ctx->rules[input] = IR_IGOTO_DUP; + } + } + ir_match_fuse_load(ctx, insn->op2, ref); + return insn->op; case IR_VAR: - return IR_SKIPPED | IR_VAR; + return IR_STATIC_ALLOCA; case IR_PARAM: #ifndef _WIN64 if (ctx->value_params && ctx->value_params[insn->op3 - 1].align) { @@ -2617,7 +2630,15 @@ store_int: return IR_VSTORE_FP; } break; + case IR_VSTORE_v: + if (IR_IS_TYPE_INT(ctx->ir_base[insn->op3].type)) { + return IR_VSTORE_INT; + } else { + return IR_VSTORE_FP; + } + break; case IR_LOAD: + case IR_LOAD_v: ir_match_fuse_addr(ctx, insn->op2); if (IR_IS_TYPE_INT(insn->type)) { return IR_LOAD_INT; @@ -2635,6 +2656,14 @@ store_int: return IR_STORE_FP; } break; + case IR_STORE_v: + ir_match_fuse_addr(ctx, insn->op2); + if (IR_IS_TYPE_INT(ctx->ir_base[insn->op3].type)) { + return IR_STORE_INT; + } else { + return IR_STORE_FP; + } + break; case IR_RLOAD: if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), insn->op2)) { return IR_SKIPPED | IR_RLOAD; @@ -3175,7 +3204,7 @@ static void ir_emit_load_imm_fp(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref sr | xorpd xmm(reg-IR_REG_FP_FIRST), xmm(reg-IR_REG_FP_FIRST) } } else { - label = ir_const_label(ctx, src); + label = ir_get_const_label(ctx, src); | ASM_FP_REG_TXT_OP movs, type, reg, [=>label] } } @@ -3229,6 +3258,38 @@ static void ir_load_local_addr(ir_ctx *ctx, ir_reg reg, ir_ref src) } } +static void ir_resolve_label_syms(ir_ctx *ctx) +{ + uint32_t b; + ir_block *bb; + + for (b = 1, bb = &ctx->cfg_blocks[b]; b <= ctx->cfg_blocks_count; bb++, b++) { + ir_insn *insn = &ctx->ir_base[bb->start]; + + if (insn->op == IR_BEGIN && insn->op2) { + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_LABEL); + ctx->ir_base[insn->op2].val.u32_hi = b; + } + } +} + +static void ir_emit_load_label_addr(ir_ctx *ctx, ir_reg reg, ir_insn *label) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + + if (!data->resolved_label_syms) { + data->resolved_label_syms = 1; + ir_resolve_label_syms(ctx); + } + + IR_ASSERT(label->op == IR_LABEL); + int b = label->val.u32_hi; + + b = ir_skip_empty_target_blocks(ctx, b); + | lea Ra(reg), aword [=>b] +} + static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) { if (IR_IS_CONST_REF(src)) { @@ -3241,9 +3302,11 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) } else if (insn->op == IR_STR) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; - int label = ir_const_label(ctx, src); + int label = ir_get_const_label(ctx, src); | lea Ra(reg), aword [=>label] + } else if (insn->op == IR_LABEL) { + ir_emit_load_label_addr(ctx, reg, insn); } else { ir_emit_load_imm_int(ctx, type, reg, insn->val.i64); } @@ -3289,7 +3352,7 @@ static void ir_emit_store_mem_int_const(ir_ctx *ctx, ir_type type, ir_mem mem, i IR_ASSERT(IR_IS_CONST_REF(src)); if (val_insn->op == IR_STR) { - int label = ir_const_label(ctx, src); + int label = ir_get_const_label(ctx, src); IR_ASSERT(tmp_reg != IR_REG_NONE); |.if X64 @@ -3298,6 +3361,11 @@ static void ir_emit_store_mem_int_const(ir_ctx *ctx, ir_type type, ir_mem mem, i |.else | ASM_TMEM_TXT_OP mov, aword, mem, =>label |.endif + } else if (val_insn->op == IR_LABEL) { + IR_ASSERT(tmp_reg != IR_REG_NONE); + tmp_reg = IR_REG_NUM(tmp_reg); + ir_emit_load_label_addr(ctx, tmp_reg, val_insn); + ir_emit_store_mem_int(ctx, type, mem, tmp_reg); } else { int64_t val = val_insn->val.i64; @@ -3726,7 +3794,8 @@ static ir_mem ir_fuse_load(ir_ctx *ctx, ir_ref root, ir_ref ref) ir_insn *load_insn = &ctx->ir_base[ref]; ir_reg reg; - IR_ASSERT(load_insn->op == IR_LOAD); + IR_ASSERT(load_insn->op == IR_LOAD || load_insn->op == IR_LOAD_v || + load_insn->op == IR_VLOAD || load_insn->op == IR_VLOAD_v); if (UNEXPECTED(ctx->rules[ref] & IR_FUSED_REG)) { reg = ir_get_fused_reg(ctx, root, ref * sizeof(ir_ref) + 2); } else { @@ -3762,9 +3831,11 @@ static void ir_emit_load_ex(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src, i } else if (insn->op == IR_STR) { ir_backend_data *data = ctx->data; dasm_State **Dst = &data->dasm_state; - int label = ir_const_label(ctx, src); + int label = ir_get_const_label(ctx, src); | lea Ra(reg), aword [=>label] + } else if (insn->op == IR_LABEL) { + ir_emit_load_label_addr(ctx, reg, insn); } else { ir_emit_load_imm_int(ctx, type, reg, insn->val.i64); } @@ -3862,7 +3933,7 @@ static void ir_emit_prologue(ir_ctx *ctx) if (ctx->flags & IR_USE_FRAME_POINTER) { fp = IR_REG_FRAME_POINTER; - offset = -(ctx->stack_frame_size - ctx->stack_frame_alignment - ctx->locals_area_size); + offset = -(ctx->stack_frame_size - ctx->locals_area_size); } else { fp = IR_REG_STACK_POINTER; offset = ctx->locals_area_size + ctx->call_stack_size; @@ -5607,7 +5678,7 @@ static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn) break; } } else if (IR_IS_CONST_REF(op2)) { - int label = ir_const_label(ctx, op2); + int label = ir_get_const_label(ctx, op2); switch (insn->op) { default: @@ -5714,7 +5785,7 @@ static void ir_emit_binop_avx(ir_ctx *ctx, ir_ref def, ir_insn *insn) break; } } else if (IR_IS_CONST_REF(op2)) { - int label = ir_const_label(ctx, op2); + int label = ir_get_const_label(ctx, op2); switch (insn->op) { default: @@ -6126,7 +6197,7 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_ } | ASM_FP_REG_REG_OP ucomis, type, op1_reg, op2_reg } else if (IR_IS_CONST_REF(op2)) { - int label = ir_const_label(ctx, op2); + int label = ir_get_const_label(ctx, op2); | ASM_FP_REG_TXT_OP ucomis, type, op1_reg, [=>label] } else { @@ -6975,7 +7046,7 @@ static void ir_emit_return_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn) } else if ((type == IR_FLOAT && value->val.f == 1.0) || (type == IR_DOUBLE && value->val.d == 1.0)) { | fld1 } else { - int label = ir_const_label(ctx, insn->op2); + int label = ir_get_const_label(ctx, insn->op2); if (type == IR_DOUBLE) { | fld qword [=>label] @@ -7260,7 +7331,20 @@ static void ir_emit_trunc(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_emit_load(ctx, src_type, op1_reg, insn->op1); } if (op1_reg != def_reg) { +#ifdef IR_TARGET_X86 + if (ir_type_size[dst_type] == 1 + && (op1_reg == IR_REG_RBP || op1_reg == IR_REG_RSI || op1_reg == IR_REG_RDI)) { + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + + ir_emit_mov(ctx, src_type, def_reg, op1_reg); + | and Rb(def_reg), 0xff + } else { + ir_emit_mov(ctx, dst_type, def_reg, op1_reg); + } +#else ir_emit_mov(ctx, dst_type, def_reg, op1_reg); +#endif } } else { ir_emit_load_ex(ctx, dst_type, def_reg, insn->op1, def); @@ -7385,7 +7469,7 @@ static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } } else if (IR_IS_CONST_REF(insn->op1)) { - int label = ir_const_label(ctx, insn->op1); + int label = ir_get_const_label(ctx, insn->op1); | ASM_FP_REG_TXT_OP movs, dst_type, def_reg, [=>label] } else { @@ -7417,13 +7501,80 @@ static void ir_emit_int2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) IR_ASSERT(IR_IS_TYPE_INT(src_type)); IR_ASSERT(IR_IS_TYPE_FP(dst_type)); IR_ASSERT(def_reg != IR_REG_NONE); + + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, src_type, op1_reg, insn->op1); + } + + if (IR_IS_TYPE_UNSIGNED(src_type) && ir_type_size[src_type] >= sizeof(void*)) { + ir_reg tmp_reg = ctx->regs[def][2]; + + IR_ASSERT(tmp_reg != IR_REG_NONE); + if (op1_reg == IR_REG_NONE) { + if (IR_IS_CONST_REF(insn->op1)) { + IR_ASSERT(0); + } else { + ir_mem mem; + + if (ir_rule(ctx, insn->op1) & IR_FUSED) { + mem = ir_fuse_load(ctx, def, insn->op1); + } else { + mem = ir_ref_spill_slot(ctx, insn->op1); + } + ir_emit_load_mem_int(ctx, src_type, tmp_reg, mem); + op1_reg = tmp_reg; + } + } + if (sizeof(void*) == 4) { + if (tmp_reg == op1_reg) { + | add Rd(op1_reg), 0x80000000 + } else { + | lea Rd(tmp_reg), dword [Rd(op1_reg)+0x80000000] + op1_reg = tmp_reg; + } + } else { +|.if X64 + | test Rq(op1_reg), Rq(op1_reg) + | js >1 + |.cold_code + |1: + if (tmp_reg != op1_reg) { + | mov Rq(tmp_reg), Rq(op1_reg) + } + | shr Rq(tmp_reg), 1 + | adc Rq(tmp_reg), 0 + if (dst_type == IR_DOUBLE) { + if (ctx->mflags & IR_X86_AVX) { + | vxorps xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + | vcvtsi2sd xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), Rq(tmp_reg) + | vaddsd xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + } else { + | pxor xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + | cvtsi2sd xmm(def_reg-IR_REG_FP_FIRST), Rq(tmp_reg) + | addsd xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + } + } else { + IR_ASSERT(dst_type == IR_FLOAT); + if (ctx->mflags & IR_X86_AVX) { + | vxorps xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + | vcvtsi2ss xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), Rq(tmp_reg) + | vaddss xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + } else { + | pxor xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + | cvtsi2ss xmm(def_reg-IR_REG_FP_FIRST), Rq(tmp_reg) + | addss xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST) + } + } + | jmp >2 + |.code +|.endif + } + } + if (op1_reg != IR_REG_NONE) { bool src64 = 0; - if (IR_REG_SPILLED(op1_reg)) { - op1_reg = IR_REG_NUM(op1_reg); - ir_emit_load(ctx, src_type, op1_reg, insn->op1); - } if (IR_IS_TYPE_SIGNED(src_type)) { if (ir_type_size[src_type] < 4) { |.if X64 @@ -7462,7 +7613,6 @@ static void ir_emit_int2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) || } |.endif } else { - // TODO: uint64_t -> double src64 = 1; } } @@ -7508,6 +7658,40 @@ static void ir_emit_int2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } |.endif } + |2: + if (sizeof(void*) == 4 && IR_IS_TYPE_UNSIGNED(src_type) && ir_type_size[src_type] >= sizeof(void*)) { + if (dst_type == IR_DOUBLE) { + uint32_t c = (sizeof(void*) == 4) ? 0x41e00000 : 0x43e00000; + if (!data->u2d_const) { + data->u2d_const = 1; + ir_rodata(ctx); + |.align 8 + |->u2d_const: + |.dword 0, c + |.code + } + if (ctx->mflags & IR_X86_AVX) { + | vaddsd xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), qword [->u2d_const] + } else { + | addsd xmm(def_reg-IR_REG_FP_FIRST), qword [->u2d_const] + } + } else { + uint32_t c = (sizeof(void*) == 4) ? 0x4f000000 : 0x5f000000; + if (!data->u2f_const) { + data->u2f_const = 1; + ir_rodata(ctx); + |.align 4 + |->u2f_const: + |.dword c + |.code + } + if (ctx->mflags & IR_X86_AVX) { + | vaddss xmm(def_reg-IR_REG_FP_FIRST), xmm(def_reg-IR_REG_FP_FIRST), dword [->u2f_const] + } else { + | addss xmm(def_reg-IR_REG_FP_FIRST), dword [->u2f_const] + } + } + } } else if (IR_IS_CONST_REF(insn->op1)) { IR_ASSERT(0); } else { @@ -7625,7 +7809,7 @@ static void ir_emit_fp2int(ir_ctx *ctx, ir_ref def, ir_insn *insn) |.endif } } else if (IR_IS_CONST_REF(insn->op1)) { - int label = ir_const_label(ctx, insn->op1); + int label = ir_get_const_label(ctx, insn->op1); if (!dst64) { if (src_type == IR_DOUBLE) { @@ -7746,7 +7930,7 @@ static void ir_emit_fp2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } } else if (IR_IS_CONST_REF(insn->op1)) { - int label = ir_const_label(ctx, insn->op1); + int label = ir_get_const_label(ctx, insn->op1); if (src_type == IR_DOUBLE) { if (ctx->mflags & IR_X86_AVX) { @@ -8429,7 +8613,7 @@ static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (ctx->flags & IR_USE_FRAME_POINTER) { fp = IR_REG_FRAME_POINTER; - reg_save_area_offset = -(ctx->stack_frame_size - ctx->stack_frame_alignment - ctx->locals_area_size); + reg_save_area_offset = -(ctx->stack_frame_size - ctx->locals_area_size); overflow_arg_area_offset = sizeof(void*) * 2 + ctx->param_stack_size; } else { fp = IR_REG_STACK_POINTER; @@ -8588,11 +8772,11 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } | add Ra(tmp_reg), IR_MAX(ir_type_size[type], sizeof(void*)) } else { - int size = (uint32_t)insn->op3 >> 3; + int size = IR_VA_ARG_SIZE(insn->op3); if (def_reg != IR_REG_NONE) { IR_ASSERT(type == IR_ADDR); - int align = 1U << (insn->op3 & 0x7); + int align = IR_VA_ARG_ALIGN(insn->op3); if (align > (int)sizeof(void*)) { | add Ra(tmp_reg), (align-1) @@ -8604,7 +8788,7 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) } #endif | mov aword [Ra(op2_reg)+offset], Ra(tmp_reg) - if (def_reg && IR_REG_SPILLED(ctx->regs[def][0])) { + if (def_reg != IR_REG_NONE && IR_REG_SPILLED(ctx->regs[def][0])) { ir_emit_store(ctx, type, def, def_reg); } #elif defined(IR_TARGET_X64) @@ -8632,8 +8816,8 @@ static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (insn->op3) { /* long struct arguemnt */ IR_ASSERT(type == IR_ADDR); - int align = 1U << (insn->op3 & 0x7); - int size = (uint32_t)insn->op3 >> 3; + int align = IR_VA_ARG_ALIGN(insn->op3); + int size = IR_VA_ARG_SIZE(insn->op3); | mov Ra(tmp_reg), aword [Ra(op2_reg)+(offset+offsetof(ir_va_list, overflow_arg_area))] if (align > (int)sizeof(void*)) { @@ -9701,6 +9885,19 @@ static void ir_emit_ijmp(ir_ctx *ctx, ir_ref def, ir_insn *insn) ir_reg op2_reg = ctx->regs[def][2]; if (IR_IS_CONST_REF(insn->op2)) { + if (ctx->ir_base[insn->op2].op == IR_LABEL) { + if (!data->resolved_label_syms) { + data->resolved_label_syms = 1; + ir_resolve_label_syms(ctx); + } + + uint32_t target = ctx->ir_base[insn->op2].val.u32_hi; + target = ir_skip_empty_target_blocks(ctx, target); + + | jmp =>target + return; + } + void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op2]); if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { @@ -10478,6 +10675,7 @@ static void ir_emit_param_move(ir_ctx *ctx, uint8_t type, ir_reg from_reg, ir_re { ir_reg fp = (ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FRAME_POINTER : IR_REG_STACK_POINTER; + offset = IR_SPILL_POS_TO_OFFSET(offset); IR_ASSERT(from_reg != IR_REG_NONE || to_reg != IR_REG_NONE); if (IR_IS_TYPE_INT(type)) { @@ -10518,6 +10716,7 @@ static void ir_emit_load_params(ir_ctx *ctx) const int8_t *int_reg_params = _ir_int_reg_params; const int8_t *fp_reg_params = _ir_fp_reg_params; int32_t stack_offset = 0; + int32_t stack_start = 0; #ifdef IR_TARGET_X86 if (sizeof(void*) == 4 && (ctx->flags & IR_FASTCALL_FUNC)) { @@ -10529,9 +10728,11 @@ static void ir_emit_load_params(ir_ctx *ctx) #endif if (ctx->flags & IR_USE_FRAME_POINTER) { - stack_offset = sizeof(void*) * 2; /* skip old frame pointer and return address */ + /* skip old frame pointer and return address */ + stack_start = sizeof(void*) * 2 + ctx->stack_frame_size; } else { - stack_offset = sizeof(void*) + ctx->stack_frame_size + ctx->call_stack_size; /* skip return address */ + /* skip return address */ + stack_start = sizeof(void*) + ctx->stack_frame_size; } n = use_list->count; for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { @@ -10573,12 +10774,9 @@ static void ir_emit_load_params(ir_ctx *ctx) if (ctx->vregs[use]) { dst_reg = IR_REG_NUM(ctx->regs[use][0]); IR_ASSERT(src_reg != IR_REG_NONE || dst_reg != IR_REG_NONE || - stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos + - ((ctx->flags & IR_USE_FRAME_POINTER) ? - -(ctx->stack_frame_size - ctx->stack_frame_alignment) : - ctx->call_stack_size)); + stack_start + stack_offset == ctx->live_intervals[ctx->vregs[use]]->stack_spill_pos); if (src_reg != dst_reg) { - ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_offset); + ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_start + stack_offset); } if (dst_reg != IR_REG_NONE && IR_REG_SPILLED(ctx->regs[use][0])) { ir_emit_store(ctx, insn->type, use, dst_reg); @@ -10665,7 +10863,7 @@ static void ir_fix_param_spills(ir_ctx *ctx) if (ctx->flags & IR_USE_FRAME_POINTER) { /* skip old frame pointer and return address */ - stack_start = sizeof(void*) * 2 + (ctx->stack_frame_size - ctx->stack_frame_alignment); + stack_start = sizeof(void*) * 2 + ctx->stack_frame_size; } else { /* skip return address */ stack_start = sizeof(void*) + ctx->stack_frame_size; @@ -10786,6 +10984,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) case IR_MERGE: case IR_LOOP_BEGIN: case IR_LOOP_END: + case IR_IGOTO_DUP: break; #ifndef IR_REG_FP_RET1 case IR_CALL: @@ -10810,7 +11009,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) IR_REGSET_EXCL(available, reg); ctx->regs[i][0] = reg | IR_REG_SPILL_STORE; } else if (def_flags & IR_USE_MUST_BE_IN_REG) { - if (insn->op == IR_VLOAD + if ((insn->op == IR_VLOAD || insn->op == IR_VLOAD_v) && ctx->live_intervals[ctx->vregs[i]] && ctx->live_intervals[ctx->vregs[i]]->stack_spill_pos != -1 && ir_is_same_mem_var(ctx, i, ctx->ir_base[insn->op2].op3)) { @@ -10850,7 +11049,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { use = *p; use_insn = &ctx->ir_base[use]; - if (use_insn->op == IR_VLOAD) { + if (use_insn->op == IR_VLOAD || use_insn->op == IR_VLOAD_v) { if (ctx->vregs[use] && !ctx->live_intervals[ctx->vregs[use]]) { ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); @@ -10861,7 +11060,7 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) ival->vreg = ctx->vregs[use]; ival->stack_spill_pos = stack_spill_pos; } - } else if (use_insn->op == IR_VSTORE) { + } else if (use_insn->op == IR_VSTORE || use_insn->op == IR_VSTORE_v) { if (!IR_IS_CONST_REF(use_insn->op3) && ctx->vregs[use_insn->op3] && !ctx->live_intervals[ctx->vregs[use_insn->op3]]) { @@ -11006,7 +11205,6 @@ void ir_fix_stack_frame(ir_ctx *ctx) ctx->stack_frame_size = IR_ALIGNED_SIZE(ctx->stack_frame_size, sizeof(void*)); ctx->stack_frame_size += additional_size; - ctx->stack_frame_alignment = 0; ctx->call_stack_size = 0; if (ctx->flags2 & IR_16B_FRAME_ALIGNMENT) { @@ -11014,12 +11212,10 @@ void ir_fix_stack_frame(ir_ctx *ctx) if (!(ctx->flags & IR_FUNCTION)) { while (IR_ALIGNED_SIZE(ctx->stack_frame_size, 16) != ctx->stack_frame_size) { ctx->stack_frame_size += sizeof(void*); - ctx->stack_frame_alignment += sizeof(void*); } } else if (ctx->flags & IR_USE_FRAME_POINTER) { while (IR_ALIGNED_SIZE(ctx->stack_frame_size + sizeof(void*) * 2, 16) != ctx->stack_frame_size + sizeof(void*) * 2) { ctx->stack_frame_size += sizeof(void*); - ctx->stack_frame_alignment += sizeof(void*); } } else { if (!(ctx->flags & IR_NO_STACK_COMBINE)) { @@ -11028,7 +11224,6 @@ void ir_fix_stack_frame(ir_ctx *ctx) while (IR_ALIGNED_SIZE(ctx->stack_frame_size + ctx->call_stack_size + sizeof(void*), 16) != ctx->stack_frame_size + ctx->call_stack_size + sizeof(void*)) { ctx->stack_frame_size += sizeof(void*); - ctx->stack_frame_alignment += sizeof(void*); } } } @@ -11061,6 +11256,8 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) int ret; void *entry; size_t size; + ir_ref igoto_dup_ref = IR_UNUSED; + uint32_t igoto_dup_block = 0; data.ra_data.unused_slot_4 = 0; data.ra_data.unused_slot_2 = 0; @@ -11073,11 +11270,13 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) data.double_abs_const = 0; data.float_abs_const = 0; data.double_zero_const = 0; + data.u2d_const = 0; + data.u2f_const = 0; + data.resolved_label_syms = 0; ctx->data = &data; if (!ctx->live_intervals) { ctx->stack_frame_size = 0; - ctx->stack_frame_alignment = 0; ctx->call_stack_size = 0; ctx->used_preserved_regs = 0; ir_allocate_unique_spill_slots(ctx); @@ -11099,7 +11298,6 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) } ctx->stack_frame_size = ctx->fixed_stack_frame_size; ctx->call_stack_size = ctx->fixed_call_stack_size; - ctx->stack_frame_alignment = 0; } Dst = &data.dasm_state; @@ -11420,6 +11618,35 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_TAILCALL: ir_emit_tailcall(ctx, i, insn); break; + case IR_IGOTO_DUP: + if (bb->flags & IR_BB_DESSA_MOVES) { + ir_emit_dessa_moves(ctx, b, bb); + } + IR_ASSERT(!igoto_dup_ref && !igoto_dup_block); + igoto_dup_ref = i; + igoto_dup_block = b; + b = ctx->cfg_edges[bb->successors]; + bb = &ctx->cfg_blocks[b]; + i = bb->start; + insn = &ctx->ir_base[i]; + rule = &ctx->rules[i]; + break; + case IR_IGOTO: + if ((ctx->ir_base[insn->op1].op == IR_MERGE || ctx->ir_base[insn->op1].op == IR_LOOP_BEGIN) + && (ctx->rules[ctx->ir_base[insn->op1].op1] & IR_RULE_MASK) == IR_IGOTO_DUP + && igoto_dup_ref) { + ir_emit_ijmp(ctx, i, insn); + b = igoto_dup_block; + bb = &ctx->cfg_blocks[b]; + i = igoto_dup_ref; + insn = &ctx->ir_base[i]; + rule = &ctx->rules[i]; + igoto_dup_block= 0; + igoto_dup_ref = 0; + break; + } + IR_ASSERT(!igoto_dup_ref && !igoto_dup_block); + IR_FALLTHROUGH; case IR_IJMP: ir_emit_ijmp(ctx, i, insn); break; @@ -11449,6 +11676,7 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) ir_emit_vaddr(ctx, i, insn); break; case IR_VLOAD: + case IR_VLOAD_v: ir_emit_vload(ctx, i, insn); break; case IR_VSTORE_INT: @@ -11691,6 +11919,28 @@ next_block:; } while (i != 0); } + if ((ctx->flags2 & IR_HAS_BLOCK_ADDR) && ctx->loader && ctx->loader->add_label) { + for (b = 1, bb = &ctx->cfg_blocks[b]; b <= ctx->cfg_blocks_count; bb++, b++) { + ir_insn *insn = &ctx->ir_base[bb->start]; + + if (insn->op == IR_BEGIN && insn->op2) { + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_LABEL); + ctx->ir_base[insn->op2].val.u32_hi = 0; + ctx->loader->add_label(ctx->loader, ir_get_str(ctx, ctx->ir_base[insn->op2].val.str), + (char*)entry + dasm_getpclabel(&data.dasm_state, ir_skip_empty_target_blocks(ctx, b))); + } + } + } else if (data.resolved_label_syms) { + for (b = 1, bb = &ctx->cfg_blocks[b]; b <= ctx->cfg_blocks_count; bb++, b++) { + ir_insn *insn = &ctx->ir_base[bb->start]; + + if (insn->op == IR_BEGIN && insn->op2) { + IR_ASSERT(ctx->ir_base[insn->op2].op == IR_LABEL); + ctx->ir_base[insn->op2].val.u32_hi = 0; + } + } + } + dasm_free(&data.dasm_state); ir_mem_flush(entry, size); diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 9dc6ea1c3b003..10be237d06b8e 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -493,7 +493,7 @@ static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, && func->type == ZEND_INTERNAL_FUNCTION && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0 && arg_num < func->internal_function.num_args) { - const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num]; + const zend_arg_info *arg_info = &func->internal_function.arg_info[arg_num]; if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL && ZEND_TYPE_IS_SET(arg_info->type) diff --git a/ext/pdo/pdo_dbh.c b/ext/pdo/pdo_dbh.c index beb3665c4d924..34f19e364faa7 100644 --- a/ext/pdo/pdo_dbh.c +++ b/ext/pdo/pdo_dbh.c @@ -1321,6 +1321,7 @@ static void cls_method_dtor(zval *el) /* {{{ */ { if (func->common.attributes) { zend_hash_release(func->common.attributes); } + zend_free_internal_arg_info(&func->internal_function, false); efree(func); } /* }}} */ @@ -1336,6 +1337,7 @@ static void cls_method_pdtor(zval *el) /* {{{ */ { if (func->common.attributes) { zend_hash_release(func->common.attributes); } + zend_free_internal_arg_info(&func->internal_function, true); pefree(func, 1); } /* }}} */ @@ -1408,7 +1410,19 @@ bool pdo_hash_methods(pdo_dbh_object_t *dbh_obj, int kind) if (funcs->arg_info) { zend_internal_function_info *info = (zend_internal_function_info*)funcs->arg_info; - func.arg_info = (zend_internal_arg_info*)funcs->arg_info + 1; + uint32_t num_arg_info = 1 + funcs->num_args; + if (func.fn_flags & ZEND_ACC_VARIADIC) { + num_arg_info++; + } + + zend_arg_info *arg_info = safe_pemalloc(num_arg_info, + sizeof(zend_arg_info), 0, dbh->is_persistent); + for (uint32_t i = 0; i < num_arg_info; i++) { + zend_convert_internal_arg_info(&arg_info[i], + &funcs->arg_info[i], i == 0, dbh->is_persistent); + } + + func.arg_info = arg_info + 1; func.num_args = funcs->num_args; if (info->required_num_args == (uint32_t)-1) { func.required_num_args = funcs->num_args; diff --git a/ext/reflection/php_reflection.c b/ext/reflection/php_reflection.c index d6e55c982b425..c6f681ee2276c 100644 --- a/ext/reflection/php_reflection.c +++ b/ext/reflection/php_reflection.c @@ -763,11 +763,6 @@ static void format_default_value(smart_str *str, zval *value) { } } -static inline bool has_internal_arg_info(const zend_function *fptr) { - return fptr->type == ZEND_INTERNAL_FUNCTION - && !(fptr->common.fn_flags & ZEND_ACC_USER_ARG_INFO); -} - /* {{{ _parameter_string */ static void _parameter_string(smart_str *str, zend_function *fptr, struct _zend_arg_info *arg_info, uint32_t offset, bool required, char* indent) { @@ -789,17 +784,15 @@ static void _parameter_string(smart_str *str, zend_function *fptr, struct _zend_ if (ZEND_ARG_IS_VARIADIC(arg_info)) { smart_str_appends(str, "..."); } - smart_str_append_printf(str, "$%s", has_internal_arg_info(fptr) - ? ((zend_internal_arg_info*)arg_info)->name : ZSTR_VAL(arg_info->name)); + smart_str_append_printf(str, "$%s", ZSTR_VAL(arg_info->name)); if (!required && !ZEND_ARG_IS_VARIADIC(arg_info)) { if (fptr->type == ZEND_INTERNAL_FUNCTION) { smart_str_appends(str, " = "); /* TODO: We don't have a way to fetch the default value for an internal function * with userland arg info. */ - if (has_internal_arg_info(fptr) - && ((zend_internal_arg_info*)arg_info)->default_value) { - smart_str_appends(str, ((zend_internal_arg_info*)arg_info)->default_value); + if (arg_info->default_value) { + smart_str_append(str, arg_info->default_value); } else { smart_str_appends(str, ""); } @@ -1463,11 +1456,7 @@ static void reflection_parameter_factory(zend_function *fptr, zval *closure_obje } prop_name = reflection_prop_name(object); - if (has_internal_arg_info(fptr)) { - ZVAL_STRING(prop_name, ((zend_internal_arg_info*)arg_info)->name); - } else { - ZVAL_STR_COPY(prop_name, arg_info->name); - } + ZVAL_STR_COPY(prop_name, arg_info->name); } /* }}} */ @@ -1653,8 +1642,7 @@ static zend_result get_parameter_default(zval *result, parameter_reference *para /* We don't have a way to determine the default value for this case right now. */ return FAILURE; } - return zend_get_default_from_internal_arg_info( - result, (zend_internal_arg_info *) param->arg_info); + return zend_get_default_from_internal_arg_info(result, param->arg_info); } else { zval *default_value = get_default_from_recv((zend_op_array *) param->fptr, param->offset); if (!default_value) { @@ -2590,22 +2578,11 @@ ZEND_METHOD(ReflectionParameter, __construct) uint32_t i; position = -1; - if (has_internal_arg_info(fptr)) { - for (i = 0; i < num_args; i++) { - if (arg_info[i].name) { - if (strcmp(((zend_internal_arg_info*)arg_info)[i].name, ZSTR_VAL(arg_name)) == 0) { - position = i; - break; - } - } - } - } else { - for (i = 0; i < num_args; i++) { - if (arg_info[i].name) { - if (zend_string_equals(arg_name, arg_info[i].name)) { - position = i; - break; - } + for (i = 0; i < num_args; i++) { + if (arg_info[i].name) { + if (zend_string_equals(arg_name, arg_info[i].name)) { + position = i; + break; } } } @@ -2646,11 +2623,7 @@ ZEND_METHOD(ReflectionParameter, __construct) prop_name = reflection_prop_name(object); zval_ptr_dtor(prop_name); - if (has_internal_arg_info(fptr)) { - ZVAL_STRING(prop_name, ((zend_internal_arg_info*)arg_info)[position].name); - } else { - ZVAL_STR_COPY(prop_name, arg_info[position].name); - } + ZVAL_STR_COPY(prop_name, arg_info[position].name); return; failure: @@ -2689,11 +2662,7 @@ ZEND_METHOD(ReflectionParameter, getName) ZEND_PARSE_PARAMETERS_NONE(); GET_REFLECTION_OBJECT_PTR(param); - if (has_internal_arg_info(param->fptr)) { - RETURN_STRING(((zend_internal_arg_info *) param->arg_info)->name); - } else { - RETURN_STR_COPY(param->arg_info->name); - } + RETURN_STR_COPY(param->arg_info->name); } /* }}} */ @@ -2947,8 +2916,7 @@ ZEND_METHOD(ReflectionParameter, isDefaultValueAvailable) GET_REFLECTION_OBJECT_PTR(param); if (param->fptr->type == ZEND_INTERNAL_FUNCTION) { - RETURN_BOOL(!(param->fptr->common.fn_flags & ZEND_ACC_USER_ARG_INFO) - && ((zend_internal_arg_info*) (param->arg_info))->default_value); + RETURN_BOOL(param->arg_info->default_value); } else { zval *default_value = get_default_from_recv((zend_op_array *)param->fptr, param->offset); RETURN_BOOL(default_value != NULL); diff --git a/ext/uri/uriparser/src/UriParse.c b/ext/uri/uriparser/src/UriParse.c index ed851e94fbd9c..cada02301e00c 100644 --- a/ext/uri/uriparser/src/UriParse.c +++ b/ext/uri/uriparser/src/UriParse.c @@ -81,8 +81,7 @@ static const URI_CHAR * URI_FUNC(ParseAuthority)(URI_TYPE(ParserState) * state, static const URI_CHAR * URI_FUNC(ParseAuthorityTwo)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast); -static const URI_CHAR * URI_FUNC(ParseHexZero)(URI_TYPE(ParserState) * state, - const URI_CHAR * first, +static const URI_CHAR * URI_FUNC(ParseHexZero)(const URI_CHAR * first, const URI_CHAR * afterLast); static const URI_CHAR * URI_FUNC(ParseHierPart)(URI_TYPE(ParserState) * state, const URI_CHAR * first, @@ -92,10 +91,6 @@ static const URI_CHAR * URI_FUNC(ParseIpFutLoop)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory); -static const URI_CHAR * URI_FUNC(ParseIpFutStopGo)(URI_TYPE(ParserState) * state, - const URI_CHAR * first, - const URI_CHAR * afterLast, - UriMemoryManager * memory); static const URI_CHAR * URI_FUNC(ParseIpLit2)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, @@ -116,10 +111,6 @@ static const URI_CHAR * URI_FUNC(ParseOwnHost2)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory); -static const URI_CHAR * URI_FUNC(ParseOwnHostUserInfo)(URI_TYPE(ParserState) * state, - const URI_CHAR * first, - const URI_CHAR * afterLast, - UriMemoryManager * memory); static const URI_CHAR * URI_FUNC(ParseOwnHostUserInfoNz)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, @@ -160,8 +151,7 @@ static const URI_CHAR * URI_FUNC(ParsePctSubUnres)(URI_TYPE(ParserState) * state const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory); -static const URI_CHAR * URI_FUNC(ParsePort)(URI_TYPE(ParserState) * state, - const URI_CHAR * first, +static const URI_CHAR * URI_FUNC(ParsePort)(const URI_CHAR * first, const URI_CHAR * afterLast); static const URI_CHAR * URI_FUNC(ParseQueryFrag)(URI_TYPE(ParserState) * state, const URI_CHAR * first, @@ -292,8 +282,7 @@ URI_FUNC(ParseAuthorityTwo)(URI_TYPE(ParserState) * state, const URI_CHAR * firs switch (*first) { case _UT(':'): { - const URI_CHAR * const afterPort = - URI_FUNC(ParsePort)(state, first + 1, afterLast); + const URI_CHAR * const afterPort = URI_FUNC(ParsePort)(first + 1, afterLast); if (afterPort == NULL) { return NULL; } @@ -311,16 +300,17 @@ URI_FUNC(ParseAuthorityTwo)(URI_TYPE(ParserState) * state, const URI_CHAR * firs * [hexZero]->[HEXDIG][hexZero] * [hexZero]-> */ -static const URI_CHAR * URI_FUNC(ParseHexZero)(URI_TYPE(ParserState) * state, - const URI_CHAR * first, +static const URI_CHAR * URI_FUNC(ParseHexZero)(const URI_CHAR * first, const URI_CHAR * afterLast) { +tail_call: if (first >= afterLast) { return afterLast; } switch (*first) { case URI_SET_HEXDIG(_UT): - return URI_FUNC(ParseHexZero)(state, first + 1, afterLast); + first += 1; + goto tail_call; default: return first; @@ -356,49 +346,37 @@ static URI_INLINE const URI_CHAR * URI_FUNC(ParseHierPart)(URI_TYPE(ParserState) * [ipFutLoop]->[subDelims][ipFutStopGo] * [ipFutLoop]->[unreserved][ipFutStopGo] * [ipFutLoop]-><:>[ipFutStopGo] + * + * [ipFutStopGo]->[ipFutLoop] + * [ipFutStopGo]-> */ static const URI_CHAR * URI_FUNC(ParseIpFutLoop)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { - if (first >= afterLast) { - URI_FUNC(StopSyntax)(state, afterLast, memory); - return NULL; - } + const URI_CHAR * const originalFirst = first; - switch (*first) { - case _UT(':'): - case URI_SET_SUB_DELIMS(_UT): - case URI_SET_UNRESERVED(_UT): - return URI_FUNC(ParseIpFutStopGo)(state, first + 1, afterLast, memory); + while (first < afterLast) { + switch (*first) { + case _UT(':'): + case URI_SET_SUB_DELIMS(_UT): + case URI_SET_UNRESERVED(_UT): + first += 1; + break; - default: - URI_FUNC(StopSyntax)(state, first, memory); - return NULL; + default: + goto done_looping; + break; + } } -} -/* - * [ipFutStopGo]->[ipFutLoop] - * [ipFutStopGo]-> - */ -static const URI_CHAR * URI_FUNC(ParseIpFutStopGo)(URI_TYPE(ParserState) * state, - const URI_CHAR * first, - const URI_CHAR * afterLast, - UriMemoryManager * memory) { - if (first >= afterLast) { - return afterLast; +done_looping: + if (first == originalFirst) { + URI_FUNC(StopSyntax)(state, first, memory); + return NULL; } - switch (*first) { - case _UT(':'): - case URI_SET_SUB_DELIMS(_UT): - case URI_SET_UNRESERVED(_UT): - return URI_FUNC(ParseIpFutLoop)(state, first, afterLast, memory); - - default: - return first; - } + return first; } /* @@ -430,7 +408,7 @@ static const URI_CHAR * URI_FUNC(ParseIpFuture)(URI_TYPE(ParserState) * state, case URI_SET_HEXDIG(_UT): { const URI_CHAR * afterIpFutLoop; const URI_CHAR * const afterHexZero = - URI_FUNC(ParseHexZero)(state, first + 2, afterLast); + URI_FUNC(ParseHexZero)(first + 2, afterLast); if (afterHexZero == NULL) { return NULL; } @@ -832,6 +810,7 @@ static const URI_CHAR * URI_FUNC(ParseMustBeSegmentNzNc)(URI_TYPE(ParserState) * const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { if (!URI_FUNC(PushPathSegment)(state, state->uri->scheme.first, first, memory)) { /* SEGMENT BOTH */ @@ -849,14 +828,15 @@ static const URI_CHAR * URI_FUNC(ParseMustBeSegmentNzNc)(URI_TYPE(ParserState) * if (afterPctEncoded == NULL) { return NULL; } - return URI_FUNC(ParseMustBeSegmentNzNc)(state, afterPctEncoded, afterLast, - memory); + first = afterPctEncoded; + goto tail_call; } case _UT('@'): case URI_SET_SUB_DELIMS(_UT): case URI_SET_UNRESERVED(_UT): - return URI_FUNC(ParseMustBeSegmentNzNc)(state, first + 1, afterLast, memory); + first += 1; + goto tail_call; case _UT('/'): { const URI_CHAR * afterZeroMoreSlashSegs; @@ -953,6 +933,7 @@ static const URI_CHAR * URI_FUNC(ParseOwnHost2)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { if (!URI_FUNC(OnExitOwnHost2)(state, first, memory)) { URI_FUNC(StopMalloc)(state, memory); @@ -970,7 +951,8 @@ static const URI_CHAR * URI_FUNC(ParseOwnHost2)(URI_TYPE(ParserState) * state, if (afterPctSubUnres == NULL) { return NULL; } - return URI_FUNC(ParseOwnHost2)(state, afterPctSubUnres, afterLast, memory); + first = afterPctSubUnres; + goto tail_call; } default: @@ -1006,74 +988,69 @@ static URI_INLINE UriBool URI_FUNC(OnExitOwnHostUserInfo)(URI_TYPE(ParserState) return URI_TRUE; /* Success */ } -/* - * [ownHostUserInfo]->[ownHostUserInfoNz] - * [ownHostUserInfo]-> - */ -static URI_INLINE const URI_CHAR * -URI_FUNC(ParseOwnHostUserInfo)(URI_TYPE(ParserState) * state, const URI_CHAR * first, - const URI_CHAR * afterLast, UriMemoryManager * memory) { - if (first >= afterLast) { - if (!URI_FUNC(OnExitOwnHostUserInfo)(state, first, memory)) { - URI_FUNC(StopMalloc)(state, memory); - return NULL; - } - return afterLast; - } - - switch (*first) { - case URI_SET_PCHAR(_UT): - return URI_FUNC(ParseOwnHostUserInfoNz)(state, first, afterLast, memory); - - default: - if (!URI_FUNC(OnExitOwnHostUserInfo)(state, first, memory)) { - URI_FUNC(StopMalloc)(state, memory); - return NULL; - } - return first; - } -} - /* * [ownHostUserInfoNz]->[pctSubUnres][ownHostUserInfo] * [ownHostUserInfoNz]-><:>[ownPortUserInfo] * [ownHostUserInfoNz]-><@>[ownHost] + * + * [ownHostUserInfo]->[ownHostUserInfoNz] + * [ownHostUserInfo]-> */ static const URI_CHAR * URI_FUNC(ParseOwnHostUserInfoNz)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { - if (first >= afterLast) { - URI_FUNC(StopSyntax)(state, afterLast, memory); - return NULL; - } + const URI_CHAR * const originalFirst = first; + + while (first < afterLast) { + switch (*first) { + case _UT('%'): + case URI_SET_SUB_DELIMS(_UT): + case URI_SET_UNRESERVED(_UT): { + const URI_CHAR * const afterPctSubUnres = + URI_FUNC(ParsePctSubUnres)(state, first, afterLast, memory); + if (afterPctSubUnres == NULL) { + return NULL; + } + first = afterPctSubUnres; + break; + } - switch (*first) { - case _UT('%'): - case URI_SET_SUB_DELIMS(_UT): - case URI_SET_UNRESERVED(_UT): { - const URI_CHAR * const afterPctSubUnres = - URI_FUNC(ParsePctSubUnres)(state, first, afterLast, memory); - if (afterPctSubUnres == NULL) { - return NULL; + default: + goto done_looping; + break; } - return URI_FUNC(ParseOwnHostUserInfo)(state, afterPctSubUnres, afterLast, memory); } - case _UT(':'): - state->uri->hostText.afterLast = first; /* HOST END */ - state->uri->portText.first = first + 1; /* PORT BEGIN */ - return URI_FUNC(ParseOwnPortUserInfo)(state, first + 1, afterLast, memory); +done_looping: + if (first < afterLast) { + switch (*first) { + case _UT(':'): + state->uri->hostText.afterLast = first; /* HOST END */ + state->uri->portText.first = first + 1; /* PORT BEGIN */ + return URI_FUNC(ParseOwnPortUserInfo)(state, first + 1, afterLast, memory); - case _UT('@'): - state->uri->userInfo.afterLast = first; /* USERINFO END */ - state->uri->hostText.first = first + 1; /* HOST BEGIN */ - return URI_FUNC(ParseOwnHost)(state, first + 1, afterLast, memory); + case _UT('@'): + state->uri->userInfo.afterLast = first; /* USERINFO END */ + state->uri->hostText.first = first + 1; /* HOST BEGIN */ + return URI_FUNC(ParseOwnHost)(state, first + 1, afterLast, memory); - default: - URI_FUNC(StopSyntax)(state, first, memory); + default: + break; + } + } + + if (first == originalFirst) { + URI_FUNC(StopSyntax)(state, afterLast, memory); return NULL; } + + if (!URI_FUNC(OnExitOwnHostUserInfo)(state, first, memory)) { + URI_FUNC(StopMalloc)(state, memory); + return NULL; + } + + return first; } static URI_INLINE UriBool URI_FUNC(OnExitOwnPortUserInfo)(URI_TYPE(ParserState) * state, @@ -1117,6 +1094,7 @@ static const URI_CHAR * URI_FUNC(ParseOwnPortUserInfo)(URI_TYPE(ParserState) * s const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { if (!URI_FUNC(OnExitOwnPortUserInfo)(state, first, memory)) { URI_FUNC(StopMalloc)(state, memory); @@ -1140,7 +1118,8 @@ static const URI_CHAR * URI_FUNC(ParseOwnPortUserInfo)(URI_TYPE(ParserState) * s return URI_FUNC(ParseOwnUserInfo)(state, first + 1, afterLast, memory); case URI_SET_DIGIT(_UT): - return URI_FUNC(ParseOwnPortUserInfo)(state, first + 1, afterLast, memory); + first += 1; + goto tail_call; case _UT('%'): state->uri->portText.first = NULL; /* Not a port, reset */ @@ -1176,6 +1155,7 @@ static const URI_CHAR * URI_FUNC(ParseOwnUserInfo)(URI_TYPE(ParserState) * state const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { URI_FUNC(StopSyntax)(state, afterLast, memory); return NULL; @@ -1190,11 +1170,13 @@ static const URI_CHAR * URI_FUNC(ParseOwnUserInfo)(URI_TYPE(ParserState) * state if (afterPctSubUnres == NULL) { return NULL; } - return URI_FUNC(ParseOwnUserInfo)(state, afterPctSubUnres, afterLast, memory); + first = afterPctSubUnres; + goto tail_call; } case _UT(':'): - return URI_FUNC(ParseOwnUserInfo)(state, first + 1, afterLast, memory); + first += 1; + goto tail_call; case _UT('@'): /* SURE */ @@ -1254,6 +1236,7 @@ static const URI_CHAR * URI_FUNC(ParsePathAbsEmpty)(URI_TYPE(ParserState) * stat const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { return afterLast; } @@ -1270,7 +1253,8 @@ static const URI_CHAR * URI_FUNC(ParsePathAbsEmpty)(URI_TYPE(ParserState) * stat URI_FUNC(StopMalloc)(state, memory); return NULL; } - return URI_FUNC(ParsePathAbsEmpty)(state, afterSegment, afterLast, memory); + first = afterSegment; + goto tail_call; } default: @@ -1443,16 +1427,17 @@ static const URI_CHAR * URI_FUNC(ParsePctSubUnres)(URI_TYPE(ParserState) * state * [port]->[DIGIT][port] * [port]-> */ -static const URI_CHAR * URI_FUNC(ParsePort)(URI_TYPE(ParserState) * state, - const URI_CHAR * first, +static const URI_CHAR * URI_FUNC(ParsePort)(const URI_CHAR * first, const URI_CHAR * afterLast) { +tail_call: if (first >= afterLast) { return afterLast; } switch (*first) { case URI_SET_DIGIT(_UT): - return URI_FUNC(ParsePort)(state, first + 1, afterLast); + first += 1; + goto tail_call; default: return first; @@ -1469,6 +1454,7 @@ static const URI_CHAR * URI_FUNC(ParseQueryFrag)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { return afterLast; } @@ -1480,12 +1466,14 @@ static const URI_CHAR * URI_FUNC(ParseQueryFrag)(URI_TYPE(ParserState) * state, if (afterPchar == NULL) { return NULL; } - return URI_FUNC(ParseQueryFrag)(state, afterPchar, afterLast, memory); + first = afterPchar; + goto tail_call; } case _UT('/'): case _UT('?'): - return URI_FUNC(ParseQueryFrag)(state, first + 1, afterLast, memory); + first += 1; + goto tail_call; default: return first; @@ -1500,6 +1488,7 @@ static const URI_CHAR * URI_FUNC(ParseSegment)(URI_TYPE(ParserState) * state, const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { return afterLast; } @@ -1511,7 +1500,8 @@ static const URI_CHAR * URI_FUNC(ParseSegment)(URI_TYPE(ParserState) * state, if (afterPchar == NULL) { return NULL; } - return URI_FUNC(ParseSegment)(state, afterPchar, afterLast, memory); + first = afterPchar; + goto tail_call; } default: @@ -1572,6 +1562,7 @@ static const URI_CHAR * URI_FUNC(ParseSegmentNzNcOrScheme2)(URI_TYPE(ParserState const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { if (!URI_FUNC(OnExitSegmentNzNcOrScheme2)(state, first, memory)) { URI_FUNC(StopMalloc)(state, memory); @@ -1586,7 +1577,8 @@ static const URI_CHAR * URI_FUNC(ParseSegmentNzNcOrScheme2)(URI_TYPE(ParserState case _UT('-'): case URI_SET_ALPHA(_UT): case URI_SET_DIGIT(_UT): - return URI_FUNC(ParseSegmentNzNcOrScheme2)(state, first + 1, afterLast, memory); + first += 1; + goto tail_call; case _UT('%'): { const URI_CHAR * const afterPctEncoded = @@ -1796,6 +1788,7 @@ static const URI_CHAR * URI_FUNC(ParseZeroMoreSlashSegs)(URI_TYPE(ParserState) * const URI_CHAR * first, const URI_CHAR * afterLast, UriMemoryManager * memory) { +tail_call: if (first >= afterLast) { return afterLast; } @@ -1812,7 +1805,8 @@ static const URI_CHAR * URI_FUNC(ParseZeroMoreSlashSegs)(URI_TYPE(ParserState) * URI_FUNC(StopMalloc)(state, memory); return NULL; } - return URI_FUNC(ParseZeroMoreSlashSegs)(state, afterSegment, afterLast, memory); + first = afterSegment; + goto tail_call; } default: diff --git a/ext/zend_test/test.c b/ext/zend_test/test.c index 4e06b2106ce59..fd1a51c5776c3 100644 --- a/ext/zend_test/test.c +++ b/ext/zend_test/test.c @@ -37,10 +37,12 @@ #include "Zend/Optimizer/zend_optimizer.h" #include "Zend/zend_alloc.h" #include "test_arginfo.h" +#include "tmp_methods_arginfo.h" #include "zend_call_stack.h" #include "zend_exceptions.h" #include "zend_mm_custom_handlers.h" #include "ext/uri/php_uri.h" +#include "zend_observer.h" #if defined(HAVE_LIBXML) && !defined(PHP_WIN32) # include @@ -1030,16 +1032,38 @@ static ZEND_FUNCTION(zend_test_log_err_debug) php_log_err_with_severity(ZSTR_VAL(str), LOG_DEBUG); } +typedef struct _zend_test_object { + zend_internal_function *tmp_method; + zend_object std; +} zend_test_object; + static zend_object *zend_test_class_new(zend_class_entry *class_type) { - zend_object *obj = zend_objects_new(class_type); - object_properties_init(obj, class_type); - obj->handlers = &zend_test_class_handlers; - return obj; + zend_test_object *intern = zend_object_alloc(sizeof(zend_test_object), class_type); + zend_object_std_init(&intern->std, class_type); + object_properties_init(&intern->std, class_type); + return &intern->std; +} + +static void zend_test_class_free_obj(zend_object *object) +{ + zend_test_object *intern = (zend_test_object*)((char*)object - XtOffsetOf(zend_test_object, std)); + + if (intern->tmp_method) { + zend_internal_function *func = intern->tmp_method; + intern->tmp_method = NULL; + zend_string_release_ex(func->function_name, 0); + zend_free_internal_arg_info(func, false); + efree(func); + } + + zend_object_std_dtor(object); } static zend_function *zend_test_class_method_get(zend_object **object, zend_string *name, const zval *key) { + zend_test_object *intern = (zend_test_object*)((char*)(*object) - XtOffsetOf(zend_test_object, std)); + if (zend_string_equals_literal_ci(name, "test")) { zend_internal_function *fptr; @@ -1058,6 +1082,41 @@ static zend_function *zend_test_class_method_get(zend_object **object, zend_stri fptr->handler = ZEND_FN(zend_test_func); fptr->doc_comment = NULL; + return (zend_function*)fptr; + } else if (zend_string_equals_literal_ci(name, "testTmpMethodWithArgInfo")) { + if (intern->tmp_method) { + return (zend_function*)intern->tmp_method; + } + + const zend_function_entry *entry = &class_ZendTestTmpMethods_methods[0]; + zend_internal_function *fptr = emalloc(sizeof(zend_internal_function)); + memset(fptr, 0, sizeof(zend_internal_function)); + fptr->type = ZEND_INTERNAL_FUNCTION; + fptr->handler = entry->handler; + fptr->function_name = zend_string_init(entry->fname, strlen(entry->fname), false); + fptr->scope = intern->std.ce; + fptr->prototype = NULL; + fptr->T = ZEND_OBSERVER_ENABLED; + fptr->fn_flags = ZEND_ACC_PUBLIC | ZEND_ACC_NEVER_CACHE; + + zend_internal_function_info *info = (zend_internal_function_info*)entry->arg_info; + + uint32_t num_arg_info = 1 + entry->num_args; + zend_arg_info *arg_info = safe_emalloc(num_arg_info, sizeof(zend_arg_info), 0); + for (uint32_t i = 0; i < num_arg_info; i++) { + zend_convert_internal_arg_info(&arg_info[i], &entry->arg_info[i], i == 0, false); + } + + fptr->arg_info = arg_info + 1; + fptr->num_args = entry->num_args; + if (info->required_num_args == (uint32_t)-1) { + fptr->required_num_args = entry->num_args; + } else { + fptr->required_num_args = info->required_num_args; + } + + intern->tmp_method = fptr; + return (zend_function*)fptr; } return zend_std_get_method(object, name, key); @@ -1145,6 +1204,18 @@ static ZEND_METHOD(_ZendTestClass, variadicTest) { object_init_ex(return_value, zend_get_called_scope(execute_data)); } +ZEND_METHOD(ZendTestTmpMethods, testTmpMethodWithArgInfo) +{ + zend_object *obj; + zend_string *str; + + ZEND_PARSE_PARAMETERS_START(0, 2); + Z_PARAM_OPTIONAL; + Z_PARAM_OBJ_OR_NULL(obj); + Z_PARAM_STR(str); + ZEND_PARSE_PARAMETERS_END(); +} + static ZEND_METHOD(_ZendTestChildClass, returnsThrowable) { ZEND_PARSE_PARAMETERS_NONE(); @@ -1450,11 +1521,14 @@ PHP_MINIT_FUNCTION(zend_test) register_ZendTestClass_dnf_property(zend_test_class); zend_test_class->create_object = zend_test_class_new; zend_test_class->get_static_method = zend_test_class_static_method_get; + zend_test_class->default_object_handlers = &zend_test_class_handlers; zend_test_child_class = register_class__ZendTestChildClass(zend_test_class); memcpy(&zend_test_class_handlers, &std_object_handlers, sizeof(zend_object_handlers)); zend_test_class_handlers.get_method = zend_test_class_method_get; + zend_test_class_handlers.free_obj = zend_test_class_free_obj; + zend_test_class_handlers.offset = XtOffsetOf(zend_test_object, std); zend_test_gen_stub_flag_compatibility_test = register_class_ZendTestGenStubFlagCompatibilityTest(); diff --git a/ext/zend_test/tmp_methods.stub.php b/ext/zend_test/tmp_methods.stub.php new file mode 100644 index 0000000000000..af479c7d5422f --- /dev/null +++ b/ext/zend_test/tmp_methods.stub.php @@ -0,0 +1,11 @@ +common.num_args) { if (arginfo) { - if (func->type == ZEND_INTERNAL_FUNCTION) { - arg_name = (char *) ((zend_internal_arg_info *) &arginfo[i])->name; - } else { - arg_name = ZSTR_VAL(arginfo[i].name); - } + arg_name = ZSTR_VAL(arginfo[i].name); } smart_str_appends(s, arg_name ? arg_name : "?"); smart_str_appendc(s, '='); @@ -213,11 +209,7 @@ static void phpdbg_dump_prototype(zval *tmp) /* {{{ */ char *arg_name = NULL; if (arginfo) { - if (func->type == ZEND_INTERNAL_FUNCTION) { - arg_name = (char *)((zend_internal_arg_info *)&arginfo[j])->name; - } else { - arg_name = ZSTR_VAL(arginfo[j].name); - } + arg_name = ZSTR_VAL(arginfo[j].name); } if (!is_variadic) {