From d1139cc10e7d0e069910bfa9e97d88ae4b51c873 Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Sun, 27 Oct 2024 11:42:49 +0000 Subject: [PATCH 01/14] Allow .aot files in raw demo --- samples/linking/CMakeLists.txt | 7 +++++-- samples/linking/cmake/FindWAMRC.cmake | 27 +++++++++++++++++++++++++ samples/linking/cmake/FindWASISDK.cmake | 1 + samples/linking/raw/main.c | 6 ++++++ samples/linking/wasm/CMakeLists.txt | 19 ++++++++++++++++- 5 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 samples/linking/cmake/FindWAMRC.cmake diff --git a/samples/linking/CMakeLists.txt b/samples/linking/CMakeLists.txt index 1d00f778ca..51225b0f9f 100644 --- a/samples/linking/CMakeLists.txt +++ b/samples/linking/CMakeLists.txt @@ -14,7 +14,9 @@ string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) include(CheckPIESupported) # AOT and JIT byd default -set(WAMR_BUILD_AOT 0) +if(NOT WAMR_BUILD_AOT) + set(WAMR_BUILD_AOT 0) +endif() set(WAMR_BUILD_INTERP 1) set(WAMR_BUILD_JIT 0) # wasm32-wasi @@ -33,7 +35,7 @@ set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) add_library(vmlib SHARED ${WAMR_RUNTIME_LIB_SOURCE}) target_include_directories(vmlib INTERFACE ${WAMR_ROOT_DIR}/core/iwasm/include) -target_link_libraries (vmlib ${LLVM_AVAILABLE_LIBS} -lm -ldl) +target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} -lm -ldl) ################ samples ################ add_subdirectory(raw) @@ -45,6 +47,7 @@ ExternalProject_Add(wasm CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm -B build -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DWAMR_BUILD_AOT=${WAMR_BUILD_AOT} BUILD_COMMAND ${CMAKE_COMMAND} --build build INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} EXCLUDE_FROM_ALL NO diff --git a/samples/linking/cmake/FindWAMRC.cmake b/samples/linking/cmake/FindWAMRC.cmake new file mode 100644 index 0000000000..20f9416f74 --- /dev/null +++ b/samples/linking/cmake/FindWAMRC.cmake @@ -0,0 +1,27 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +find_path(WAMRC_HOME + wamr-compiler + PATHS ${CMAKE_CURRENT_SOURCE_DIR}/../../.. + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_file(WAMRC_BIN + wamrc + HINTS ${WAMRC_HOME}/wamr-compiler/build + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_package_handle_standard_args(WAMRC REQUIRED_VARS WAMRC_BIN) +mark_as_advanced(WAMRC_BIN) diff --git a/samples/linking/cmake/FindWASISDK.cmake b/samples/linking/cmake/FindWASISDK.cmake index 5cdfea41e4..0353b7de26 100644 --- a/samples/linking/cmake/FindWASISDK.cmake +++ b/samples/linking/cmake/FindWASISDK.cmake @@ -11,6 +11,7 @@ find_path(WASISDK_HOME REQUIRED ) + string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) diff --git a/samples/linking/raw/main.c b/samples/linking/raw/main.c index fe80f762e2..41a5410ba2 100644 --- a/samples/linking/raw/main.c +++ b/samples/linking/raw/main.c @@ -21,7 +21,13 @@ main(int argc, char *argv_main[]) /* wasm module file */ char *buffer; uint32 buf_size; +#if WASM_ENABLE_AOT != 0 + printf("Loading AOT file...\n"); + buffer = bh_read_file_to_buffer("import_memory.aot", &buf_size); +#else + printf("Loading WASM file...\n"); buffer = bh_read_file_to_buffer("import_memory.wasm", &buf_size); +#endif if (!buffer) { printf("Open wasm file failed.\n"); goto destroy_runtime; diff --git a/samples/linking/wasm/CMakeLists.txt b/samples/linking/wasm/CMakeLists.txt index 92543dfc47..7fe8e401cf 100644 --- a/samples/linking/wasm/CMakeLists.txt +++ b/samples/linking/wasm/CMakeLists.txt @@ -1,10 +1,15 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required (VERSION 3.14) +cmake_minimum_required(VERSION 3.14) project(linking_samples_wasm) +if(WAMR_BUILD_AOT EQUAL 1) + list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake) + find_package(WAMRC REQUIRED) +endif() + add_executable(import_memory import_memory.c) target_link_options(import_memory PRIVATE @@ -16,3 +21,15 @@ target_link_options(import_memory ) set_target_properties(import_memory PROPERTIES SUFFIX .wasm) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/import_memory.wasm DESTINATION .) + +if(WAMR_BUILD_AOT EQUAL 1) + add_custom_target ( + import_memory_aot + ALL + DEPENDS ${WAMRC_BIN} import_memory + COMMAND ${WAMRC_BIN} -o import_memory.aot import_memory.wasm + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/import_memory.aot DESTINATION .) +endif() + From 9189eaeeba9b68105d86e21aa6fd5e3e513a471a Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Mon, 28 Oct 2024 02:31:05 +0000 Subject: [PATCH 02/14] Allow to emit and load import_memory info --- core/iwasm/aot/aot_loader.c | 64 ++++++++- core/iwasm/compilation/aot.c | 126 ++++++++++++----- core/iwasm/compilation/aot_emit_aot_file.c | 151 ++++++++++++++++++--- 3 files changed, 279 insertions(+), 62 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 01e246aa32..cb5246ad18 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -963,6 +963,11 @@ load_custom_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, static void destroy_import_memories(AOTImportMemory *import_memories) { + if (!import_memories) + return; + + import_memories->module_name = NULL; + import_memories->memory_name = NULL; wasm_runtime_free(import_memories); } @@ -1035,6 +1040,53 @@ load_mem_init_data_list(const uint8 **p_buf, const uint8 *buf_end, return false; } +static bool +load_import_memory_info(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, bool is_load_from_file_buf, + char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + + read_uint32(buf, buf_end, module->import_memory_count); + + if (module->import_memory_count == 0) { + *p_buf = buf; + return true; + } + + uint64 size = sizeof(AOTImportMemory) * (uint64)module->import_memory_count; + AOTImportMemory *import_memories = + loader_malloc(size, error_buf, error_buf_size); + if (!import_memories) { + set_error_buf(error_buf, error_buf_size, + "allocate import_memories failed"); + return false; + } + + module->import_memories = import_memories; + for (uint32 i = 0; i < module->import_memory_count; i++) { + AOTImportMemory *import_memory = import_memories + i; + + read_uint32(buf, buf_end, import_memory->mem_type.flags); + if (!wasm_memory_check_flags(import_memory->mem_type.flags, error_buf, + error_buf_size, true)) { + set_error_buf(error_buf, error_buf_size, "invalid memory flags"); + return false; + } + + read_uint32(buf, buf_end, import_memory->mem_type.num_bytes_per_page); + read_uint32(buf, buf_end, import_memory->mem_type.init_page_count); + read_uint32(buf, buf_end, import_memory->mem_type.max_page_count); + read_string(buf, buf_end, import_memory->module_name); + read_string(buf, buf_end, import_memory->memory_name); + } + + *p_buf = buf; + return true; +fail: + return false; +} + static bool load_memory_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, char *error_buf, uint32 error_buf_size) @@ -1043,12 +1095,10 @@ load_memory_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, uint64 total_size; const uint8 *buf = *p_buf; - read_uint32(buf, buf_end, module->import_memory_count); - /* We don't support import_memory_count > 0 currently */ - if (module->import_memory_count > 0) - return false; - read_uint32(buf, buf_end, module->memory_count); + + /* because of reserved. memory_count always >= 1*/ + total_size = sizeof(AOTMemory) * (uint64)module->memory_count; if (!(module->memories = loader_malloc(total_size, error_buf, error_buf_size))) { @@ -2414,7 +2464,9 @@ load_init_data_section(const uint8 *buf, const uint8 *buf_end, { const uint8 *p = buf, *p_end = buf_end; - if (!load_memory_info(&p, p_end, module, error_buf, error_buf_size) + if (!load_import_memory_info(&p, p_end, module, is_load_from_file_buf, + error_buf, error_buf_size) + || !load_memory_info(&p, p_end, module, error_buf, error_buf_size) || !load_table_info(&p, p_end, module, error_buf, error_buf_size) || !load_type_info(&p, p_end, module, error_buf, error_buf_size) || !load_import_global_info(&p, p_end, module, is_load_from_file_buf, diff --git a/core/iwasm/compilation/aot.c b/core/iwasm/compilation/aot.c index 0a2cae1f0f..7640559dcd 100644 --- a/core/iwasm/compilation/aot.c +++ b/core/iwasm/compilation/aot.c @@ -31,6 +31,82 @@ aot_set_last_error(const char *error) aot_error[0] = '\0'; } +static AOTImportMemory * +aot_create_import_memories(const WASMModule *module, uint32 import_memory_count) +{ + uint64 size; + AOTImportMemory *import_memories; + unsigned i; + + bh_assert(import_memory_count == module->import_memory_count); + + size = sizeof(AOTImportMemory) * (uint64)import_memory_count; + if (size >= UINT32_MAX) { + aot_set_last_error("too big allocate size"); + return NULL; + } + + import_memories = wasm_runtime_malloc((uint32)size); + if (!module->import_memories) { + aot_set_last_error("allocate memory failed."); + return NULL; + } + + memset(import_memories, 0, (uint32)size); + + for (i = 0; i < module->import_memory_count; i++) { + AOTImportMemory *memory_aot = import_memories + i; + WASMImport *memory_wasm = module->import_memories + i; + + memory_aot->module_name = memory_wasm->u.memory.module_name; + memory_aot->memory_name = memory_wasm->u.memory.field_name; + memory_aot->mem_type.flags = memory_wasm->u.memory.mem_type.flags; + memory_aot->mem_type.init_page_count = + memory_wasm->u.memory.mem_type.init_page_count; + memory_aot->mem_type.max_page_count = + memory_wasm->u.memory.mem_type.max_page_count; + memory_aot->mem_type.num_bytes_per_page = + memory_wasm->u.memory.mem_type.num_bytes_per_page; + } + + return import_memories; +} + +static AOTMemory * +aot_create_memories(const WASMModule *module, uint32 memory_count) +{ + uint64 size; + AOTMemory *memories; + unsigned i; + + size = (uint64)memory_count * sizeof(AOTMemory); + if (size >= UINT32_MAX) { + aot_set_last_error("too big allocate size"); + return NULL; + } + + memories = wasm_runtime_malloc((uint32)size); + if (!memories) { + aot_set_last_error("create memories array failed.\n"); + return NULL; + } + + memset(memories, 0, size); + + /* in reserved case, assign nothing */ + for (i = 0; i < module->memory_count; i++) { + AOTMemory *memory_aot = memories + i; + WASMMemory *memory_wasm = module->memories + i; + + memory_aot->flags = memory_wasm->flags; + memory_aot->init_page_count = memory_wasm->init_page_count; + memory_aot->max_page_count = memory_wasm->max_page_count; + memory_aot->num_bytes_per_page = memory_wasm->num_bytes_per_page; + } + + return memories; +} + static void aot_destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count) { @@ -534,52 +610,32 @@ aot_create_comp_data(WASMModule *module, const char *target_arch, memset(comp_data, 0, sizeof(AOTCompData)); - comp_data->memory_count = - module->import_memory_count + module->memory_count; + /* Create import memories */ + comp_data->import_memory_count = module->import_memory_count; + if (comp_data->import_memory_count) { + comp_data->import_memories = + aot_create_import_memories(module, comp_data->import_memory_count); + if (!comp_data->import_memories) + goto fail; + } - /* TODO: create import memories */ + /* Create memories */ + comp_data->memory_count = module->memory_count; /* Allocate memory for memory array, reserve one AOTMemory space at least */ /* TODO: multi-memory */ - if (!comp_data->memory_count) + if (comp_data->memory_count == 0) comp_data->memory_count = 1; - size = (uint64)comp_data->memory_count * sizeof(AOTMemory); - if (size >= UINT32_MAX - || !(comp_data->memories = wasm_runtime_malloc((uint32)size))) { - aot_set_last_error("create memories array failed.\n"); + comp_data->memories = aot_create_memories(module, comp_data->memory_count); + if (!comp_data->memories) goto fail; - } - memset(comp_data->memories, 0, size); - if (!(module->import_memory_count + module->memory_count)) { + /* inherit reserved memory. keep it in memories[] */ + if (!module->memory_count) { comp_data->memories[0].num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; } - /* Set memory page count */ - for (i = 0; i < module->import_memory_count + module->memory_count; i++) { - if (i < module->import_memory_count) { - comp_data->memories[i].flags = - module->import_memories[i].u.memory.mem_type.flags; - comp_data->memories[i].num_bytes_per_page = - module->import_memories[i].u.memory.mem_type.num_bytes_per_page; - comp_data->memories[i].init_page_count = - module->import_memories[i].u.memory.mem_type.init_page_count; - comp_data->memories[i].max_page_count = - module->import_memories[i].u.memory.mem_type.max_page_count; - } - else { - j = i - module->import_memory_count; - comp_data->memories[i].flags = module->memories[j].flags; - comp_data->memories[i].num_bytes_per_page = - module->memories[j].num_bytes_per_page; - comp_data->memories[i].init_page_count = - module->memories[j].init_page_count; - comp_data->memories[i].max_page_count = - module->memories[j].max_page_count; - } - } - /* Create memory data segments */ comp_data->mem_init_data_count = module->data_seg_count; if (comp_data->mem_init_data_count > 0 diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 8fa2053083..cd48fed917 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -172,6 +172,9 @@ get_mem_init_data_list_size(AOTCompContext *comp_ctx, AOTMemInitData **mem_init_data = mem_init_data_list; uint32 size = 0, i; + /* mem_init_data_count */ + size = (uint32)sizeof(uint32); + for (i = 0; i < mem_init_data_count; i++, mem_init_data++) { size = align_uint(size, 4); size += get_mem_init_data_size(comp_ctx, *mem_init_data); @@ -180,10 +183,30 @@ get_mem_init_data_list_size(AOTCompContext *comp_ctx, } static uint32 -get_import_memory_size(AOTCompData *comp_data) +get_import_memory_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) { - /* currently we only emit import_memory_count = 0 */ - return sizeof(uint32); + AOTImportMemory *import_memory = NULL; + uint32 i = 0; + uint32 size = 0; + + /* import_memory_count */ + size = (uint32)sizeof(uint32); + + /* AOTImportMemory[] */ + for (i = 0, import_memory = comp_data->import_memories; + i < comp_data->import_memory_count; i++, import_memory++) { + size = align_uint(size, 2); + + /* u32 * 4 */ + size += (uint32)sizeof(uint32) * 4; + /* module_name */ + size += get_string_size(comp_ctx, import_memory->module_name); + size = align_uint(size, 2); + /* field_name */ + size += get_string_size(comp_ctx, import_memory->memory_name); + } + + return size; } static uint32 @@ -198,10 +221,36 @@ get_memory_size(AOTCompData *comp_data) static uint32 get_mem_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) { + /* + * ------------------------------------------------------- + * | u32 import_memory_count + * ------------------------------------------------------- + * | | u8 module_name + * | | u8 field_name + * | | u32 flags + * | AOTImportMemory[import_memory_count] | u32 num_bytes_per_page + * | | u32 init_page_count + * | | u32 max_page_count + * ------------------------------------------------------- + * | u32 memory_count + * ------------------------------------------------------- + * | | u32 flags + * | AOTMemoryType[memory_count] | u32 num_bytes_per_page + * | | u32 init_page_count + * | | u32 max_page_count + * ------------------------------------------------------- + * | u32 mem_init_data_count + * ------------------------------------------------------- + * | | u32 init expr type + * | AOTMemInitData[mem_init_data_count] | u32 init expr value + * | | u32 byte count + * | | u8* bytes + * ------------------------------------------------------- + */ /* import_memory_size + memory_size + init_data_count + init_data_list */ - return get_import_memory_size(comp_data) + get_memory_size(comp_data) - + (uint32)sizeof(uint32) + return get_import_memory_size(comp_ctx, comp_data) + + get_memory_size(comp_data) + get_mem_init_data_list_size(comp_ctx, comp_data->mem_init_data_list, comp_data->mem_init_data_count); @@ -1745,30 +1794,83 @@ aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, AOTCompContext *comp_ctx, InitializerExpression *expr); static bool -aot_emit_mem_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, - AOTCompContext *comp_ctx, AOTCompData *comp_data, - AOTObjectData *obj_data) +aot_emit_import_memory_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) { uint32 offset = *p_offset, i; - AOTMemInitData **init_datas = comp_data->mem_init_data_list; + AOTImportMemory *import_memory = NULL; *p_offset = offset = align_uint(offset, 4); - /* Emit import memory count, only emit 0 currently. - TODO: emit the actual import memory count and - the full import memory info. */ - EMIT_U32(0); + /* Emit import memory count */ + EMIT_U32(comp_data->import_memory_count); + + for (i = 0, import_memory = comp_data->import_memories; + i < comp_data->import_memory_count; i++, import_memory++) { + offset = align_uint(offset, 2); + + EMIT_U32(import_memory->mem_type.flags); + EMIT_U32(import_memory->mem_type.num_bytes_per_page); + EMIT_U32(import_memory->mem_type.init_page_count); + EMIT_U32(import_memory->mem_type.max_page_count); + + EMIT_STR(import_memory->module_name); + offset = align_uint(offset, 2); + + EMIT_STR(import_memory->memory_name); + } + + if (offset - *p_offset != get_import_memory_size(comp_ctx, comp_data)) { + aot_set_last_error("emit import memory info failed."); + return false; + } + + *p_offset = offset; + return true; +} + +static bool +aot_emit_memory_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + AOTMemory *memory = NULL; + + *p_offset = offset = align_uint(offset, 4); /* Emit memory count */ EMIT_U32(comp_data->memory_count); + /* Emit memory items */ - for (i = 0; i < comp_data->memory_count; i++) { - EMIT_U32(comp_data->memories[i].flags); - EMIT_U32(comp_data->memories[i].num_bytes_per_page); - EMIT_U32(comp_data->memories[i].init_page_count); - EMIT_U32(comp_data->memories[i].max_page_count); + for (i = 0, memory = comp_data->memories; i < comp_data->memory_count; + i++, memory++) { + EMIT_U32(memory->flags); + EMIT_U32(memory->num_bytes_per_page); + EMIT_U32(memory->init_page_count); + EMIT_U32(memory->max_page_count); + } + + if (offset - *p_offset != get_memory_size(comp_data)) { + aot_set_last_error("emit memory info failed."); + return false; } + *p_offset = offset; + return true; +} + +static bool +aot_emit_memory_init_data_list(uint8 *buf, uint8 *buf_end, uint32 *p_offset, + AOTCompContext *comp_ctx, AOTCompData *comp_data, + AOTObjectData *obj_data) +{ + uint32 offset = *p_offset, i; + AOTMemInitData **init_datas = comp_data->mem_init_data_list; + + *p_offset = offset = align_uint(offset, 4); + /* Emit mem init data count */ EMIT_U32(comp_data->mem_init_data_count); /* Emit mem init data items */ @@ -1793,8 +1895,10 @@ aot_emit_mem_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, EMIT_BUF(init_datas[i]->bytes, init_datas[i]->byte_count); } - if (offset - *p_offset != get_mem_info_size(comp_ctx, comp_data)) { - aot_set_last_error("emit memory info failed."); + if (offset - *p_offset + != get_mem_init_data_list_size(comp_ctx, comp_data->mem_init_data_list, + comp_data->mem_init_data_count)) { + aot_set_last_error("emit memory init data list failed."); return false; } @@ -2383,7 +2487,12 @@ aot_emit_init_data_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, EMIT_U32(AOT_SECTION_TYPE_INIT_DATA); EMIT_U32(section_size); - if (!aot_emit_mem_info(buf, buf_end, &offset, comp_ctx, comp_data, obj_data) + if (!aot_emit_import_memory_info(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data) + || !aot_emit_memory_info(buf, buf_end, &offset, comp_ctx, comp_data, + obj_data) + || !aot_emit_memory_init_data_list(buf, buf_end, &offset, comp_ctx, + comp_data, obj_data) || !aot_emit_table_info(buf, buf_end, &offset, comp_ctx, comp_data, obj_data) || !aot_emit_type_info(buf, buf_end, &offset, comp_ctx, comp_data, From 14de3193e4bc9853ac9a271116ab88e8aab61efa Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Mon, 28 Oct 2024 03:51:02 +0000 Subject: [PATCH 03/14] create/destroy memory api for aot --- core/iwasm/aot/aot_runtime.c | 395 +++++++++++++++----------- core/iwasm/aot/aot_runtime.h | 6 + core/iwasm/common/wasm_memory.c | 11 +- core/iwasm/interpreter/wasm_runtime.c | 4 + 4 files changed, 239 insertions(+), 177 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 7e6d6360c2..a6a06df017 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -824,169 +824,192 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, } static void -memories_deinstantiate(AOTModuleInstance *module_inst) +memory_deinstantiate(AOTMemoryInstance *memory) { - uint32 i; - AOTMemoryInstance *memory_inst; + if (!memory) { + return; + } - for (i = 0; i < module_inst->memory_count; i++) { - memory_inst = module_inst->memories[i]; - if (memory_inst) { #if WASM_ENABLE_SHARED_MEMORY != 0 - if (shared_memory_is_shared(memory_inst)) { - uint32 ref_count = shared_memory_dec_reference(memory_inst); - /* if the reference count is not zero, - don't free the memory */ - if (ref_count > 0) - continue; - } + if (shared_memory_is_shared(memory)) { + uint32 ref_count = shared_memory_dec_reference(memory); + /* if the reference count is not zero, + don't free the memory */ + if (ref_count > 0) { + return; + } + } #endif - if (memory_inst->heap_handle) { - mem_allocator_destroy(memory_inst->heap_handle); - wasm_runtime_free(memory_inst->heap_handle); - } - if (memory_inst->memory_data) { - wasm_deallocate_linear_memory(memory_inst); - } - } + if (memory->heap_handle) { + mem_allocator_destroy(memory->heap_handle); + wasm_runtime_free(memory->heap_handle); + memory->heap_handle = NULL; + } + + if (memory->memory_data) { + wasm_deallocate_linear_memory(memory); + memory->memory_data = NULL; + } +} + +static void +memories_deinstantiate(AOTModuleInstance *module_inst) +{ + /* import memories created by host or linker. released by them too */ + for (uint32 i = module_inst->module->import_memory_count; + i < module_inst->memory_count; i++) { + memory_deinstantiate(module_inst->memories[i]); } wasm_runtime_free(module_inst->memories); } static AOTMemoryInstance * -memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, - AOTModule *module, AOTMemoryInstance *memory_inst, - AOTMemory *memory, uint32 memory_idx, uint32 heap_size, - uint32 max_memory_pages, char *error_buf, +memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, + AOTMemoryInstance *memory, uint32 memory_idx, + uint32 num_bytes_per_page, uint32 init_page_count, + uint32 max_page_count, uint32 heap_size, uint32 flags, + uint8 *aux_heap_base_global_data, char *error_buf, uint32 error_buf_size) { - void *heap_handle; - uint32 num_bytes_per_page = memory->num_bytes_per_page; - uint32 init_page_count = memory->init_page_count; - uint32 max_page_count = wasm_runtime_get_max_mem( - max_memory_pages, memory->init_page_count, memory->max_page_count); - uint32 default_max_pages; - uint32 inc_page_count, global_idx; + uint32 inc_page_count, default_max_pages; uint32 bytes_of_last_page, bytes_to_page_end; uint64 aux_heap_base, heap_offset = (uint64)num_bytes_per_page * init_page_count; uint64 memory_data_size, max_memory_data_size; - uint8 *p = NULL, *global_addr; - bool is_memory64 = memory->flags & MEMORY64_FLAG; bool is_shared_memory = false; #if WASM_ENABLE_SHARED_MEMORY != 0 - is_shared_memory = memory->flags & SHARED_MEMORY_FLAG ? true : false; + is_shared_memory = flags & SHARED_MEMORY_FLAG ? true : false; + /* Shared memory */ if (is_shared_memory && parent != NULL) { - AOTMemoryInstance *shared_memory_instance; - bh_assert(memory_idx == 0); bh_assert(parent->memory_count > memory_idx); - shared_memory_instance = parent->memories[memory_idx]; - shared_memory_inc_reference(shared_memory_instance); - return shared_memory_instance; + memory = parent->memories[memory_idx]; + shared_memory_inc_reference(memory); + return memory; } -#endif +#else + (void)parent; + (void)memory_idx; + (void)flags; +#endif /* #if WASM_ENABLE_SHARED_MEMORY != 0 */ #if WASM_ENABLE_MEMORY64 != 0 - if (is_memory64) { - default_max_pages = DEFAULT_MEM64_MAX_PAGES; + if (flags & MEMORY64_FLAG) { + memory->is_memory64 = 1; } - else #endif - { - default_max_pages = DEFAULT_MAX_PAGES; - } - if (heap_size > 0 && module->malloc_func_index != (uint32)-1 - && module->free_func_index != (uint32)-1) { - /* Disable app heap, use malloc/free function exported - by wasm app to allocate/free memory instead */ - heap_size = 0; - } + default_max_pages = + memory->is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; - /* If initial memory is the largest size allowed, disallowing insert host - * managed heap */ - if (heap_size > 0 && heap_offset == MAX_LINEAR_MEMORY_SIZE) { - set_error_buf(error_buf, error_buf_size, - "failed to insert app heap into linear memory, " - "try using `--heap-size=0` option"); - return NULL; - } + /* The app heap should be in the default memory */ + if (memory_idx == 0) { + if (heap_size > 0 && module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1) { + /* Disable app heap, use malloc/free function exported + by wasm app to allocate/free memory instead */ + heap_size = 0; + } - if (init_page_count == max_page_count && init_page_count == 1) { - /* If only one page and at most one page, we just append - the app heap to the end of linear memory, enlarge the - num_bytes_per_page, and don't change the page count */ - heap_offset = num_bytes_per_page; - num_bytes_per_page += heap_size; - if (num_bytes_per_page < heap_size) { + /* If initial memory is the largest size allowed, disallowing insert + * host managed heap */ + if (heap_size > 0 && heap_offset == MAX_LINEAR_MEMORY_SIZE) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " "try using `--heap-size=0` option"); return NULL; } - } - else if (heap_size > 0) { - if (init_page_count == max_page_count && init_page_count == 0) { - /* If the memory data size is always 0, we resize it to - one page for app heap */ - num_bytes_per_page = heap_size; - heap_offset = 0; - inc_page_count = 1; + + if (init_page_count == max_page_count && init_page_count == 1) { + /* If only one page and at most one page, we just append + the app heap to the end of linear memory, enlarge the + num_bytes_per_page, and don't change the page count */ + heap_offset = num_bytes_per_page; + num_bytes_per_page += heap_size; + if (num_bytes_per_page < heap_size) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } } - else if (module->aux_heap_base_global_index != (uint32)-1 - && module->aux_heap_base - < (uint64)num_bytes_per_page * init_page_count) { - /* Insert app heap before __heap_base */ - aux_heap_base = module->aux_heap_base; - bytes_of_last_page = aux_heap_base % num_bytes_per_page; - if (bytes_of_last_page == 0) - bytes_of_last_page = num_bytes_per_page; - bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; - inc_page_count = - (heap_size - bytes_to_page_end + num_bytes_per_page - 1) - / num_bytes_per_page; - heap_offset = aux_heap_base; - aux_heap_base += heap_size; - - bytes_of_last_page = aux_heap_base % num_bytes_per_page; - if (bytes_of_last_page == 0) - bytes_of_last_page = num_bytes_per_page; - bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; - if (bytes_to_page_end < 1 * BH_KB) { - aux_heap_base += 1 * BH_KB; - inc_page_count++; + else if (heap_size > 0) { + if (init_page_count == max_page_count && init_page_count == 0) { + /* If the memory data size is always 0, we resize it to + one page for app heap */ + num_bytes_per_page = heap_size; + heap_offset = 0; + inc_page_count = 1; } + else if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base + < (uint64)num_bytes_per_page * init_page_count) { + /* Insert app heap before __heap_base */ + aux_heap_base = module->aux_heap_base; + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + inc_page_count = + (heap_size - bytes_to_page_end + num_bytes_per_page - 1) + / num_bytes_per_page; + heap_offset = aux_heap_base; + aux_heap_base += heap_size; + + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + if (bytes_to_page_end < 1 * BH_KB) { + aux_heap_base += 1 * BH_KB; + inc_page_count++; + } - /* Adjust __heap_base global value */ - global_idx = module->aux_heap_base_global_index - - module->import_global_count; - global_addr = module_inst->global_data - + module->globals[global_idx].data_offset; - *(uint32 *)global_addr = (uint32)aux_heap_base; - LOG_VERBOSE("Reset __heap_base global to %" PRIu64, aux_heap_base); - } - else { - /* Insert app heap before new page */ - inc_page_count = - (heap_size + num_bytes_per_page - 1) / num_bytes_per_page; - heap_offset = (uint64)num_bytes_per_page * init_page_count; - heap_size = (uint64)num_bytes_per_page * inc_page_count; - if (heap_size > 0) - heap_size -= 1 * BH_KB; - } - init_page_count += inc_page_count; - max_page_count += inc_page_count; - if (init_page_count > default_max_pages) { - set_error_buf(error_buf, error_buf_size, - "failed to insert app heap into linear memory, " - "try using `--heap-size=0` option"); - return NULL; + /* Adjust __heap_base global value */ + if (aux_heap_base_global_data == NULL) { + set_error_buf( + error_buf, error_buf_size, + "auxiliary heap base global data should not be NULL"); + return NULL; + } + +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + /* For memory64, the global value should be i64 */ + *(uint64 *)aux_heap_base_global_data = aux_heap_base; + } + else +#endif + { + /* For memory32, the global value should be i32 */ + *(uint32 *)aux_heap_base_global_data = + (uint32)aux_heap_base; + } + LOG_VERBOSE("Reset __heap_base global to %" PRIu64, + aux_heap_base); + } + else { + /* Insert app heap before new page */ + inc_page_count = + (heap_size + num_bytes_per_page - 1) / num_bytes_per_page; + heap_offset = (uint64)num_bytes_per_page * init_page_count; + heap_size = (uint64)num_bytes_per_page * inc_page_count; + if (heap_size > 0) + heap_size -= 1 * BH_KB; + } + init_page_count += inc_page_count; + max_page_count += inc_page_count; + if (init_page_count > default_max_pages) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } + if (max_page_count > default_max_pages) + max_page_count = default_max_pages; } - if (max_page_count > default_max_pages) - max_page_count = default_max_pages; } LOG_VERBOSE("Memory instantiate:"); @@ -998,49 +1021,46 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, heap_size); max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; - bh_assert(max_memory_data_size <= GET_MAX_LINEAR_MEMORY_SIZE(is_memory64)); + bh_assert(max_memory_data_size + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); (void)max_memory_data_size; - /* TODO: memory64 uses is_memory64 flag */ - if (wasm_allocate_linear_memory(&p, is_shared_memory, is_memory64, - num_bytes_per_page, init_page_count, - max_page_count, &memory_data_size) + bh_assert(memory != NULL); + + if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory, + memory->is_memory64, num_bytes_per_page, + init_page_count, max_page_count, + &memory_data_size) != BHT_OK) { set_error_buf(error_buf, error_buf_size, "allocate linear memory failed"); return NULL; } - memory_inst->module_type = Wasm_Module_AoT; - memory_inst->num_bytes_per_page = num_bytes_per_page; - memory_inst->cur_page_count = init_page_count; - memory_inst->max_page_count = max_page_count; - memory_inst->memory_data_size = memory_data_size; -#if WASM_ENABLE_MEMORY64 != 0 - if (is_memory64) { - memory_inst->is_memory64 = 1; - } -#endif + memory->module_type = Wasm_Module_AoT; + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = init_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = memory_data_size; /* Init memory info */ - memory_inst->memory_data = p; - memory_inst->memory_data_end = p + memory_data_size; + if (memory_idx == 0) { + memory->heap_data = memory->memory_data + heap_offset; + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data_end = memory->memory_data + memory_data_size; + } /* Initialize heap info */ - memory_inst->heap_data = p + heap_offset; - memory_inst->heap_data_end = p + heap_offset + heap_size; - if (heap_size > 0) { + if (memory_idx == 0 && heap_size > 0) { uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); - if (!(heap_handle = runtime_malloc((uint64)heap_struct_size, error_buf, - error_buf_size))) { + if (!(memory->heap_handle = runtime_malloc( + (uint64)heap_struct_size, error_buf, error_buf_size))) { goto fail1; } - memory_inst->heap_handle = heap_handle; - if (!mem_allocator_create_with_struct_and_pool( - heap_handle, heap_struct_size, memory_inst->heap_data, + memory->heap_handle, heap_struct_size, memory->heap_data, heap_size)) { set_error_buf(error_buf, error_buf_size, "init app heap failed"); goto fail2; @@ -1048,23 +1068,25 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, } if (memory_data_size > 0) { - wasm_runtime_set_mem_bound_check_bytes(memory_inst, memory_data_size); + wasm_runtime_set_mem_bound_check_bytes(memory, memory_data_size); } #if WASM_ENABLE_SHARED_MEMORY != 0 if (is_shared_memory) { - memory_inst->is_shared_memory = 1; - memory_inst->ref_count = 1; + memory->is_shared_memory = 1; + memory->ref_count = 1; } #endif - return memory_inst; + LOG_VERBOSE("Memory instantiate success."); + return memory; fail2: - if (heap_size > 0) - wasm_runtime_free(memory_inst->heap_handle); + if (memory_idx == 0 && heap_size > 0) + wasm_runtime_free(memory->heap_handle); fail1: - wasm_deallocate_linear_memory(memory_inst); + if (memory->memory_data) + wasm_deallocate_linear_memory(memory); return NULL; } @@ -1104,17 +1126,18 @@ aot_get_memory_with_index(AOTModuleInstance *module_inst, uint32 index) } static bool -memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, - AOTModule *module, uint32 heap_size, - uint32 max_memory_pages, char *error_buf, - uint32 error_buf_size) +memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, + AOTModuleInstance *parent, uint32 heap_size, + uint32 max_memory_pages, uint8 *aux_heap_base_global_data, + char *error_buf, uint32 error_buf_size) { uint32 global_index, global_data_offset, length; - uint32 i, memory_count = module->memory_count; - AOTMemoryInstance *memories, *memory_inst; + uint32 i, memory_count = module->import_memory_count + module->memory_count; + AOTMemoryInstance *memory; AOTMemInitData *data_seg; uint64 total_size; mem_offset_t base_offset; + uint32 mem_index = 0; module_inst->memory_count = memory_count; total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count; @@ -1123,20 +1146,27 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, return false; } - memories = module_inst->global_table_data.memory_instances; - for (i = 0; i < memory_count; i++, memories++) { - memory_inst = memory_instantiate( - module_inst, parent, module, memories, &module->memories[i], i, - heap_size, max_memory_pages, error_buf, error_buf_size); - if (!memory_inst) { + for (i = 0, mem_index = module->import_memory_count, + memory = module_inst->global_table_data.memory_instances + + module->import_memory_count; + i < memory_count; i++, memory++, mem_index++) { + AOTMemory *memory_type = module->memories + i; + uint32 max_page_count = wasm_runtime_get_max_mem( + max_memory_pages, memory_type->init_page_count, + memory_type->max_page_count); + + module_inst->memories[mem_index] = memory_instantiate( + module, parent, memory, mem_index, memory_type->num_bytes_per_page, + memory_type->init_page_count, max_page_count, heap_size, + memory_type->flags, aux_heap_base_global_data, error_buf, + error_buf_size); + if (!module_inst->memories[i]) { return false; } - - module_inst->memories[i] = memory_inst; } /* Get default memory instance */ - memory_inst = aot_get_default_memory(module_inst); + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); if (!memory_inst) { /* Ignore setting memory init data if no memory inst is created */ return true; @@ -1246,6 +1276,21 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, return true; } +AOTMemoryInstance * +aot_create_memory(const AOTModule *module, const AOTMemoryType *type, + uint32 index) +{ + bh_assert(false && "Unsupported operation"); + return NULL; +} + +void +aot_destroy_memory(AOTMemoryInstance *memory) +{ + memory_deinstantiate(memory); + wasm_runtime_free(memory); +} + static bool init_func_ptrs(AOTModuleInstance *module_inst, AOTModule *module, char *error_buf, uint32 error_buf_size) @@ -2168,10 +2213,14 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) if (module_inst->export_functions) wasm_runtime_free(module_inst->export_functions); -#if WASM_ENABLE_MULTI_MEMORY != 0 + if (module_inst->export_globals) + wasm_runtime_free(module_inst->export_globals); + if (module_inst->export_memories) wasm_runtime_free(module_inst->export_memories); -#endif + + if (module_inst->export_tables) + wasm_runtime_free(module_inst->export_tables); if (extra->functions) { uint32 func_idx; diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index bf5e4366c7..c86db32c3d 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -545,6 +545,12 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, void aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst); +AOTMemoryInstance * +aot_create_memory(const AOTModule *module, const AOTMemoryType *type, + uint32 index); + +void +aot_destroy_memory(AOTMemoryInstance *memory); /** * Lookup an exported function in the AOT module instance. * diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 18c0383a21..52797ea29c 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -1547,8 +1547,9 @@ wasm_runtime_create_memory(WASMModuleCommon *module, WASMMemoryType *type) #endif #if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) - bh_assert(false && "Unsupported operation"); + if (module->module_type == Wasm_Module_AoT) { + return aot_create_memory((AOTModule *)module, type, index); + } #endif LOG_ERROR("create memory failed, invalid module type"); @@ -1570,8 +1571,10 @@ wasm_runtime_destroy_memory(WASMModuleCommon *const module, #endif #if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) - bh_assert(false && "Unsupported operation"); + if (module->module_type == Wasm_Module_AoT) { + aot_destroy_memory(memory); + return; + } #endif LOG_ERROR("destroy memory failed, invalid module type"); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index b13c5760b4..7599f9168f 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -445,6 +445,10 @@ memory_instantiate(const WASMModule *module, WASMModuleInstance *parent, LOG_VERBOSE("Memory instantiate:"); LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", num_bytes_per_page, init_page_count, max_page_count); + LOG_VERBOSE(" data offset: %" PRIu64 ", stack size: %d", + module->aux_data_end, module->aux_stack_size); + LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %u\n", heap_offset, + heap_size); #ifndef NDEBUG { From 505dbefac0a563aed426eddfd3b3297a43fb3d5a Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Wed, 30 Oct 2024 05:04:45 +0000 Subject: [PATCH 04/14] Refactor memory instantiation logic - ~99% same with memory_instantiation() in wasm_runtime.c - new apis to create and destroy memory --- core/iwasm/aot/aot_runtime.c | 158 +++++++++++++++++++++++++---------- 1 file changed, 116 insertions(+), 42 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index a6a06df017..e5d45a5e6a 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -864,19 +864,24 @@ memories_deinstantiate(AOTModuleInstance *module_inst) wasm_runtime_free(module_inst->memories); } +/* + * TODO: can be optimized by merging with another memory_instantiate() in + * wasm_runtime.c + */ static AOTMemoryInstance * memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, - AOTMemoryInstance *memory, uint32 memory_idx, + AOTMemoryInstance *memory, AOTMemoryInstance *parent_memory, uint32 num_bytes_per_page, uint32 init_page_count, uint32 max_page_count, uint32 heap_size, uint32 flags, uint8 *aux_heap_base_global_data, char *error_buf, uint32 error_buf_size) { - uint32 inc_page_count, default_max_pages; + uint32 default_max_pages; uint32 bytes_of_last_page, bytes_to_page_end; - uint64 aux_heap_base, - heap_offset = (uint64)num_bytes_per_page * init_page_count; - uint64 memory_data_size, max_memory_data_size; + uint64 aux_heap_base; + uint64 max_memory_data_size; + + bh_assert(memory != NULL); bool is_shared_memory = false; #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -884,14 +889,14 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, /* Shared memory */ if (is_shared_memory && parent != NULL) { - bh_assert(parent->memory_count > memory_idx); - memory = parent->memories[memory_idx]; + bh_assert(parent_memory != NULL); + memory = parent_memory; shared_memory_inc_reference(memory); return memory; } #else (void)parent; - (void)memory_idx; + (void)parent_memory; (void)flags; #endif /* #if WASM_ENABLE_SHARED_MEMORY != 0 */ @@ -904,8 +909,10 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, default_max_pages = memory->is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; - /* The app heap should be in the default memory */ - if (memory_idx == 0) { + /* TODO: use small functions to reduce the complexity */ + /* adjust heap_size, heap_offset and num_bytes_per_page */ + uint64 heap_offset = (uint64)num_bytes_per_page * init_page_count; + { if (heap_size > 0 && module->malloc_func_index != (uint32)-1 && module->free_func_index != (uint32)-1) { /* Disable app heap, use malloc/free function exported @@ -915,7 +922,8 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, /* If initial memory is the largest size allowed, disallowing insert * host managed heap */ - if (heap_size > 0 && heap_offset == MAX_LINEAR_MEMORY_SIZE) { + if (heap_size > 0 + && heap_offset == GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " "try using `--heap-size=0` option"); @@ -936,6 +944,8 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, } } else if (heap_size > 0) { + uint32 inc_page_count = 0; + if (init_page_count == max_page_count && init_page_count == 0) { /* If the memory data size is always 0, we resize it to one page for app heap */ @@ -1017,16 +1027,17 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, num_bytes_per_page, init_page_count, max_page_count); LOG_VERBOSE(" data offset: %" PRIu64 ", stack size: %d", module->aux_data_end, module->aux_stack_size); - LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %d\n", heap_offset, - heap_size); - - max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; - bh_assert(max_memory_data_size - <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); - (void)max_memory_data_size; - bh_assert(memory != NULL); +#ifndef NDEBUG + { + max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; + bh_assert(max_memory_data_size + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); + (void)max_memory_data_size; + } +#endif + uint64 memory_data_size = 0; if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory, memory->is_memory64, num_bytes_per_page, init_page_count, max_page_count, @@ -1037,25 +1048,28 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, return NULL; } + /* Init memory info */ memory->module_type = Wasm_Module_AoT; memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = init_page_count; memory->max_page_count = max_page_count; memory->memory_data_size = memory_data_size; - - /* Init memory info */ - if (memory_idx == 0) { - memory->heap_data = memory->memory_data + heap_offset; - memory->heap_data_end = memory->heap_data + heap_size; - memory->memory_data_end = memory->memory_data + memory_data_size; - } + memory->heap_data = memory->memory_data + heap_offset; + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data_end = memory->memory_data + memory->memory_data_size; /* Initialize heap info */ - if (memory_idx == 0 && heap_size > 0) { + if (heap_size > 0) { + LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %d\n", heap_offset, + heap_size); + uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); - if (!(memory->heap_handle = runtime_malloc( - (uint64)heap_struct_size, error_buf, error_buf_size))) { + memory->heap_handle = + runtime_malloc((uint64)heap_struct_size, error_buf, error_buf_size); + if (!memory->heap_handle) { + set_error_buf(error_buf, error_buf_size, + "allocate app heap failed"); goto fail1; } @@ -1082,8 +1096,10 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, return memory; fail2: - if (memory_idx == 0 && heap_size > 0) + if (heap_size > 0) { wasm_runtime_free(memory->heap_handle); + memory->heap_handle = NULL; + } fail1: if (memory->memory_data) wasm_deallocate_linear_memory(memory); @@ -1108,13 +1124,26 @@ aot_lookup_memory(AOTModuleInstance *module_inst, char const *name) #endif } +/* + * [0] could be internal or external + * + * Because the reserved memory(aot_create_comp_data() in + * core/iwasm/compilation/aot.c), + */ AOTMemoryInstance * aot_get_default_memory(AOTModuleInstance *module_inst) { - if (module_inst->memories) - return module_inst->memories[0]; - else + if (!module_inst->memories) { + LOG_WARNING("memories[] is not instantiated"); return NULL; + } + + AOTMemoryInstance *default_memory = module_inst->memories[0]; + bh_assert(default_memory && "default memory(AOT) can't be NULL"); + if (!default_memory) { + LOG_ERROR("default memory(AOT) is NULL"); + } + return default_memory; } AOTMemoryInstance * @@ -1156,10 +1185,12 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, memory_type->max_page_count); module_inst->memories[mem_index] = memory_instantiate( - module, parent, memory, mem_index, memory_type->num_bytes_per_page, - memory_type->init_page_count, max_page_count, heap_size, - memory_type->flags, aux_heap_base_global_data, error_buf, - error_buf_size); + module, parent, memory, parent ? parent->memories[mem_index] : NULL, + memory_type->num_bytes_per_page, memory_type->init_page_count, + max_page_count, + /* only inst->memories[0] will have a app heap */ + mem_index == 0 ? heap_size : 0, memory_type->flags, + aux_heap_base_global_data, error_buf, error_buf_size); if (!module_inst->memories[i]) { return false; } @@ -1168,8 +1199,8 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, /* Get default memory instance */ AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); if (!memory_inst) { - /* Ignore setting memory init data if no memory inst is created */ - return true; + set_error_buf(error_buf, error_buf_size, "get default memory failed"); + return false; } for (i = 0; i < module->mem_init_data_count; i++) { @@ -1280,13 +1311,46 @@ AOTMemoryInstance * aot_create_memory(const AOTModule *module, const AOTMemoryType *type, uint32 index) { - bh_assert(false && "Unsupported operation"); - return NULL; + if (!module || !type) + return NULL; + + AOTMemoryInstance *memory = NULL; + char error_buf[64] = { 0 }; + + memory = + runtime_malloc(sizeof(AOTMemoryInstance), error_buf, sizeof(error_buf)); + if (!memory) { + LOG_ERROR("Failed to allocate AOTMemoryInstance. %s", error_buf); + return NULL; + } + + /* + * use provided max_page_count of type instead of adjusting with + * wasm_runtime_get_max_mem() + */ + if (!memory_instantiate(module, + NULL, // parent + memory, + NULL, // parent_memory + type->num_bytes_per_page, type->init_page_count, + type->max_page_count, + 0, // no app heap for host + type->flags, + NULL, // aux_heap_base_global_data + error_buf, sizeof(error_buf))) { + wasm_runtime_free(memory); + return NULL; + } + + return memory; } void aot_destroy_memory(AOTMemoryInstance *memory) { + if (!memory) + return; + memory_deinstantiate(memory); wasm_runtime_free(memory); } @@ -1754,7 +1818,9 @@ check_linked_symbol(AOTModule *module, char *error_buf, uint32 error_buf_size) AOTModuleInstance * aot_instantiate(AOTModule *module, AOTModuleInstance *parent, WASMExecEnv *exec_env_main, uint32 stack_size, uint32 heap_size, - uint32 max_memory_pages, char *error_buf, uint32 error_buf_size) + uint32 max_memory_pages, uint32 import_count, + const WASMExternInstance *imports, char *error_buf, + uint32 error_buf_size) { AOTModuleInstance *module_inst; #if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_REF_TYPES != 0 @@ -1773,6 +1839,14 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, bool ret = false; #endif +#if WASM_ENABLE_MULTI_MODULE == 0 + if (module->import_count > 0 && !imports) { + set_error_buf(error_buf, error_buf_size, + "argument imports is NULL while module has imports"); + return NULL; + } +#endif + /* Check heap size */ heap_size = align_uint(heap_size, 8); if (heap_size > APP_HEAP_SIZE_MAX) From e4fd6eec7e854edbeb33f6e71960883b9621f6dd Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Thu, 31 Oct 2024 07:07:56 +0000 Subject: [PATCH 05/14] Refactor AOT memory handling and improve memory initialization logic --- core/iwasm/aot/aot_loader.c | 22 +++-- core/iwasm/aot/aot_runtime.c | 109 ++++++++++++++++----- core/iwasm/aot/aot_runtime.h | 6 +- core/iwasm/common/wasm_memory.c | 2 +- core/iwasm/common/wasm_runtime_common.c | 3 +- core/iwasm/compilation/aot.c | 18 ++-- core/iwasm/compilation/aot_compiler.h | 22 ++++- core/iwasm/compilation/aot_emit_aot_file.c | 91 ++++++++--------- core/iwasm/compilation/aot_emit_control.c | 3 +- core/iwasm/compilation/aot_emit_memory.c | 20 ++-- core/iwasm/compilation/aot_llvm.c | 12 +-- core/iwasm/interpreter/wasm_runtime.c | 15 +-- 12 files changed, 200 insertions(+), 123 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index cb5246ad18..e01af5f178 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -996,6 +996,13 @@ load_mem_init_data_list(const uint8 **p_buf, const uint8 *buf_end, uint64 size; uint32 i; + read_uint32(buf, buf_end, module->mem_init_data_count); + + if (module->mem_init_data_count == 0) { + *p_buf = buf; + return true; + } + /* Allocate memory */ size = sizeof(AOTMemInitData *) * (uint64)module->mem_init_data_count; if (!(module->mem_init_data_list = data_list = @@ -1097,7 +1104,10 @@ load_memory_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, read_uint32(buf, buf_end, module->memory_count); - /* because of reserved. memory_count always >= 1*/ + if (module->memory_count == 0) { + *p_buf = buf; + return true; + } total_size = sizeof(AOTMemory) * (uint64)module->memory_count; if (!(module->memories = @@ -1118,14 +1128,6 @@ load_memory_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, read_uint32(buf, buf_end, module->memories[i].max_page_count); } - read_uint32(buf, buf_end, module->mem_init_data_count); - - /* load memory init data list */ - if (module->mem_init_data_count > 0 - && !load_mem_init_data_list(&buf, buf_end, module, error_buf, - error_buf_size)) - return false; - *p_buf = buf; return true; fail: @@ -2467,6 +2469,8 @@ load_init_data_section(const uint8 *buf, const uint8 *buf_end, if (!load_import_memory_info(&p, p_end, module, is_load_from_file_buf, error_buf, error_buf_size) || !load_memory_info(&p, p_end, module, error_buf, error_buf_size) + || !load_mem_init_data_list(&p, p_end, module, error_buf, + error_buf_size) || !load_table_info(&p, p_end, module, error_buf, error_buf_size) || !load_type_info(&p, p_end, module, error_buf, error_buf_size) || !load_import_global_info(&p, p_end, module, is_load_from_file_buf, diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index e5d45a5e6a..809a7afb3d 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -178,6 +178,27 @@ get_prev_frame(WASMExecEnv *exec_env, void *cur_frame) } #endif +static const WASMExternInstance * +get_extern_instance_from_imports(const WASMExternInstance *imports, + uint32 import_count, + wasm_import_export_kind_t kind, uint32 index) +{ + for (uint32 i = 0, target_kind_index = 0; i < import_count; i++) { + if (imports[i].kind != kind) { + continue; + } + + if (target_kind_index != index) { + target_kind_index++; + continue; + } + + return imports + i; + } + + return NULL; +} + static bool check_global_init_expr(const AOTModule *module, uint32 global_index, char *error_buf, uint32 error_buf_size) @@ -879,7 +900,6 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, uint32 default_max_pages; uint32 bytes_of_last_page, bytes_to_page_end; uint64 aux_heap_base; - uint64 max_memory_data_size; bh_assert(memory != NULL); @@ -1030,7 +1050,8 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, #ifndef NDEBUG { - max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; + uint64 max_memory_data_size = + (uint64)num_bytes_per_page * max_page_count; bh_assert(max_memory_data_size <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); (void)max_memory_data_size; @@ -1118,16 +1139,15 @@ aot_lookup_memory(AOTModuleInstance *module_inst, char const *name) return NULL; #else (void)module_inst->export_memories; - if (!module_inst->memories) - return NULL; - return module_inst->memories[0]; + return aot_get_default_memory(module_inst); #endif } /* * [0] could be internal or external * - * Because the reserved memory(aot_create_comp_data() in + * it won't be NULL because of the reserved + * memory(aot_create_comp_data() in * core/iwasm/compilation/aot.c), */ AOTMemoryInstance * @@ -1158,14 +1178,11 @@ static bool memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, AOTModuleInstance *parent, uint32 heap_size, uint32 max_memory_pages, uint8 *aux_heap_base_global_data, + const WASMExternInstance *imports, uint32 import_count, char *error_buf, uint32 error_buf_size) { - uint32 global_index, global_data_offset, length; - uint32 i, memory_count = module->import_memory_count + module->memory_count; - AOTMemoryInstance *memory; - AOTMemInitData *data_seg; + uint32 memory_count = module->import_memory_count + module->memory_count; uint64 total_size; - mem_offset_t base_offset; uint32 mem_index = 0; module_inst->memory_count = memory_count; @@ -1175,10 +1192,37 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, return false; } - for (i = 0, mem_index = module->import_memory_count, - memory = module_inst->global_table_data.memory_instances - + module->import_memory_count; - i < memory_count; i++, memory++, mem_index++) { + /* process imported memories */ + for (mem_index = 0; mem_index < module->import_memory_count; mem_index++) { + AOTImportMemory *memory_type = module->import_memories + mem_index; + + const WASMExternInstance *extern_inst = + get_extern_instance_from_imports(imports, import_count, + WASM_IMPORT_EXPORT_KIND_MEMORY, + mem_index); + if (!extern_inst) { + LOG_ERROR("missing a import memory(%s, %s)", + memory_type->module_name, memory_type->memory_name); + return false; + } + + /* just in case */ +#ifndef NDEBUG + if (strcmp(memory_type->memory_name, extern_inst->field_name)) { + LOG_ERROR( + "mismatched import memory name: expect \"%s\", got \"%s\"", + memory_type->memory_name, extern_inst->field_name); + return false; + } +#endif + module_inst->memories[mem_index] = extern_inst->u.memory; + } + + /* process internal memories */ + AOTMemoryInstance *memory = module_inst->global_table_data.memory_instances; + for (uint32 i = 0; mem_index < memory_count; mem_index++, memory++) { + bh_assert(i == mem_index - module->import_memory_count); + AOTMemory *memory_type = module->memories + i; uint32 max_page_count = wasm_runtime_get_max_mem( max_memory_pages, memory_type->init_page_count, @@ -1203,8 +1247,9 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, return false; } - for (i = 0; i < module->mem_init_data_count; i++) { - data_seg = module->mem_init_data_list[i]; + for (uint32 i = 0; i < module->mem_init_data_count; i++) { + AOTMemInitData *data_seg = module->mem_init_data_list[i]; + mem_offset_t base_offset; #if WASM_ENABLE_BULK_MEMORY != 0 if (data_seg->is_passive) continue; @@ -1222,7 +1267,8 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, /* Resolve memory data base offset */ if (data_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { - global_index = data_seg->offset.u.global_index; + uint32 global_index = data_seg->offset.u.global_index; + uint32 global_data_offset = 0; if (!check_global_init_expr(module, global_index, error_buf, error_buf_size)) { @@ -1282,7 +1328,7 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, } /* check offset + length(could be zero) */ - length = data_seg->byte_count; + uint32 length = data_seg->byte_count; if (base_offset + length > memory_inst->memory_data_size) { LOG_DEBUG("base_offset(%" PR_MEM_OFFSET ") + length(%d) > memory_data_size(%" PRIu64 ")", @@ -1308,8 +1354,7 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, } AOTMemoryInstance * -aot_create_memory(const AOTModule *module, const AOTMemoryType *type, - uint32 index) +aot_create_memory(const AOTModule *module, const AOTMemoryType *type) { if (!module || !type) return NULL; @@ -1840,9 +1885,12 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, #endif #if WASM_ENABLE_MULTI_MODULE == 0 - if (module->import_count > 0 && !imports) { + uint32 total_import_count = + module->import_func_count + module->import_global_count + + module->import_memory_count + module->import_table_count; + if (total_import_count > 0 && !imports) { set_error_buf(error_buf, error_buf_size, - "argument imports is NULL while module has imports"); + "imports is NULL while module has imports"); return NULL; } #endif @@ -1981,6 +2029,16 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, if (!global_instantiate(module_inst, module, error_buf, error_buf_size)) goto fail; + /* __heap_base */ + uint8 *aux_heap_base_global_data = NULL; + if (module_inst->e->globals + && module->aux_heap_base_global_index < module->global_count) { + aux_heap_base_global_data = + module_inst->global_data + + module_inst->e->globals[module->aux_heap_base_global_index] + .data_offset; + } + /* Initialize table info */ p += module->global_data_size; module_inst->table_count = module->table_count + module->import_table_count; @@ -1989,8 +2047,9 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, goto fail; /* Initialize memory space */ - if (!memories_instantiate(module_inst, parent, module, heap_size, - max_memory_pages, error_buf, error_buf_size)) + if (!memories_instantiate(module, module_inst, parent, heap_size, + max_memory_pages, aux_heap_base_global_data, + imports, import_count, error_buf, error_buf_size)) goto fail; /* Initialize function pointers */ diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index c86db32c3d..e2d67aa730 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -533,7 +533,8 @@ aot_resolve_import_func(AOTModule *module, AOTImportFunc *import_func); AOTModuleInstance * aot_instantiate(AOTModule *module, AOTModuleInstance *parent, WASMExecEnv *exec_env_main, uint32 stack_size, uint32 heap_size, - uint32 max_memory_pages, char *error_buf, + uint32 max_memory_pages, uint32 import_count, + const WASMExternInstance *imports, char *error_buf, uint32 error_buf_size); /** @@ -546,8 +547,7 @@ void aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst); AOTMemoryInstance * -aot_create_memory(const AOTModule *module, const AOTMemoryType *type, - uint32 index); +aot_create_memory(const AOTModule *module, const AOTMemoryType *type); void aot_destroy_memory(AOTMemoryInstance *memory); diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 52797ea29c..71195880a8 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -1548,7 +1548,7 @@ wasm_runtime_create_memory(WASMModuleCommon *module, WASMMemoryType *type) #if WASM_ENABLE_AOT != 0 if (module->module_type == Wasm_Module_AoT) { - return aot_create_memory((AOTModule *)module, type, index); + return aot_create_memory((AOTModule *)module, type); } #endif diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index ac61571866..6de6175390 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -1638,7 +1638,8 @@ wasm_runtime_instantiate_internal(WASMModuleCommon *module, if (module->module_type == Wasm_Module_AoT) return (WASMModuleInstanceCommon *)aot_instantiate( (AOTModule *)module, (AOTModuleInstance *)parent, exec_env_main, - stack_size, heap_size, max_memory_pages, error_buf, error_buf_size); + stack_size, heap_size, max_memory_pages, import_count, imports, + error_buf, error_buf_size); #endif set_error_buf(error_buf, error_buf_size, "Instantiate module failed, invalid module type"); diff --git a/core/iwasm/compilation/aot.c b/core/iwasm/compilation/aot.c index 7640559dcd..9c05f72b51 100644 --- a/core/iwasm/compilation/aot.c +++ b/core/iwasm/compilation/aot.c @@ -615,24 +615,28 @@ aot_create_comp_data(WASMModule *module, const char *target_arch, if (comp_data->import_memory_count) { comp_data->import_memories = aot_create_import_memories(module, comp_data->import_memory_count); - if (!comp_data->import_memories) + if (!comp_data->import_memories) { goto fail; + } } /* Create memories */ comp_data->memory_count = module->memory_count; /* Allocate memory for memory array, reserve one AOTMemory space at least */ - /* TODO: multi-memory */ - if (comp_data->memory_count == 0) + if (module->import_memory_count + module->memory_count == 0) comp_data->memory_count = 1; - comp_data->memories = aot_create_memories(module, comp_data->memory_count); - if (!comp_data->memories) - goto fail; + if (comp_data->memory_count) { + comp_data->memories = + aot_create_memories(module, comp_data->memory_count); + if (!comp_data->memories) { + goto fail; + } + } /* inherit reserved memory. keep it in memories[] */ - if (!module->memory_count) { + if (module->import_memory_count + module->memory_count == 0) { comp_data->memories[0].num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; } diff --git a/core/iwasm/compilation/aot_compiler.h b/core/iwasm/compilation/aot_compiler.h index 06d8e42bdb..a4a5c9fecc 100644 --- a/core/iwasm/compilation/aot_compiler.h +++ b/core/iwasm/compilation/aot_compiler.h @@ -528,8 +528,28 @@ set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type) wasm_runtime_free(aot_value); \ } while (0) +/* requires comp_ctx */ +#define DEFAULT_MEMORY_TYPE \ + (comp_ctx->comp_data->import_memory_count \ + ? comp_ctx->comp_data->import_memories[0].mem_type \ + : comp_ctx->comp_data->memories[0]) + +#define DEFAULT_MEMORY_TYPE_FLAG (DEFAULT_MEMORY_TYPE.flags) + +#define DEFAULT_MEMORY_TYPE_NUM_BYTES_PER_PAGE \ + (DEFAULT_MEMORY_TYPE.num_bytes_per_page) + +#define DEFAULT_MEMORY_TYPE_INIT_PAGE_COUNT \ + (DEFAULT_MEMORY_TYPE.init_page_count) + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define IS_SHARED_MEMORY (DEFAULT_MEMORY_TYPE_FLAG & SHARED_MEMORY_FLAG) +#else +#define IS_SHARED_MEMORY 0 +#endif + #if WASM_ENABLE_MEMORY64 != 0 -#define IS_MEMORY64 (comp_ctx->comp_data->memories[0].flags & MEMORY64_FLAG) +#define IS_MEMORY64 (DEFAULT_MEMORY_TYPE_FLAG & MEMORY64_FLAG) #define MEMORY64_COND_VALUE(VAL_IF_ENABLED, VAL_IF_DISABLED) \ (IS_MEMORY64 ? VAL_IF_ENABLED : VAL_IF_DISABLED) #define IS_TABLE64(i) \ diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index cd48fed917..876f3873b8 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -165,17 +165,15 @@ get_mem_init_data_size(AOTCompContext *comp_ctx, AOTMemInitData *mem_init_data) } static uint32 -get_mem_init_data_list_size(AOTCompContext *comp_ctx, - AOTMemInitData **mem_init_data_list, - uint32 mem_init_data_count) +get_mem_init_data_list_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) { - AOTMemInitData **mem_init_data = mem_init_data_list; + AOTMemInitData **mem_init_data = comp_data->mem_init_data_list; uint32 size = 0, i; /* mem_init_data_count */ size = (uint32)sizeof(uint32); - for (i = 0; i < mem_init_data_count; i++, mem_init_data++) { + for (i = 0; i < comp_data->mem_init_data_count; i++, mem_init_data++) { size = align_uint(size, 4); size += get_mem_init_data_size(comp_ctx, *mem_init_data); } @@ -210,7 +208,7 @@ get_import_memory_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) } static uint32 -get_memory_size(AOTCompData *comp_data) +get_memory_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) { /* memory_count + count * (flags + num_bytes_per_page + init_page_count + max_page_count) */ @@ -218,44 +216,6 @@ get_memory_size(AOTCompData *comp_data) + comp_data->memory_count * sizeof(uint32) * 4); } -static uint32 -get_mem_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) -{ - /* - * ------------------------------------------------------- - * | u32 import_memory_count - * ------------------------------------------------------- - * | | u8 module_name - * | | u8 field_name - * | | u32 flags - * | AOTImportMemory[import_memory_count] | u32 num_bytes_per_page - * | | u32 init_page_count - * | | u32 max_page_count - * ------------------------------------------------------- - * | u32 memory_count - * ------------------------------------------------------- - * | | u32 flags - * | AOTMemoryType[memory_count] | u32 num_bytes_per_page - * | | u32 init_page_count - * | | u32 max_page_count - * ------------------------------------------------------- - * | u32 mem_init_data_count - * ------------------------------------------------------- - * | | u32 init expr type - * | AOTMemInitData[mem_init_data_count] | u32 init expr value - * | | u32 byte count - * | | u8* bytes - * ------------------------------------------------------- - */ - /* import_memory_size + memory_size - + init_data_count + init_data_list */ - return get_import_memory_size(comp_ctx, comp_data) - + get_memory_size(comp_data) - + get_mem_init_data_list_size(comp_ctx, - comp_data->mem_init_data_list, - comp_data->mem_init_data_count); -} - static uint32 get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data, InitializerExpression *expr) @@ -816,7 +776,43 @@ get_init_data_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data, { uint32 size = 0; - size += get_mem_info_size(comp_ctx, comp_data); + /* + * ------------------------------------------------------- + * | u32 import_memory_count + * ------------------------------------------------------- + * | | u8 module_name + * | | u8 field_name + * | | u32 flags + * | AOTImportMemory[import_memory_count] | u32 num_bytes_per_page + * | | u32 init_page_count + * | | u32 max_page_count + * ------------------------------------------------------- + * | padding + * ------------------------------------------------------- + * | u32 memory_count + * ------------------------------------------------------- + * | | u32 flags + * | AOTMemoryType[memory_count] | u32 num_bytes_per_page + * | | u32 init_page_count + * | | u32 max_page_count + * ------------------------------------------------------- + * | padding (TBC: previous aot doesn't have this padding by design) + * ------------------------------------------------------- + * | u32 mem_init_data_count + * ------------------------------------------------------- + * | | u32 init expr type + * | AOTMemInitData[mem_init_data_count] | u32 init expr value + * | | u32 byte count + * | | u8* bytes + * ------------------------------------------------------- + */ + size += get_import_memory_size(comp_ctx, comp_data); + + size = align_uint(size, 4); + size += get_memory_size(comp_ctx, comp_data); + + size = align_uint(size, 4); + size += get_mem_init_data_list_size(comp_ctx, comp_data); size = align_uint(size, 4); size += get_table_info_size(comp_ctx, comp_data); @@ -1852,7 +1848,7 @@ aot_emit_memory_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, EMIT_U32(memory->max_page_count); } - if (offset - *p_offset != get_memory_size(comp_data)) { + if (offset - *p_offset != get_memory_size(comp_ctx, comp_data)) { aot_set_last_error("emit memory info failed."); return false; } @@ -1896,8 +1892,7 @@ aot_emit_memory_init_data_list(uint8 *buf, uint8 *buf_end, uint32 *p_offset, } if (offset - *p_offset - != get_mem_init_data_list_size(comp_ctx, comp_data->mem_init_data_list, - comp_data->mem_init_data_count)) { + != get_mem_init_data_list_size(comp_ctx, comp_data)) { aot_set_last_error("emit memory init data list failed."); return false; } diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index 1c50fe75f6..c2f190c867 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -881,8 +881,7 @@ check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef terminate_addr, terminate_flags, flag, offset, res; LLVMBasicBlockRef terminate_block, non_terminate_block; AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; - bool is_shared_memory = - comp_ctx->comp_data->memories[0].flags & 0x02 ? true : false; + bool is_shared_memory = IS_SHARED_MEMORY; /* Only need to check the suspend flags when memory is shared since shared memory must be enabled for multi-threading */ diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 869a1dbb27..b3b4d33366 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -128,8 +128,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, bool is_target_64bit, is_local_of_aot_value = false; bool is_const = false; #if WASM_ENABLE_SHARED_MEMORY != 0 - bool is_shared_memory = - comp_ctx->comp_data->memories[0].flags & SHARED_MEMORY_FLAG; + bool is_shared_memory = IS_SHARED_MEMORY; #endif #if WASM_ENABLE_MEMORY64 == 0 bool is_memory64 = false; @@ -207,10 +206,8 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, value = const_value; } uint64 mem_offset = value + (uint64)offset; - uint32 num_bytes_per_page = - comp_ctx->comp_data->memories[0].num_bytes_per_page; - uint32 init_page_count = - comp_ctx->comp_data->memories[0].init_page_count; + uint32 num_bytes_per_page = DEFAULT_MEMORY_TYPE_NUM_BYTES_PER_PAGE; + uint32 init_page_count = DEFAULT_MEMORY_TYPE_INIT_PAGE_COUNT; uint64 mem_data_size = (uint64)num_bytes_per_page * init_page_count; if (alignp != NULL) { @@ -408,8 +405,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, && !(is_local_of_aot_value && aot_checked_addr_list_find(func_ctx, local_idx_of_aot_value, offset, bytes))) { - uint32 init_page_count = - comp_ctx->comp_data->memories[0].init_page_count; + uint32 init_page_count = DEFAULT_MEMORY_TYPE_INIT_PAGE_COUNT; if (init_page_count == 0) { LLVMValueRef mem_size; @@ -1149,7 +1145,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Get memory base address and memory data size */ #if WASM_ENABLE_SHARED_MEMORY != 0 - bool is_shared_memory = comp_ctx->comp_data->memories[0].flags & 0x02; + bool is_shared_memory = IS_SHARED_MEMORY; if (func_ctx->mem_space_unchanged || is_shared_memory) { #else @@ -1174,10 +1170,8 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (LLVMIsEfficientConstInt(offset) && LLVMIsEfficientConstInt(bytes)) { uint64 mem_offset = (uint64)LLVMConstIntGetZExtValue(offset); uint64 mem_len = (uint64)LLVMConstIntGetZExtValue(bytes); - uint32 num_bytes_per_page = - comp_ctx->comp_data->memories[0].num_bytes_per_page; - uint32 init_page_count = - comp_ctx->comp_data->memories[0].init_page_count; + uint32 num_bytes_per_page = DEFAULT_MEMORY_TYPE_NUM_BYTES_PER_PAGE; + uint32 init_page_count = DEFAULT_MEMORY_TYPE_INIT_PAGE_COUNT; uint64 mem_data_size = (uint64)num_bytes_per_page * init_page_count; if (mem_data_size > 0 && mem_offset + mem_len <= mem_data_size) { /* inside memory space */ diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index fb1c4308b2..45e97b7425 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1204,11 +1204,10 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, func_ctx->mem_space_unchanged = mem_space_unchanged; - memory_count = module->memory_count + module->import_memory_count; - /* If the module doesn't have memory, reserve - one mem_info space with empty content */ - if (memory_count == 0) - memory_count = 1; + /* comp_data contains processed reservation */ + memory_count = comp_ctx->comp_data->import_memory_count + + comp_ctx->comp_data->memory_count; + bh_assert(memory_count > 0); if (!(func_ctx->mem_info = wasm_runtime_malloc(sizeof(AOTMemInfo) * memory_count))) { @@ -1219,8 +1218,7 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Currently we only create memory info for memory 0 */ /* Load memory base address */ #if WASM_ENABLE_SHARED_MEMORY != 0 - is_shared_memory = - comp_ctx->comp_data->memories[0].flags & 0x02 ? true : false; + is_shared_memory = IS_SHARED_MEMORY; if (is_shared_memory) { LLVMValueRef shared_mem_addr; offset = I32_CONST(offsetof(AOTModuleInstance, memories)); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 7599f9168f..8d272142b6 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -300,15 +300,15 @@ memory_instantiate(const WASMModule *module, WASMModuleInstance *parent, uint8 *aux_heap_base_global_data, char *error_buf, uint32 error_buf_size) { + bh_assert(memory != NULL); + bool is_shared_memory = false; #if WASM_ENABLE_SHARED_MEMORY != 0 is_shared_memory = flags & SHARED_MEMORY_FLAG ? true : false; - bh_assert(memory != NULL); - /* shared memory */ if (is_shared_memory && parent != NULL) { - bh_assert(parent->memory_count > 0); + bh_assert(parent_memory != NULL); memory = parent_memory; shared_memory_inc_reference(memory); return memory; @@ -360,6 +360,7 @@ memory_instantiate(const WASMModule *module, WASMModuleInstance *parent, } else if (heap_size > 0) { uint32 inc_page_count = 0; + if (init_page_count == max_page_count && init_page_count == 0) { /* If the memory data size is always 0, we resize it to one page for app heap */ @@ -447,8 +448,6 @@ memory_instantiate(const WASMModule *module, WASMModuleInstance *parent, num_bytes_per_page, init_page_count, max_page_count); LOG_VERBOSE(" data offset: %" PRIu64 ", stack size: %d", module->aux_data_end, module->aux_stack_size); - LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %u\n", heap_offset, - heap_size); #ifndef NDEBUG { @@ -613,6 +612,7 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, uint32 max_page_count = wasm_runtime_get_max_mem( max_memory_pages, module->memories[i].init_page_count, module->memories[i].max_page_count); + if (!(memories[mem_index] = memory_instantiate( module, parent, memory, parent ? parent->memories[mem_index] : NULL, @@ -635,6 +635,9 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, WASMMemoryInstance * wasm_create_memory(const WASMModule *module, const WASMMemoryType *type) { + if (!module || !type) + return NULL; + WASMMemoryInstance *memory = NULL; char error_buf[64] = { 0 }; @@ -655,7 +658,7 @@ wasm_create_memory(const WASMModule *module, const WASMMemoryType *type) NULL, // parent_memory type->num_bytes_per_page, type->init_page_count, type->max_page_count, - 0, // no heap_size from host side + 0, // no app heap for host type->flags, NULL, // aux_heap_base_global_data error_buf, sizeof(error_buf))) { From 454d576d7d5b4405653c6318649b09e4e4e3648f Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Thu, 31 Oct 2024 12:42:41 +0000 Subject: [PATCH 06/14] fix memory instantiation issues --- core/iwasm/aot/aot_loader.c | 1 + core/iwasm/aot/aot_runtime.c | 23 ++++++++++++++++++---- core/iwasm/compilation/aot_emit_table.c | 15 +++++++------- core/iwasm/compilation/aot_emit_variable.c | 7 ++++--- core/iwasm/interpreter/wasm_runtime.c | 18 +++++++++++++++++ 5 files changed, 50 insertions(+), 14 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index e01af5f178..4be5c87240 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -3855,6 +3855,7 @@ has_module_memory64(AOTModule *module) { /* TODO: multi-memories for now assuming the memory idx type is consistent * across multi-memories */ + /*FIXME: support import memory */ if (module->import_memory_count > 0) return !!(module->import_memories[0].mem_type.flags & MEMORY64_FLAG); else if (module->memory_count > 0) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 809a7afb3d..497883f424 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1192,8 +1192,11 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, return false; } + AOTMemoryInstance *memory = module_inst->global_table_data.memory_instances; + /* process imported memories */ - for (mem_index = 0; mem_index < module->import_memory_count; mem_index++) { + for (mem_index = 0; mem_index < module->import_memory_count; + mem_index++, memory++) { AOTImportMemory *memory_type = module->import_memories + mem_index; const WASMExternInstance *extern_inst = @@ -1215,11 +1218,20 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, return false; } #endif + /* + *TODO: + * - either memories[x] points to an external AOTMemoryInstance. + * - or memories[x] points to an internal AOTMemoryInstance in + * global_table_data + * + * the first case is simple for maintaining resource management + */ module_inst->memories[mem_index] = extern_inst->u.memory; + bh_memcpy_s(memory, sizeof(AOTMemoryInstance), extern_inst->u.memory, + sizeof(AOTMemoryInstance)); } /* process internal memories */ - AOTMemoryInstance *memory = module_inst->global_table_data.memory_instances; for (uint32 i = 0; mem_index < memory_count; mem_index++, memory++) { bh_assert(i == mem_index - module->import_memory_count); @@ -1875,7 +1887,8 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, const uint32 module_inst_struct_size = offsetof(AOTModuleInstance, global_table_data.bytes); const uint64 module_inst_mem_inst_size = - (uint64)module->memory_count * sizeof(AOTMemoryInstance); + (uint64)(module->memory_count + module->import_memory_count) + * sizeof(AOTMemoryInstance); uint64 total_size, table_size = 0; uint8 *p; uint32 i, extra_info_offset; @@ -3648,7 +3661,9 @@ aot_get_module_mem_consumption(const AOTModule *module, mem_conspn->tables_size = sizeof(AOTTable) * module->table_count; - mem_conspn->memories_size = sizeof(AOTMemory) * module->memory_count; + mem_conspn->memories_size = + sizeof(AOTMemory) + * (module->memory_count + moudle->import_memory_count); mem_conspn->globals_size = sizeof(AOTGlobal) * module->global_count; mem_conspn->exports_size = sizeof(AOTExport) * module->export_count; diff --git a/core/iwasm/compilation/aot_emit_table.c b/core/iwasm/compilation/aot_emit_table.c index e8dae410bd..f8c9bf6bed 100644 --- a/core/iwasm/compilation/aot_emit_table.c +++ b/core/iwasm/compilation/aot_emit_table.c @@ -90,13 +90,14 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx, AOTImportTable *imp_tbls = comp_ctx->comp_data->import_tables; AOTTable *tbls = comp_ctx->comp_data->tables; - offset = - offsetof(AOTModuleInstance, global_table_data.bytes) - + (uint64)comp_ctx->comp_data->memory_count * sizeof(AOTMemoryInstance) - /* Get global data size according to target info */ - + (comp_ctx->pointer_size == sizeof(uint64) - ? comp_ctx->comp_data->global_data_size_64bit - : comp_ctx->comp_data->global_data_size_32bit); + offset = offsetof(AOTModuleInstance, global_table_data.bytes) + + (uint64)(comp_ctx->comp_data->memory_count + + comp_ctx->comp_data->import_memory_count) + * sizeof(AOTMemoryInstance) + /* Get global data size according to target info */ + + (comp_ctx->pointer_size == sizeof(uint64) + ? comp_ctx->comp_data->global_data_size_64bit + : comp_ctx->comp_data->global_data_size_32bit); while (i < tbl_idx && i < comp_ctx->comp_data->import_table_count) { offset += offsetof(AOTTableInstance, elems); diff --git a/core/iwasm/compilation/aot_emit_variable.c b/core/iwasm/compilation/aot_emit_variable.c index fed8c5550e..bb6d10b8ec 100644 --- a/core/iwasm/compilation/aot_emit_variable.c +++ b/core/iwasm/compilation/aot_emit_variable.c @@ -161,9 +161,10 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, global_ptr, global, res; LLVMTypeRef ptr_type = NULL; - global_base_offset = - offsetof(AOTModuleInstance, global_table_data.bytes) - + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count; + global_base_offset = offsetof(AOTModuleInstance, global_table_data.bytes) + + sizeof(AOTMemoryInstance) + * (comp_ctx->comp_data->memory_count + + comp_ctx->comp_data->import_memory_count); bh_assert(global_idx < import_global_count + comp_data->global_count); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 8d272142b6..5f76a5e39a 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2741,6 +2741,24 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } module_inst->memories[import_memory_index] = import->u.memory; + { + /* + *TODO: + * - either memories[x] points to an external + *AOTMemoryInstance. + * - or memories[x] points to an internal AOTMemoryInstance + *in global_table_data + * + * the first case is simple for maintaining resource + *management + */ + WASMMemoryInstance *memory = + module_inst->global_table_data.memory_instances + + import_memory_index; + bh_memcpy_s(memory, sizeof(WASMMemoryInstance), + import->u.memory, sizeof(WASMMemoryInstance)); + } + import_memory_index++; } } From 929e4a336fa798f4ea8e854e85488bd6161f8d42 Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Fri, 1 Nov 2024 03:11:49 +0000 Subject: [PATCH 07/14] Refactor aot_instantiate function parameters for improved clarity --- core/iwasm/aot/aot_runtime.c | 5 ++--- core/iwasm/aot/aot_runtime.h | 5 ++--- core/iwasm/common/wasm_runtime_common.c | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 497883f424..22ea6073fc 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1875,9 +1875,8 @@ check_linked_symbol(AOTModule *module, char *error_buf, uint32 error_buf_size) AOTModuleInstance * aot_instantiate(AOTModule *module, AOTModuleInstance *parent, WASMExecEnv *exec_env_main, uint32 stack_size, uint32 heap_size, - uint32 max_memory_pages, uint32 import_count, - const WASMExternInstance *imports, char *error_buf, - uint32 error_buf_size) + uint32 max_memory_pages, const WASMExternInstance *imports, + uint32 import_count, char *error_buf, uint32 error_buf_size) { AOTModuleInstance *module_inst; #if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_REF_TYPES != 0 diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index e2d67aa730..9d58a873a8 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -533,9 +533,8 @@ aot_resolve_import_func(AOTModule *module, AOTImportFunc *import_func); AOTModuleInstance * aot_instantiate(AOTModule *module, AOTModuleInstance *parent, WASMExecEnv *exec_env_main, uint32 stack_size, uint32 heap_size, - uint32 max_memory_pages, uint32 import_count, - const WASMExternInstance *imports, char *error_buf, - uint32 error_buf_size); + uint32 max_memory_pages, const WASMExternInstance *imports, + uint32 import_count, char *error_buf, uint32 error_buf_size); /** * Deinstantiate a AOT module instance, destroy the resources. diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 6de6175390..11e5ed0b73 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -1638,7 +1638,7 @@ wasm_runtime_instantiate_internal(WASMModuleCommon *module, if (module->module_type == Wasm_Module_AoT) return (WASMModuleInstanceCommon *)aot_instantiate( (AOTModule *)module, (AOTModuleInstance *)parent, exec_env_main, - stack_size, heap_size, max_memory_pages, import_count, imports, + stack_size, heap_size, max_memory_pages, imports, import_count, error_buf, error_buf_size); #endif set_error_buf(error_buf, error_buf_size, From ac405d3ec1f1fdb04a9a85d9d6905a0c7fa0c39d Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Fri, 1 Nov 2024 04:21:20 +0000 Subject: [PATCH 08/14] Add wasm_runtime_get_extern_instance function and refactor memory instantiation logic in wasm_runtime.c --- core/iwasm/aot/aot_runtime.c | 32 ++----- core/iwasm/common/wasm_runtime_common.c | 21 +++++ core/iwasm/common/wasm_runtime_common.h | 7 +- core/iwasm/interpreter/wasm_runtime.c | 117 +++++++++++------------- 4 files changed, 85 insertions(+), 92 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 22ea6073fc..19e1965e56 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -178,27 +178,6 @@ get_prev_frame(WASMExecEnv *exec_env, void *cur_frame) } #endif -static const WASMExternInstance * -get_extern_instance_from_imports(const WASMExternInstance *imports, - uint32 import_count, - wasm_import_export_kind_t kind, uint32 index) -{ - for (uint32 i = 0, target_kind_index = 0; i < import_count; i++) { - if (imports[i].kind != kind) { - continue; - } - - if (target_kind_index != index) { - target_kind_index++; - continue; - } - - return imports + i; - } - - return NULL; -} - static bool check_global_init_expr(const AOTModule *module, uint32 global_index, char *error_buf, uint32 error_buf_size) @@ -1187,6 +1166,7 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, module_inst->memory_count = memory_count; total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count; + if (!(module_inst->memories = runtime_malloc(total_size, error_buf, error_buf_size))) { return false; @@ -1200,7 +1180,7 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, AOTImportMemory *memory_type = module->import_memories + mem_index; const WASMExternInstance *extern_inst = - get_extern_instance_from_imports(imports, import_count, + wasm_runtime_get_extern_instance(imports, import_count, WASM_IMPORT_EXPORT_KIND_MEMORY, mem_index); if (!extern_inst) { @@ -1232,10 +1212,10 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, } /* process internal memories */ - for (uint32 i = 0; mem_index < memory_count; mem_index++, memory++) { - bh_assert(i == mem_index - module->import_memory_count); + for (; mem_index < memory_count; mem_index++, memory++) { + AOTMemory *memory_type = + module->memories + mem_index - module->import_memory_count; - AOTMemory *memory_type = module->memories + i; uint32 max_page_count = wasm_runtime_get_max_mem( max_memory_pages, memory_type->init_page_count, memory_type->max_page_count); @@ -1247,7 +1227,7 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, /* only inst->memories[0] will have a app heap */ mem_index == 0 ? heap_size : 0, memory_type->flags, aux_heap_base_global_data, error_buf, error_buf_size); - if (!module_inst->memories[i]) { + if (!module_inst->memories[mem_index]) { return false; } } diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 11e5ed0b73..b86817c44e 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -7944,3 +7944,24 @@ wasm_runtime_disinherit_imports(WASMModuleCommon *module, LOG_ERROR("disinherit imports failed, invalid module type"); } #endif /* WASM_ENABLE_LIB_WASI_THREADS != 0 || WASM_ENABLE_THREAD_MGR != 0 */ + +const WASMExternInstance * +wasm_runtime_get_extern_instance(const WASMExternInstance *imports, + uint32 import_count, + wasm_import_export_kind_t kind, uint32 index) +{ + for (uint32 i = 0, target_kind_index = 0; i < import_count; i++) { + if (imports[i].kind != kind) { + continue; + } + + if (target_kind_index != index) { + target_kind_index++; + continue; + } + + return imports + i; + } + + return NULL; +} diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 58e99a2730..c1a36746be 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -1231,7 +1231,12 @@ void wasm_runtime_disinherit_imports(WASMModuleCommon *module, WASMExternInstance *imports, int32 import_count); -#endif +#endif /* WASM_ENABLE_LIB_WASI_THREADS != 0 || WASM_ENABLE_THREAD_MGR != 0 */ + +const WASMExternInstance * +wasm_runtime_get_extern_instance(const WASMExternInstance *imports, + uint32 import_count, + wasm_import_export_kind_t kind, uint32 index); #ifdef __cplusplus } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 5f76a5e39a..037faa72c0 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -535,12 +535,13 @@ static WASMMemoryInstance ** memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, WASMModuleInstance *parent, uint32 heap_size, uint32 max_memory_pages, uint8 *aux_heap_base_global_data, + const WASMExternInstance *imports, uint32 import_count, char *error_buf, uint32 error_buf_size) { uint32 mem_index = 0, i, memory_count = module->import_memory_count + module->memory_count; uint64 total_size; - WASMMemoryInstance **memories, *memory; + WASMMemoryInstance **memories; total_size = sizeof(WASMMemoryInstance *) * (uint64)memory_count; @@ -548,25 +549,29 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, return NULL; } - memory = module_inst->global_table_data.memory_instances; + WASMMemoryInstance *memory = + module_inst->global_table_data.memory_instances; /* instantiate memories from import section */ + for (mem_index = 0; mem_index < module->import_memory_count; + mem_index++, memory++) { + WASMMemoryImport *memory_type = + &((module->import_memories + mem_index)->u.memory); + #if WASM_ENABLE_MULTI_MODULE != 0 - WASMImport *import = module->import_memories; - for (i = 0; i < module->import_memory_count; i++, import++, memory++) { // TODO: ? make sure import->u.memory.import_module is set properly - if (import->u.memory.import_module != NULL) { + if (memory_type->import_module != NULL) { WASMModuleInstance *module_inst_linked; if (!(module_inst_linked = get_sub_module_inst( - module_inst, import->u.memory.import_module))) { + module_inst, memory_type->import_module))) { set_error_buf(error_buf, error_buf_size, "unknown memory"); memories_deinstantiate(module_inst, memories, memory_count); return NULL; } if (!(memories[mem_index++] = wasm_lookup_memory( - module_inst_linked, import->u.memory.field_name))) { + module_inst_linked, memory_type->field_name))) { set_error_buf(error_buf, error_buf_size, "unknown memory"); memories_deinstantiate(module_inst, memories, memory_count); return NULL; @@ -576,12 +581,12 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, // TODO: Although it is for inherited memory, it misses a situation // where the memory is imported from host or other modules uint32 num_bytes_per_page = - import->u.memory.mem_type.num_bytes_per_page; - uint32 init_page_count = import->u.memory.mem_type.init_page_count; + memory_type->mem_type.num_bytes_per_page; + uint32 init_page_count = memory_type->mem_type.init_page_count; uint32 max_page_count = wasm_runtime_get_max_mem( - max_memory_pages, import->u.memory.mem_type.init_page_count, - import->u.memory.mem_type.max_page_count); - uint32 flags = import->u.memory.mem_type.flags; + max_memory_pages, memory_type->mem_type.init_page_count, + memory_type->mem_type.max_page_count); + uint32 flags = memory_type->mem_type.flags; if (!(memories[mem_index] = memory_instantiate( module, parent, memory, @@ -595,17 +600,45 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, } mem_index++; } +#else + + const WASMExternInstance *extern_inst = + wasm_runtime_get_extern_instance(imports, import_count, + WASM_IMPORT_EXPORT_KIND_MEMORY, + mem_index); + if (!extern_inst) { + LOG_ERROR("missing a import memory(%s, %s)", + memory_type->module_name, memory_type->field_name); + return false; + } + + /* just in case */ +#ifndef NDEBUG + if (strcmp(memory_type->field_name, extern_inst->field_name)) { + LOG_ERROR( + "mismatched import memory name: expect \"%s\", got \"%s\"", + memory_type->field_name, extern_inst->field_name); + return false; + } +#endif + /* + *TODO: + * - either memories[x] points to an external AOTMemoryInstance. + * - or memories[x] points to an internal AOTMemoryInstance in + * global_table_data + * + * the first case is simple for maintaining resource management + */ + memories[mem_index] = extern_inst->u.memory; + bh_memcpy_s(memory, sizeof(WASMMemoryInstance), extern_inst->u.memory, + sizeof(WASMMemoryInstance)); +#endif /* WASM_ENABLE_MULTI_MODULE != 0 */ } bh_assert(mem_index == module->import_memory_count); bh_assert(memory == module_inst->global_table_data.memory_instances + module->import_memory_count); -#else - mem_index = module->import_memory_count; - memory = module_inst->global_table_data.memory_instances - + module->import_memory_count; -#endif /* WASM_ENABLE_MULTI_MODULE != 0 */ /* instantiate memories from memory section */ for (i = 0; i < module->memory_count; i++, memory++) { @@ -2673,7 +2706,8 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, if ((module_inst->memory_count > 0 && !(module_inst->memories = memories_instantiate( module, module_inst, parent, heap_size, max_memory_pages, - aux_heap_base_global_data, error_buf, error_buf_size))) + aux_heap_base_global_data, imports, import_count, error_buf, + error_buf_size))) || (module_inst->table_count > 0 && !(module_inst->tables = tables_instantiate(module, module_inst, first_table, @@ -2718,53 +2752,6 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, goto fail; } -#if WASM_ENABLE_MULTI_MODULE == 0 - /* imports */ - /* - * const WASMExternInstance *imports should have the same order - * with import section content in .wasm. - */ - { - uint32 import_memory_index = 0; - uint32 import_index = 0; - for (; import_index < import_count; import_index++) { - /* TODO: remove me if the entire instantiation linking feature is - * complete */ - if (imports == NULL) - break; - - const WASMExternInstance *import = imports + import_index; - if (import->kind == WASM_IMPORT_EXPORT_KIND_MEMORY) { - if (import_memory_index >= module->import_memory_count) { - LOG_ERROR("provided import memory not match requirement"); - goto fail; - } - - module_inst->memories[import_memory_index] = import->u.memory; - { - /* - *TODO: - * - either memories[x] points to an external - *AOTMemoryInstance. - * - or memories[x] points to an internal AOTMemoryInstance - *in global_table_data - * - * the first case is simple for maintaining resource - *management - */ - WASMMemoryInstance *memory = - module_inst->global_table_data.memory_instances - + import_memory_index; - bh_memcpy_s(memory, sizeof(WASMMemoryInstance), - import->u.memory, sizeof(WASMMemoryInstance)); - } - - import_memory_index++; - } - } - } -#endif - if (global_count > 0) { /* Initialize the global data */ global_data = module_inst->global_data; From 365cee961c321e8c44c0507a527c8903592f4bfe Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Fri, 1 Nov 2024 10:44:06 +0000 Subject: [PATCH 09/14] Implement import inheritance and disinheritance functions for WASI threads --- .gitignore | 3 +- core/iwasm/aot/aot_runtime.c | 85 +++++++++++++++++++ core/iwasm/aot/aot_runtime.h | 10 +++ core/iwasm/common/wasm_runtime_common.c | 6 +- core/iwasm/compilation/aot_emit_memory.c | 2 +- .../lib-pthread/lib_pthread_wrapper.c | 3 +- .../lib_wasi_threads_wrapper.c | 3 +- 7 files changed, 106 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index f939a68d16..bf1eeb5623 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ tests/benchmarks/coremark/coremark* samples/workload/include/** !samples/workload/include/.gitkeep -# core/iwasm/libraries/wasi-threads +core/iwasm/libraries/lib-wasi-threads/**/*.aot +core/iwasm/libraries/lib-wasi-threads/**/*.wasm diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 19e1965e56..8327b8ab61 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1881,6 +1881,10 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, module->import_func_count + module->import_global_count + module->import_memory_count + module->import_table_count; if (total_import_count > 0 && !imports) { + /* + * TODO: might be too strict + * might wasm_runtime_create_imports_with_builtin() here by default + */ set_error_buf(error_buf, error_buf_size, "imports is NULL while module has imports"); return NULL; @@ -5453,3 +5457,84 @@ aot_resolve_import_func(AOTModule *module, AOTImportFunc *import_func) #endif return import_func->func_ptr_linked != NULL; } + +#if WASM_ENABLE_LIB_WASI_THREADS != 0 || WASM_ENABLE_THREAD_MGR != 0 +/* + * The function is used to create a new WASMExternInstance list + * for a spawned thread. + */ +int32 +aot_inherit_imports(AOTModule *module, AOTModuleInstance *inst, + WASMExternInstance *out, int32 out_len) +{ + if (!module || !inst || !out) + return -1; + + int32 spawned_import_count = + wasm_runtime_get_import_count((WASMModuleCommon *)module); + if (spawned_import_count > out_len) { + LOG_WARNING("The number of imported functions is more than the " + "length of provided buffer "); + return -1; + } + + for (int32 i = 0, import_memory_index = 0; i < spawned_import_count; i++) { + wasm_import_t import_type = { 0 }; + + wasm_runtime_get_import_type((WASMModuleCommon *)module, i, + &import_type); + + out[i].module_name = import_type.module_name; + out[i].field_name = import_type.name; + out[i].kind = import_type.kind; + + if (import_type.kind == WASM_IMPORT_EXPORT_KIND_MEMORY) { + out[i].u.memory = inst->memories[import_memory_index]; +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_inc_reference(inst->memories[import_memory_index]); +#endif + import_memory_index++; + } + else { + LOG_WARNING("for spawned, inherit() skips import(%s,%s) kind %d", + import_type.module_name, import_type.name, + import_type.kind); + } + } + + return 0; +} + +void +aot_disinherit_imports(AOTModule *module, WASMExternInstance *imports, + int32 import_count) +{ + if (!module || !imports) + return; + + int32 spawned_import_count = + wasm_runtime_get_import_count((WASMModuleCommon *)module); + if (spawned_import_count > import_count) { + LOG_WARNING("The number of imported functions is more than the " + "length of provided buffer "); + return; + } + + for (int32 i = 0; i < import_count; i++) { + WASMExternInstance *import = imports + i; + + if (import->kind == WASM_IMPORT_EXPORT_KIND_MEMORY) { + if (!import->u.memory) + continue; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_dec_reference(import->u.memory); +#endif + } + else { + LOG_WARNING("for spawned, disinherit() skips import(%s,%s) kind %d", + import->module_name, import->field_name, import->kind); + } + } +} +#endif /* WASM_ENABLE_LIB_WASI_THREADS != 0 || WASM_ENABLE_THREAD_MGR != 0 */ diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 9d58a873a8..2b1b66ee25 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -860,6 +860,16 @@ aot_set_module_name(AOTModule *module, const char *name, char *error_buf, const char * aot_get_module_name(AOTModule *module); +#if WASM_ENABLE_LIB_WASI_THREADS != 0 || WASM_ENABLE_THREAD_MGR != 0 +int32 +aot_inherit_imports(AOTModule *module, AOTModuleInstance *inst, + WASMExternInstance *out, int32 out_len); + +void +aot_disinherit_imports(AOTModule *module, WASMExternInstance *imports, + int32 import_count); +#endif /* WASM_ENABLE_LIB_WASI_THREADS != 0 || WASM_ENABLE_THREAD_MGR != 0 */ + #ifdef __cplusplus } /* end of extern "C" */ #endif diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index b86817c44e..782e0b6576 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -7919,7 +7919,8 @@ wasm_runtime_inherit_imports(WASMModuleCommon *module, #endif #if WASM_ENABLE_AOT != 0 if (module->module_type == Wasm_Module_AoT) { - bh_assert(false && "Unsupported operation"); + return aot_inherit_imports((AOTModule *)module, + (AOTModuleInstance *)inst, out, out_len); } #endif LOG_ERROR("inherit imports failed, invalid module type"); @@ -7938,7 +7939,8 @@ wasm_runtime_disinherit_imports(WASMModuleCommon *module, #endif #if WASM_ENABLE_AOT != 0 if (module->module_type == Wasm_Module_AoT) { - bh_assert(false && "Unsupported operation"); + return aot_disinherit_imports((AOTModule *)module, imports, + import_count); } #endif LOG_ERROR("disinherit imports failed, invalid module type"); diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index b3b4d33366..1c386746c3 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -124,7 +124,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; AOTValue *aot_value_top; uint32 local_idx_of_aot_value = 0; - uint64 const_value; + uint64 const_value = 0; bool is_target_64bit, is_local_of_aot_value = false; bool is_const = false; #if WASM_ENABLE_SHARED_MEMORY != 0 diff --git a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c index 18355b2df8..97f1195adc 100644 --- a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c +++ b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c @@ -561,7 +561,7 @@ pthread_create_wrapper(wasm_exec_env_t exec_env, uint32 aux_stack_size; uint64 aux_stack_start = 0; int32 ret = -1; - int32 spawned_import_count = ((WASMModule *)module)->import_count; + int32 spawned_import_count = 0; WASMExternInstance *spawned_imports = NULL; bh_assert(module); @@ -584,6 +584,7 @@ pthread_create_wrapper(wasm_exec_env_t exec_env, /* * build a imports list(WASMExternInstance[]) from parent's imports */ + spawned_import_count = wasm_runtime_get_import_count(module); spawned_imports = wasm_runtime_malloc(sizeof(WASMExternInstance) * spawned_import_count); if (spawned_imports == NULL) { diff --git a/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c b/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c index 9ad31ae3a2..4291ec74f3 100644 --- a/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c +++ b/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c @@ -80,7 +80,7 @@ thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg) int32 thread_id = -1; uint32 stack_size = 8192; int32 ret = -1; - int32 spawned_import_count = ((WASMModule *)module)->import_count; + int32 spawned_import_count = 0; WASMExternInstance *spawned_imports = NULL; bh_assert(module); @@ -91,6 +91,7 @@ thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg) /* * build a imports list(WASMExternInstance[]) from parent's imports */ + spawned_import_count = wasm_runtime_get_import_count(module); spawned_imports = wasm_runtime_malloc(sizeof(WASMExternInstance) * spawned_import_count); if (spawned_imports == NULL) { From b0cc5ce9d13aea8cd44b21d1ca891098ff145413 Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Fri, 1 Nov 2024 11:57:39 +0000 Subject: [PATCH 10/14] Fix memory deinstantiation loop to correctly reference import_memory_count --- core/iwasm/aot/aot_loader.c | 2 - core/iwasm/aot/aot_runtime.c | 231 ++++++++++-------- core/iwasm/interpreter/wasm_runtime.c | 8 +- .../libraries/libc-wasi/libc_wasi_wrapper.c | 5 + 4 files changed, 132 insertions(+), 114 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 4be5c87240..a9abd211dc 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1065,8 +1065,6 @@ load_import_memory_info(const uint8 **p_buf, const uint8 *buf_end, AOTImportMemory *import_memories = loader_malloc(size, error_buf, error_buf_size); if (!import_memories) { - set_error_buf(error_buf, error_buf_size, - "allocate import_memories failed"); return false; } diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 8327b8ab61..d623ec5a3c 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -857,7 +857,7 @@ static void memories_deinstantiate(AOTModuleInstance *module_inst) { /* import memories created by host or linker. released by them too */ - for (uint32 i = module_inst->module->import_memory_count; + for (uint32 i = ((AOTModule *)module_inst->module)->import_memory_count; i < module_inst->memory_count; i++) { memory_deinstantiate(module_inst->memories[i]); } @@ -911,114 +911,110 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, /* TODO: use small functions to reduce the complexity */ /* adjust heap_size, heap_offset and num_bytes_per_page */ uint64 heap_offset = (uint64)num_bytes_per_page * init_page_count; - { - if (heap_size > 0 && module->malloc_func_index != (uint32)-1 - && module->free_func_index != (uint32)-1) { - /* Disable app heap, use malloc/free function exported - by wasm app to allocate/free memory instead */ - heap_size = 0; - } + if (heap_size > 0 && module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1) { + /* Disable app heap, use malloc/free function exported + by wasm app to allocate/free memory instead */ + heap_size = 0; + } + + /* If initial memory is the largest size allowed, disallowing insert + * host managed heap */ + if (heap_size > 0 + && heap_offset == GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; + } - /* If initial memory is the largest size allowed, disallowing insert - * host managed heap */ - if (heap_size > 0 - && heap_offset == GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)) { + if (init_page_count == max_page_count && init_page_count == 1) { + /* If only one page and at most one page, we just append + the app heap to the end of linear memory, enlarge the + num_bytes_per_page, and don't change the page count */ + heap_offset = num_bytes_per_page; + num_bytes_per_page += heap_size; + if (num_bytes_per_page < heap_size) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " "try using `--heap-size=0` option"); return NULL; } - - if (init_page_count == max_page_count && init_page_count == 1) { - /* If only one page and at most one page, we just append - the app heap to the end of linear memory, enlarge the - num_bytes_per_page, and don't change the page count */ - heap_offset = num_bytes_per_page; - num_bytes_per_page += heap_size; - if (num_bytes_per_page < heap_size) { - set_error_buf(error_buf, error_buf_size, - "failed to insert app heap into linear memory, " - "try using `--heap-size=0` option"); - return NULL; + } + else if (heap_size > 0) { + uint32 inc_page_count = 0; + + if (init_page_count == max_page_count && init_page_count == 0) { + /* If the memory data size is always 0, we resize it to + one page for app heap */ + num_bytes_per_page = heap_size; + heap_offset = 0; + inc_page_count = 1; + } + else if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base + < (uint64)num_bytes_per_page * init_page_count) { + /* Insert app heap before __heap_base */ + aux_heap_base = module->aux_heap_base; + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + inc_page_count = + (heap_size - bytes_to_page_end + num_bytes_per_page - 1) + / num_bytes_per_page; + heap_offset = aux_heap_base; + aux_heap_base += heap_size; + + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + if (bytes_to_page_end < 1 * BH_KB) { + aux_heap_base += 1 * BH_KB; + inc_page_count++; } - } - else if (heap_size > 0) { - uint32 inc_page_count = 0; - if (init_page_count == max_page_count && init_page_count == 0) { - /* If the memory data size is always 0, we resize it to - one page for app heap */ - num_bytes_per_page = heap_size; - heap_offset = 0; - inc_page_count = 1; + /* Adjust __heap_base global value */ + if (aux_heap_base_global_data == NULL) { + set_error_buf( + error_buf, error_buf_size, + "auxiliary heap base global data should not be NULL"); + return NULL; } - else if (module->aux_heap_base_global_index != (uint32)-1 - && module->aux_heap_base - < (uint64)num_bytes_per_page * init_page_count) { - /* Insert app heap before __heap_base */ - aux_heap_base = module->aux_heap_base; - bytes_of_last_page = aux_heap_base % num_bytes_per_page; - if (bytes_of_last_page == 0) - bytes_of_last_page = num_bytes_per_page; - bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; - inc_page_count = - (heap_size - bytes_to_page_end + num_bytes_per_page - 1) - / num_bytes_per_page; - heap_offset = aux_heap_base; - aux_heap_base += heap_size; - - bytes_of_last_page = aux_heap_base % num_bytes_per_page; - if (bytes_of_last_page == 0) - bytes_of_last_page = num_bytes_per_page; - bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; - if (bytes_to_page_end < 1 * BH_KB) { - aux_heap_base += 1 * BH_KB; - inc_page_count++; - } - - /* Adjust __heap_base global value */ - if (aux_heap_base_global_data == NULL) { - set_error_buf( - error_buf, error_buf_size, - "auxiliary heap base global data should not be NULL"); - return NULL; - } #if WASM_ENABLE_MEMORY64 != 0 - if (memory->is_memory64) { - /* For memory64, the global value should be i64 */ - *(uint64 *)aux_heap_base_global_data = aux_heap_base; - } - else -#endif - { - /* For memory32, the global value should be i32 */ - *(uint32 *)aux_heap_base_global_data = - (uint32)aux_heap_base; - } - LOG_VERBOSE("Reset __heap_base global to %" PRIu64, - aux_heap_base); - } - else { - /* Insert app heap before new page */ - inc_page_count = - (heap_size + num_bytes_per_page - 1) / num_bytes_per_page; - heap_offset = (uint64)num_bytes_per_page * init_page_count; - heap_size = (uint64)num_bytes_per_page * inc_page_count; - if (heap_size > 0) - heap_size -= 1 * BH_KB; + if (memory->is_memory64) { + /* For memory64, the global value should be i64 */ + *(uint64 *)aux_heap_base_global_data = aux_heap_base; } - init_page_count += inc_page_count; - max_page_count += inc_page_count; - if (init_page_count > default_max_pages) { - set_error_buf(error_buf, error_buf_size, - "failed to insert app heap into linear memory, " - "try using `--heap-size=0` option"); - return NULL; + else +#endif + { + /* For memory32, the global value should be i32 */ + *(uint32 *)aux_heap_base_global_data = (uint32)aux_heap_base; } - if (max_page_count > default_max_pages) - max_page_count = default_max_pages; + LOG_VERBOSE("Reset __heap_base global to %" PRIu64, aux_heap_base); + } + else { + /* Insert app heap before new page */ + inc_page_count = + (heap_size + num_bytes_per_page - 1) / num_bytes_per_page; + heap_offset = (uint64)num_bytes_per_page * init_page_count; + heap_size = (uint64)num_bytes_per_page * inc_page_count; + if (heap_size > 0) + heap_size -= 1 * BH_KB; + } + init_page_count += inc_page_count; + max_page_count += inc_page_count; + if (init_page_count > default_max_pages) { + set_error_buf(error_buf, error_buf_size, + "failed to insert app heap into linear memory, " + "try using `--heap-size=0` option"); + return NULL; } + if (max_page_count > default_max_pages) + max_page_count = default_max_pages; } LOG_VERBOSE("Memory instantiate:"); @@ -1027,15 +1023,10 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, LOG_VERBOSE(" data offset: %" PRIu64 ", stack size: %d", module->aux_data_end, module->aux_stack_size); -#ifndef NDEBUG - { - uint64 max_memory_data_size = - (uint64)num_bytes_per_page * max_page_count; - bh_assert(max_memory_data_size - <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); - (void)max_memory_data_size; - } -#endif + uint64 max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; + bh_assert(max_memory_data_size + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); + (void)max_memory_data_size; uint64 memory_data_size = 0; if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory, @@ -1068,8 +1059,6 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, memory->heap_handle = runtime_malloc((uint64)heap_struct_size, error_buf, error_buf_size); if (!memory->heap_handle) { - set_error_buf(error_buf, error_buf_size, - "allocate app heap failed"); goto fail1; } @@ -1117,6 +1106,7 @@ aot_lookup_memory(AOTModuleInstance *module_inst, char const *name) return module_inst->export_memories[i].memory; return NULL; #else + (void)name; (void)module_inst->export_memories; return aot_get_default_memory(module_inst); #endif @@ -1179,6 +1169,7 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, mem_index++, memory++) { AOTImportMemory *memory_type = module->import_memories + mem_index; +#if WASM_ENABLE_MULTI_MODULE == 0 const WASMExternInstance *extern_inst = wasm_runtime_get_extern_instance(imports, import_count, WASM_IMPORT_EXPORT_KIND_MEMORY, @@ -1209,6 +1200,22 @@ memories_instantiate(const AOTModule *module, AOTModuleInstance *module_inst, module_inst->memories[mem_index] = extern_inst->u.memory; bh_memcpy_s(memory, sizeof(AOTMemoryInstance), extern_inst->u.memory, sizeof(AOTMemoryInstance)); +#else + uint32 max_page_count = wasm_runtime_get_max_mem( + max_memory_pages, memory_type->mem_type.init_page_count, + memory_type->mem_type.max_page_count); + + module_inst->memories[mem_index] = memory_instantiate( + module, parent, memory, parent ? parent->memories[mem_index] : NULL, + memory_type->mem_type.num_bytes_per_page, + memory_type->mem_type.init_page_count, max_page_count, + /* only inst->memories[0] will have a app heap */ + mem_index == 0 ? heap_size : 0, memory_type->mem_type.flags, + aux_heap_base_global_data, error_buf, error_buf_size); + if (!module_inst->memories[mem_index]) { + return false; + } +#endif } /* process internal memories */ @@ -1887,7 +1894,7 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, */ set_error_buf(error_buf, error_buf_size, "imports is NULL while module has imports"); - return NULL; + // return NULL; } #endif @@ -3609,6 +3616,9 @@ static void const_string_node_size_cb(void *key, void *value, void *p_const_string_size) { uint32 const_string_size = 0; + + (void)key; + const_string_size += bh_hash_map_get_elem_struct_size(); const_string_size += strlen((const char *)value) + 1; *(uint32 *)p_const_string_size += const_string_size; @@ -3646,7 +3656,7 @@ aot_get_module_mem_consumption(const AOTModule *module, mem_conspn->memories_size = sizeof(AOTMemory) - * (module->memory_count + moudle->import_memory_count); + * (module->memory_count + module->import_memory_count); mem_conspn->globals_size = sizeof(AOTGlobal) * module->global_count; mem_conspn->exports_size = sizeof(AOTExport) * module->export_count; @@ -5428,10 +5438,12 @@ aot_resolve_import_func(AOTModule *module, AOTImportFunc *import_func) char error_buf[128]; AOTModule *sub_module = NULL; #endif + import_func->func_ptr_linked = wasm_native_resolve_symbol( import_func->module_name, import_func->func_name, import_func->func_type, &import_func->signature, &import_func->attachment, &import_func->call_conv_raw); + #if WASM_ENABLE_MULTI_MODULE != 0 if (!import_func->func_ptr_linked) { if (!wasm_runtime_is_built_in_module(import_func->module_name)) { @@ -5454,7 +5466,10 @@ aot_resolve_import_func(AOTModule *module, AOTImportFunc *import_func) } } } +#else + (void)module; #endif + return import_func->func_ptr_linked != NULL; } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 037faa72c0..5aec6c48f7 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -570,7 +570,7 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, return NULL; } - if (!(memories[mem_index++] = wasm_lookup_memory( + if (!(memories[mem_index] = wasm_lookup_memory( module_inst_linked, memory_type->field_name))) { set_error_buf(error_buf, error_buf_size, "unknown memory"); memories_deinstantiate(module_inst, memories, memory_count); @@ -598,7 +598,6 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, memories_deinstantiate(module_inst, memories, memory_count); return NULL; } - mem_index++; } #else @@ -609,7 +608,7 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, if (!extern_inst) { LOG_ERROR("missing a import memory(%s, %s)", memory_type->module_name, memory_type->field_name); - return false; + return NULL; } /* just in case */ @@ -618,7 +617,7 @@ memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst, LOG_ERROR( "mismatched import memory name: expect \"%s\", got \"%s\"", memory_type->field_name, extern_inst->field_name); - return false; + return NULL; } #endif /* @@ -2068,6 +2067,7 @@ check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf, for (i = 0; i < module->import_memory_count; i++) { WASMMemoryImport *type = &((module->import_memories + i)->u.memory); WASMMemoryInstance *memory = module_inst->memories[i]; + (void)memory; if ( #if WASM_ENABLE_MULTI_MODULE != 0 !wasm_runtime_is_built_in_module(type->module_name) diff --git a/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c b/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c index 6d057a6a18..7dd1d148aa 100644 --- a/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c +++ b/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c @@ -82,6 +82,7 @@ wasi_ctx_get_curfds(wasi_ctx_t wasi_ctx) static inline struct argv_environ_values * wasi_ctx_get_argv_environ(wasm_module_inst_t module_inst, wasi_ctx_t wasi_ctx) { + (void)module_inst; if (!wasi_ctx) return NULL; return wasi_ctx->argv_environ; @@ -2171,6 +2172,8 @@ wasi_sock_send(wasm_exec_env_t exec_env, wasi_fd_t sock, wasi_errno_t err; size_t send_bytes = 0; + (void)si_flags; + if (!wasi_ctx) { return __WASI_EINVAL; } @@ -2241,6 +2244,8 @@ wasi_sock_shutdown(wasm_exec_env_t exec_env, wasi_fd_t sock, wasi_sdflags_t how) wasi_ctx_t wasi_ctx = get_wasi_ctx(module_inst); struct fd_table *curfds = wasi_ctx_get_curfds(wasi_ctx); + (void)how; + if (!wasi_ctx) return __WASI_EINVAL; From 4f003327bfeba4ce9519c3752f21833eac1ef036 Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Mon, 18 Nov 2024 07:58:01 +0000 Subject: [PATCH 11/14] Refactor aot_lookup_memory function to simplify memory lookup logic and remove unused code --- core/iwasm/aot/aot_runtime.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index d623ec5a3c..f296c519e3 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1099,17 +1099,10 @@ memory_instantiate(const AOTModule *module, AOTModuleInstance *parent, AOTMemoryInstance * aot_lookup_memory(AOTModuleInstance *module_inst, char const *name) { -#if WASM_ENABLE_MULTI_MEMORY != 0 - uint32 i; - for (i = 0; i < module_inst->export_memory_count; i++) + for (uint32 i = 0; i < module_inst->export_memory_count; i++) if (!strcmp(module_inst->export_memories[i].name, name)) return module_inst->export_memories[i].memory; return NULL; -#else - (void)name; - (void)module_inst->export_memories; - return aot_get_default_memory(module_inst); -#endif } /* From a821adfe7b9aa18857271fa0472e17a5f0e8b6c5 Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Mon, 18 Nov 2024 08:15:27 +0000 Subject: [PATCH 12/14] Add documentation for imports in InstantiationArgs structure --- core/iwasm/include/wasm_export.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index ffcd3ece69..35e6ea8697 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -280,6 +280,17 @@ typedef struct InstantiationArgs { uint32_t default_stack_size; uint32_t host_managed_heap_size; uint32_t max_memory_pages; + /* + * Callers needs to create a list of wasm_extern_inst_t that follows the + * sequence specified by the import section. + * + * The usual way to find out the sequence is by calling + * wasm_runtime_get_import_type(). + * + * The runtime will complete the import list with known elements, + * such as functions from wasi_snapshot_preview1 and registered native + * functions. The caller must take care of the parts that are not known. + */ const wasm_extern_inst_t imports; uint32_t import_count; } InstantiationArgs; From e4cde59ddab7ef70ae45ea8aa88d9826e0a3644c Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Tue, 19 Nov 2024 11:43:45 +0000 Subject: [PATCH 13/14] Fix memory allocation check and update documentation for import memory structure --- core/iwasm/compilation/aot.c | 2 +- core/iwasm/compilation/aot_emit_aot_file.c | 6 ++++-- core/iwasm/include/wasm_export.h | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/iwasm/compilation/aot.c b/core/iwasm/compilation/aot.c index 9c05f72b51..f42c27488d 100644 --- a/core/iwasm/compilation/aot.c +++ b/core/iwasm/compilation/aot.c @@ -47,7 +47,7 @@ aot_create_import_memories(const WASMModule *module, uint32 import_memory_count) } import_memories = wasm_runtime_malloc((uint32)size); - if (!module->import_memories) { + if (!import_memories) { aot_set_last_error("allocate memory failed."); return NULL; } diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 876f3873b8..ffb7efb287 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -780,12 +780,12 @@ get_init_data_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data, * ------------------------------------------------------- * | u32 import_memory_count * ------------------------------------------------------- - * | | u8 module_name - * | | u8 field_name * | | u32 flags * | AOTImportMemory[import_memory_count] | u32 num_bytes_per_page * | | u32 init_page_count * | | u32 max_page_count + * | | str module_name + * | | str field_name * ------------------------------------------------------- * | padding * ------------------------------------------------------- @@ -800,6 +800,8 @@ get_init_data_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data, * ------------------------------------------------------- * | u32 mem_init_data_count * ------------------------------------------------------- + * | | u32 is_passive/placeholder + * | | u32 memory_index/placeholder * | | u32 init expr type * | AOTMemInitData[mem_init_data_count] | u32 init expr value * | | u32 byte count diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 35e6ea8697..9d4d0b751e 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -281,15 +281,15 @@ typedef struct InstantiationArgs { uint32_t host_managed_heap_size; uint32_t max_memory_pages; /* - * Callers needs to create a list of wasm_extern_inst_t that follows the + * Callers need to create a list of wasm_extern_inst_t that follows the * sequence specified by the import section. * - * The usual way to find out the sequence is by calling + * The usual way to determine the sequence is by calling * wasm_runtime_get_import_type(). * * The runtime will complete the import list with known elements, * such as functions from wasi_snapshot_preview1 and registered native - * functions. The caller must take care of the parts that are not known. + * functions. The caller must handle the parts that are not known. */ const wasm_extern_inst_t imports; uint32_t import_count; From 01ac337174f439bc52f497afb318fa4bd3838eb5 Mon Sep 17 00:00:00 2001 From: "liang.he@intel.com" Date: Wed, 20 Nov 2024 02:42:22 +0000 Subject: [PATCH 14/14] Align memory size and offset calculations to 4-byte boundaries for import memories --- core/iwasm/compilation/aot_emit_aot_file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index ffb7efb287..a45e51f6a5 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -193,7 +193,7 @@ get_import_memory_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) /* AOTImportMemory[] */ for (i = 0, import_memory = comp_data->import_memories; i < comp_data->import_memory_count; i++, import_memory++) { - size = align_uint(size, 2); + size = align_uint(size, 4); /* u32 * 4 */ size += (uint32)sizeof(uint32) * 4; @@ -1806,7 +1806,7 @@ aot_emit_import_memory_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, for (i = 0, import_memory = comp_data->import_memories; i < comp_data->import_memory_count; i++, import_memory++) { - offset = align_uint(offset, 2); + offset = align_uint(offset, 4); EMIT_U32(import_memory->mem_type.flags); EMIT_U32(import_memory->mem_type.num_bytes_per_page);