diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index 26a9680e92f1..d2cc58c13e9d 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -68,6 +68,7 @@ CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_MIN=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y CONFIG_TIMING_FUNCTIONS=y CONFIG_WATCHDOG=y diff --git a/app/boards/intel_adsp_ace20_lnl.conf b/app/boards/intel_adsp_ace20_lnl.conf index a8888c9557d6..6aa76f72f62b 100644 --- a/app/boards/intel_adsp_ace20_lnl.conf +++ b/app/boards/intel_adsp_ace20_lnl.conf @@ -52,6 +52,7 @@ CONFIG_COUNTER=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y CONFIG_TIMING_FUNCTIONS=y diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 7af406131d76..4aa8a5bf122d 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -53,6 +53,7 @@ CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_L3_HEAP=y CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y # Zephyr / device drivers diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index c33c0e2127bb..8da2738668a0 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -48,6 +48,7 @@ CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_L3_HEAP=y CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y # Zephyr / device drivers diff --git a/posix/include/rtos/alloc.h b/posix/include/rtos/alloc.h index d64ea319f48e..1ceaef3e8b8f 100644 --- a/posix/include/rtos/alloc.h +++ b/posix/include/rtos/alloc.h @@ -167,6 +167,8 @@ int rstrlen(const char *s); */ int rstrcmp(const char *s1, const char *s2); +static inline void l3_heap_save(void) {} + /** @}*/ #endif /* __SOF_LIB_ALLOC_H__ */ diff --git a/src/include/ipc4/notification.h b/src/include/ipc4/notification.h index 0581159d90c7..207dbfb7139c 100644 --- a/src/include/ipc4/notification.h +++ b/src/include/ipc4/notification.h @@ -114,6 +114,8 @@ enum sof_ipc4_resource_type { (((SOF_IPC4_NOTIFY_FW_READY) << (SOF_IPC4_GLB_NOTIFY_TYPE_SHIFT)) |\ ((SOF_IPC4_GLB_NOTIFICATION) << (SOF_IPC4_GLB_NOTIFY_MSG_TYPE_SHIFT))) +#define SOF_IPC4_FW_READY_LIB_RESTORED BIT(15) + #define SOF_IPC4_NOTIF_HEADER(notif_type) \ ((notif_type) << (SOF_IPC4_GLB_NOTIFY_TYPE_SHIFT) | \ ((SOF_IPC4_GLB_NOTIFICATION) << (SOF_IPC4_GLB_NOTIFY_MSG_TYPE_SHIFT))) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index cc0b73e0d7e7..24e5b7bddb52 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -67,6 +67,7 @@ #if CONFIG_LIBRARY_AUTH_SUPPORT #include #endif +#include #define LIB_MANAGER_MAX_LIBS 16 #define LIB_MANAGER_LIB_ID_SHIFT 12 @@ -125,11 +126,8 @@ struct ext_library { struct ipc_lib_msg *lib_notif_pool; uint32_t lib_notif_count; + /* Only needed from SOF_IPC4_GLB_LOAD_LIBRARY_PREPARE to SOF_IPC4_GLB_LOAD_LIBRARY */ void *runtime_data; -#if CONFIG_LIBRARY_AUTH_SUPPORT - struct auth_api_ctx auth_ctx; - void *auth_buffer; -#endif }; /* lib manager context, used by lib_notification */ @@ -187,6 +185,7 @@ int lib_manager_register_module(const uint32_t component_id); const struct sof_man_fw_desc *lib_manager_get_library_manifest(int module_id); struct processing_module; +struct comp_ipc_config; /* * \brief Allocate module * diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index 7394e62725cf..44928c5a124f 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -38,4 +38,12 @@ bool comp_is_llext(struct comp_dev *comp); #define comp_is_llext(comp) false #endif +#if CONFIG_LLEXT_EXPERIMENTAL && !CONFIG_ADSP_IMR_CONTEXT_SAVE +int llext_manager_store_to_dram(void); +int llext_manager_restore_from_dram(void); +#else +#define llext_manager_store_to_dram() 0 +#define llext_manager_restore_from_dram() -ENOSYS +#endif + #endif diff --git a/src/ipc/ipc4/handler.c b/src/ipc/ipc4/handler.c index d2199be999ce..16156a29b6db 100644 --- a/src/ipc/ipc4/handler.c +++ b/src/ipc/ipc4/handler.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1485,6 +1486,17 @@ __cold static int ipc4_module_process_dx(struct ipc4_message_request *ipc4) return IPC4_BUSY; } +#if !CONFIG_ADSP_IMR_CONTEXT_SAVE + ret = llext_manager_store_to_dram(); + if (ret < 0) + ipc_cmd_err(&ipc_tr, "Error %d saving LLEXT context. Resume might fail.", + ret); + +#if CONFIG_L3_HEAP + l3_heap_save(); +#endif +#endif + #if defined(CONFIG_PM) ipc_get()->task_mask |= IPC_TASK_POWERDOWN; #endif diff --git a/src/library_manager/CMakeLists.txt b/src/library_manager/CMakeLists.txt index 31e2fc4a24be..e69ca4cca30c 100644 --- a/src/library_manager/CMakeLists.txt +++ b/src/library_manager/CMakeLists.txt @@ -5,5 +5,8 @@ if(CONFIG_LIBRARY_MANAGER) if (CONFIG_MM_DRV AND CONFIG_LLEXT) add_local_sources(sof llext_manager.c) + if(CONFIG_LLEXT_EXPERIMENTAL AND NOT CONFIG_ADSP_IMR_CONTEXT_SAVE) + add_local_sources_ifdef(CONFIG_L3_HEAP sof llext_manager_dram.c) + endif() endif() endif() diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 7040e283db03..25d719aac108 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -36,6 +36,8 @@ #if CONFIG_LIBRARY_AUTH_SUPPORT #include +#else +struct auth_api_ctx; #endif #include @@ -60,49 +62,40 @@ struct lib_manager_dma_ext { static struct ext_library loader_ext_lib; #if CONFIG_LIBRARY_AUTH_SUPPORT -static int lib_manager_auth_init(void) +static int lib_manager_auth_init(struct auth_api_ctx *auth_ctx, void **auth_buffer) { - struct ext_library *ext_lib = ext_lib_get(); int ret; if (auth_api_version().major != AUTH_API_VERSION_MAJOR) return -EINVAL; - ext_lib->auth_buffer = rballoc_align(0, SOF_MEM_CAPS_RAM, - AUTH_SCRATCH_BUFF_SZ, CONFIG_MM_DRV_PAGE_SIZE); - if (!ext_lib->auth_buffer) + *auth_buffer = rballoc_align(0, SOF_MEM_CAPS_RAM, + AUTH_SCRATCH_BUFF_SZ, CONFIG_MM_DRV_PAGE_SIZE); + if (!*auth_buffer) return -ENOMEM; - ret = auth_api_init(&ext_lib->auth_ctx, ext_lib->auth_buffer, - AUTH_SCRATCH_BUFF_SZ, IMG_TYPE_LIB); + ret = auth_api_init(auth_ctx, *auth_buffer, AUTH_SCRATCH_BUFF_SZ, IMG_TYPE_LIB); if (ret != 0) { tr_err(&lib_manager_tr, "auth_api_init() failed with error: %d", ret); - rfree(ext_lib->auth_buffer); - ret = -EACCES; + rfree(*auth_buffer); + return -EACCES; } - return ret; + return 0; } -static void lib_manager_auth_deinit(void) +static void lib_manager_auth_deinit(struct auth_api_ctx *auth_ctx, void *auth_buffer) { - struct ext_library *ext_lib = ext_lib_get(); - - if (ext_lib->auth_buffer) - memset(ext_lib->auth_buffer, 0, AUTH_SCRATCH_BUFF_SZ); - - rfree(ext_lib->auth_buffer); - ext_lib->auth_buffer = NULL; - memset(&ext_lib->auth_ctx, 0, sizeof(struct auth_api_ctx)); + ARG_UNUSED(auth_ctx); + rfree(auth_buffer); } -static int lib_manager_auth_proc(const void *buffer_data, - size_t buffer_size, enum auth_phase phase) +static int lib_manager_auth_proc(const void *buffer_data, size_t buffer_size, + enum auth_phase phase, struct auth_api_ctx *auth_ctx) { - struct ext_library *ext_lib = ext_lib_get(); int ret; - ret = auth_api_init_auth_proc(&ext_lib->auth_ctx, buffer_data, buffer_size, phase); + ret = auth_api_init_auth_proc(auth_ctx, buffer_data, buffer_size, phase); if (ret != 0) { tr_err(&lib_manager_tr, "auth_api_init_auth_proc() failed with error: %d", ret); @@ -110,10 +103,10 @@ static int lib_manager_auth_proc(const void *buffer_data, } /* The auth_api_busy() will timeouts internally in case of failure */ - while (auth_api_busy(&ext_lib->auth_ctx)) + while (auth_api_busy(auth_ctx)) ; - ret = auth_api_result(&ext_lib->auth_ctx); + ret = auth_api_result(auth_ctx); if (ret != AUTH_IMAGE_TRUSTED) { tr_err(&lib_manager_tr, "Untrusted library!"); @@ -121,7 +114,7 @@ static int lib_manager_auth_proc(const void *buffer_data, } if (phase == AUTH_PHASE_LAST) - auth_api_cleanup(&ext_lib->auth_ctx); + auth_api_cleanup(auth_ctx); return 0; } @@ -819,7 +812,7 @@ static void __sparse_cache *lib_manager_allocate_store_mem(uint32_t size, static int lib_manager_store_library(struct lib_manager_dma_ext *dma_ext, const void __sparse_cache *man_buffer, - uint32_t lib_id) + uint32_t lib_id, struct auth_api_ctx *auth_ctx) { void __sparse_cache *library_base_address; const struct sof_man_fw_desc *man_desc = (struct sof_man_fw_desc *) @@ -850,7 +843,7 @@ static int lib_manager_store_library(struct lib_manager_dma_ext *dma_ext, #if CONFIG_LIBRARY_AUTH_SUPPORT /* AUTH_PHASE_FIRST - checks library manifest only. */ ret = lib_manager_auth_proc((__sparse_force const void *)man_buffer, - MAN_MAX_SIZE_V1_8, AUTH_PHASE_FIRST); + MAN_MAX_SIZE_V1_8, AUTH_PHASE_FIRST, auth_ctx); if (ret < 0) { rfree((__sparse_force void *)library_base_address); return ret; @@ -872,7 +865,7 @@ static int lib_manager_store_library(struct lib_manager_dma_ext *dma_ext, #if CONFIG_LIBRARY_AUTH_SUPPORT /* AUTH_PHASE_LAST - do final library authentication checks */ ret = lib_manager_auth_proc((__sparse_force void *)library_base_address, - preload_size - MAN_MAX_SIZE_V1_8, AUTH_PHASE_LAST); + preload_size - MAN_MAX_SIZE_V1_8, AUTH_PHASE_LAST, auth_ctx); if (ret < 0) { rfree((__sparse_force void *)library_base_address); return ret; @@ -1003,16 +996,19 @@ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type) goto stop_dma; #if CONFIG_LIBRARY_AUTH_SUPPORT + struct auth_api_ctx auth_ctx; + void *auth_buffer; + /* Initialize authentication support */ - ret = lib_manager_auth_init(); + ret = lib_manager_auth_init(&auth_ctx, &auth_buffer); if (ret < 0) goto stop_dma; -#endif /* CONFIG_LIBRARY_AUTH_SUPPORT */ - ret = lib_manager_store_library(dma_ext, man_tmp_buffer, lib_id); + ret = lib_manager_store_library(dma_ext, man_tmp_buffer, lib_id, &auth_ctx); -#if CONFIG_LIBRARY_AUTH_SUPPORT - lib_manager_auth_deinit(); + lib_manager_auth_deinit(&auth_ctx, auth_buffer); +#else + ret = lib_manager_store_library(dma_ext, man_tmp_buffer, lib_id, NULL); #endif /* CONFIG_LIBRARY_AUTH_SUPPORT */ stop_dma: @@ -1034,7 +1030,7 @@ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type) uint32_t module_id = lib_id << LIB_MANAGER_LIB_ID_SHIFT; const struct sof_man_module *mod = lib_manager_get_module_manifest(module_id); - if (module_is_llext(mod) && !ret) + if (!ret && module_is_llext(mod)) /* Auxiliary LLEXT libraries need to be linked upon loading */ ret = llext_manager_add_library(module_id); @@ -1043,7 +1039,7 @@ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type) #endif if (!ret) - tr_info(&ipc_tr, "loaded library id: %u", lib_id); + tr_info(&lib_manager_tr, "loaded library id: %u", lib_id); return ret; } diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 213d1722c5ad..0272b0280f8e 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -113,8 +113,12 @@ static int llext_manager_load_data_from_storage(const struct llext_loader *ldr, const elf_shdr_t *shdr; enum llext_mem s_region = LLEXT_MEM_COUNT; size_t s_offset = 0; + int ret = llext_get_section_info(ldr, ext, i, &shdr, &s_region, &s_offset); - llext_get_section_info(ldr, ext, i, &shdr, &s_region, &s_offset); + if (ret < 0) { + tr_err(lib_manager_tr, "no section info: %d", ret); + continue; + } /* skip sections not in the requested region */ if (s_region != region) @@ -528,7 +532,7 @@ static int llext_manager_link_single(uint32_t module_id, const struct sof_man_fw sizeof(struct llext_buf_loader)); if (!mctx->ebl) { tr_err(&lib_manager_tr, "loader alloc failed"); - return 0; + return -ENOMEM; } uint8_t *dram_base = (uint8_t *)desc - SOF_MAN_ELF_TEXT_OFFSET; diff --git a/src/library_manager/llext_manager_dram.c b/src/library_manager/llext_manager_dram.c new file mode 100644 index 000000000000..7629da427462 --- /dev/null +++ b/src/library_manager/llext_manager_dram.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2025 Intel Corporation. + +#include + +#include +#include + +#include +#include +#include +#include + +LOG_MODULE_DECLARE(lib_manager, CONFIG_SOF_LOG_LEVEL); + +struct lib_manager_dram_storage { + struct ext_library ext_lib; + struct lib_manager_mod_ctx *ctx; + struct lib_manager_module *mod; + struct llext *llext; + struct llext_buf_loader *bldr; + struct llext_elf_sect_map *sect; + struct llext_symbol *sym; + unsigned int n_llext; +}; + +/* + * This holds the complete LLEXT manager context in DRAM over DSP shut down to + * be restored during the next boot + */ +__imrdata static struct lib_manager_dram_storage lib_manager_dram; + +/* Store LLEXT manager context in DRAM to be restored during the next boot. */ +int llext_manager_store_to_dram(void) +{ + struct ext_library *_ext_lib = ext_lib_get(); + unsigned int i, j, k, l, n_lib, n_mod, n_llext, n_sect, n_sym; + size_t buf_size; + + if (lib_manager_dram.n_llext) { + tr_err(&lib_manager_tr, "context already saved"); + return 0; + } + + /* + * Count libraries, modules, instantiated extensions, sections and exported + * symbols in them. Allocate a buffer of required size. + */ + lib_manager_dram.ext_lib = *_ext_lib; + for (i = 0, n_lib = 0, n_mod = 0, n_llext = 0, n_sect = 0, n_sym = 0; + i < ARRAY_SIZE(_ext_lib->desc); i++) + if (_ext_lib->desc[i]) { + n_lib++; + n_mod += _ext_lib->desc[i]->n_mod; + for (k = 0; k < _ext_lib->desc[i]->n_mod; k++) + if (_ext_lib->desc[i]->mod[k].ebl) { + n_llext++; + n_sect += _ext_lib->desc[i]->mod[k].llext->sect_cnt; + n_sym += _ext_lib->desc[i]->mod[k].llext->exp_tab.sym_cnt; + tr_dbg(&lib_manager_tr, "add %u exported syms", + _ext_lib->desc[i]->mod[k].llext->exp_tab.sym_cnt); + } + } + + buf_size = sizeof(lib_manager_dram.ctx[0]) * n_lib + + sizeof(lib_manager_dram.mod[0]) * n_mod + + sizeof(lib_manager_dram.sect[0]) * n_sect + + sizeof(lib_manager_dram.sym[0]) * n_sym + + (sizeof(lib_manager_dram.llext[0]) + sizeof(lib_manager_dram.bldr[0])) * n_llext; + + lib_manager_dram.ctx = rmalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_L3, + buf_size); + if (!lib_manager_dram.ctx) + return -ENOMEM; + + /* Save pointers to buffer parts, holding parts of the context */ + lib_manager_dram.mod = (struct lib_manager_module *)(lib_manager_dram.ctx + n_lib); + lib_manager_dram.sect = (struct llext_elf_sect_map *)(lib_manager_dram.mod + n_mod); + lib_manager_dram.llext = (struct llext *)(lib_manager_dram.sect + n_sect); + lib_manager_dram.bldr = (struct llext_buf_loader *)(lib_manager_dram.llext + n_llext); + lib_manager_dram.sym = (struct llext_symbol *)(lib_manager_dram.bldr + n_llext); + + tr_dbg(&lib_manager_tr, "backup %u libs of %u modules with %u LLEXT with %u sections", + n_lib, n_mod, n_llext, n_sect); + + tr_dbg(&lib_manager_tr, "backup %p to %p, mod %p, loader %p", + lib_manager_dram.ctx, (void *)((uint8_t *)lib_manager_dram.ctx + buf_size), + lib_manager_dram.mod, lib_manager_dram.bldr); + + /* Walk all libraries */ + for (i = 0, j = 0, l = 0, n_mod = 0, n_sect = 0, n_sym = 0; + i < ARRAY_SIZE(_ext_lib->desc); i++) { + if (!_ext_lib->desc[i]) + continue; + + struct lib_manager_module *mod = _ext_lib->desc[i]->mod; + + /* Copy all modules in each library */ + lib_manager_dram.ctx[j++] = *_ext_lib->desc[i]; + memcpy(lib_manager_dram.mod + n_mod, mod, + sizeof(lib_manager_dram.mod[0]) * _ext_lib->desc[i]->n_mod); + tr_dbg(&lib_manager_tr, "lib %u base %p", j - 1, + lib_manager_dram.ctx[j - 1].base_addr); + n_mod += _ext_lib->desc[i]->n_mod; + + /* + * Copy instantiated extensions. Note that only modules, that + * were used, have their LLEXT context instantiated. + */ + for (k = 0; k < _ext_lib->desc[i]->n_mod; k++) { + if (!mod[k].llext) + continue; + + tr_dbg(&lib_manager_tr, "mod %u of %u sections", k, + mod[k].llext->sect_cnt); + + /* Copy the extension and the loader */ + lib_manager_dram.llext[l] = *mod[k].llext; + lib_manager_dram.bldr[l] = *mod[k].ebl; + + /* Copy the section map */ + memcpy(lib_manager_dram.sect + n_sect, mod[k].ebl->loader.sect_map, + mod[k].llext->sect_cnt * sizeof(lib_manager_dram.sect[0])); + n_sect += mod[k].llext->sect_cnt; + + /* Copy exported symbols */ + if (mod[k].llext->exp_tab.sym_cnt) { + memcpy(lib_manager_dram.sym + n_sym, mod[k].llext->exp_tab.syms, + mod[k].llext->exp_tab.sym_cnt * + sizeof(lib_manager_dram.sym[0])); + lib_manager_dram.llext[l].exp_tab.syms = lib_manager_dram.sym + + n_sym; + n_sym += mod[k].llext->exp_tab.sym_cnt; + } + + l++; + } + } + + /* Also flatten dependency lists */ + int ret = llext_relink_dependency(lib_manager_dram.llext, n_llext); + + if (ret < 0) { + tr_err(&lib_manager_tr, "Inconsistent dependencies!"); + return ret; + } + + lib_manager_dram.n_llext = n_llext; + /* Make sure, that the data is actually in the DRAM, not just in data cache */ + dcache_writeback_region((__sparse_force void __sparse_cache *)&lib_manager_dram, + sizeof(lib_manager_dram)); + dcache_writeback_region((__sparse_force void __sparse_cache *)lib_manager_dram.ctx, + buf_size); + + return 0; +} + +int llext_manager_restore_from_dram(void) +{ + lib_manager_init(); + + struct ext_library *_ext_lib = ext_lib_get(); + unsigned int i, j, k, n_mod, n_llext, n_sect, n_sym; + + if (!lib_manager_dram.n_llext || !lib_manager_dram.ctx) { + tr_dbg(&lib_manager_tr, "No modules saved"); + dcache_writeback_region((__sparse_force void __sparse_cache *)&lib_manager_dram, + sizeof(lib_manager_dram)); + return 0; + } + + /* arrays of pointers for llext_restore() */ + void **ptr_array = rmalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, + sizeof(*ptr_array) * lib_manager_dram.n_llext * 2); + + if (!ptr_array) + return -ENOMEM; + + struct llext_loader **ldr = (struct llext_loader **)ptr_array; + struct llext **llext = (struct llext **)(ptr_array + lib_manager_dram.n_llext); + + *_ext_lib = lib_manager_dram.ext_lib; + + /* The external loop walks all the libraries */ + for (i = 0, j = 0, n_mod = 0, n_llext = 0, n_sect = 0, n_sym = 0; + i < ARRAY_SIZE(_ext_lib->desc); i++) { + if (!lib_manager_dram.ext_lib.desc[i]) { + _ext_lib->desc[i] = NULL; + continue; + } + + /* Panics on failure - use the same zone as during the first boot */ + struct lib_manager_mod_ctx *ctx = rmalloc(SOF_MEM_ZONE_SYS, SOF_MEM_FLAG_COHERENT, + SOF_MEM_CAPS_RAM, sizeof(*ctx)); + + /* Restore the library context */ + *ctx = lib_manager_dram.ctx[j++]; + + /* Allocate and restore all the modules in the library */ + struct lib_manager_module *mod = rmalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, + ctx->n_mod * sizeof(ctx->mod[0])); + + if (!mod) { + tr_err(&lib_manager_tr, "module allocation failure"); + goto nomem; + } + tr_dbg(&lib_manager_tr, "%u modules alloc %p base %p copy %#zx", + ctx->n_mod, (void *)mod, ctx->base_addr, ctx->n_mod * sizeof(ctx->mod[0])); + + memcpy(mod, lib_manager_dram.mod + n_mod, sizeof(mod[0]) * ctx->n_mod); + n_mod += ctx->n_mod; + ctx->mod = mod; + + /* Second level: enumerate modules in each library */ + for (k = 0; k < ctx->n_mod; k++) { + if (!mod[k].llext) + /* Not instantiated - nothing to restore */ + continue; + + /* Loaders are supplied by the caller */ + struct llext_buf_loader *bldr = rmalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(*bldr)); + + if (!bldr) { + tr_err(&lib_manager_tr, "loader allocation failure"); + goto nomem; + } + + /* Extensions have to be restored by Zephyr, collect pointers first */ + llext[n_llext] = lib_manager_dram.llext + n_llext; + + *bldr = lib_manager_dram.bldr[n_llext]; + + bldr->loader.sect_map = lib_manager_dram.sect + n_sect; + + n_sect += llext[n_llext]->sect_cnt; + if (llext[n_llext]->exp_tab.sym_cnt) { + /* + * Just a check, that we're restoring exported + * symbols correctly + */ + tr_dbg(&lib_manager_tr, "got %u exported symbols", + llext[n_llext]->exp_tab.sym_cnt); + + if (llext[n_llext]->exp_tab.syms != lib_manager_dram.sym + n_sym) { + tr_err(&lib_manager_tr, + "bug detected! pointer mismatch %p vs. %p", + (void *)llext[n_llext]->exp_tab.syms, + (void *)(lib_manager_dram.sym + n_sym)); + goto nomem; + } + + n_sym += llext[n_llext]->exp_tab.sym_cnt; + } + + mod[k].ebl = bldr; + + ldr[n_llext++] = &bldr->loader; + } + + _ext_lib->desc[i] = ctx; + } + + /* Let Zephyr restore extensions and its own internal bookkeeping */ + int ret = llext_restore(llext, ldr, lib_manager_dram.n_llext); + + if (ret < 0) { + tr_err(&lib_manager_tr, "Zephyr failed to restore: %d", ret); + goto nomem; + } + + /* Rewrite to correct LLEXT pointers, created by Zephyr */ + for (i = 0, n_llext = 0; i < ARRAY_SIZE(_ext_lib->desc); i++) { + struct lib_manager_mod_ctx *ctx = _ext_lib->desc[i]; + + if (!ctx) + continue; + + struct lib_manager_module *mod = ctx->mod; + + for (k = 0; k < ctx->n_mod; k++) { + if (mod[k].llext) + mod[k].llext = llext[n_llext++]; + } + } + + tr_info(&lib_manager_tr, "restored %u modules with %u LLEXT", n_mod, n_llext); + + rfree(lib_manager_dram.ctx); + lib_manager_dram.ctx = NULL; + lib_manager_dram.sect = NULL; + lib_manager_dram.llext = NULL; + lib_manager_dram.bldr = NULL; + lib_manager_dram.sym = NULL; + rfree(ldr); + + lib_manager_dram.n_llext = 0; + + return 0; + +nomem: + tr_err(&lib_manager_tr, "Restore failed"); + for (i = 0; i < ARRAY_SIZE(_ext_lib->desc); i++) { + struct lib_manager_mod_ctx *ctx = _ext_lib->desc[i]; + + if (!ctx) + continue; + + struct lib_manager_module *mod = ctx->mod; + + if (!mod) + continue; + + for (k = 0; k < ctx->n_mod; k++) { + if (mod[k].llext) + llext_unload(&mod[k].llext); + + if (mod[k].ebl) + rfree(mod[k].ebl); + } + + rfree(mod); + rfree(ctx); + } + + /* at least create a sane empty lib-manager context */ + memset(_ext_lib->desc, 0, sizeof(_ext_lib->desc)); + + rfree(ldr); + + return -ENOMEM; +} diff --git a/src/platform/intel/ace/platform.c b/src/platform/intel/ace/platform.c index edc36ab5dcf8..1fde2713049d 100644 --- a/src/platform/intel/ace/platform.c +++ b/src/platform/intel/ace/platform.c @@ -67,6 +67,7 @@ __cold int platform_boot_complete(uint32_t boot_message) /* get any IPC specific boot message and optional data */ ipc_boot_complete_msg(&header, 0); + header.pri |= boot_message; struct ipc_msg msg = { .header = header.pri, diff --git a/west.yml b/west.yml index 09d91a0a1453..ae60859bc0a8 100644 --- a/west.yml +++ b/west.yml @@ -43,7 +43,7 @@ manifest: - name: zephyr repo-path: zephyr - revision: 8c6b2c8c88773810d64a47ed3b6206b7d6e8290f + revision: 9075d5335596080291ebcc448819fed2110dcb9a remote: zephyrproject # Import some projects listed in zephyr/west.yml@revision diff --git a/xtos/include/rtos/alloc.h b/xtos/include/rtos/alloc.h index d64ea319f48e..1ceaef3e8b8f 100644 --- a/xtos/include/rtos/alloc.h +++ b/xtos/include/rtos/alloc.h @@ -167,6 +167,8 @@ int rstrlen(const char *s); */ int rstrcmp(const char *s1, const char *s2); +static inline void l3_heap_save(void) {} + /** @}*/ #endif /* __SOF_LIB_ALLOC_H__ */ diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index 69228ea8f9c2..18351ea44d1c 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -136,6 +136,11 @@ static inline void *rbrealloc(void *ptr, uint32_t flags, uint32_t caps, */ void rfree(void *ptr); +/** + * Save L3 heap over DSP reset + */ +void l3_heap_save(void); + /* TODO: remove - debug only - only needed for linking */ static inline void heap_trace_all(int force) {} diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index 3cfc6eb97296..3225fbce4f69 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -126,6 +126,7 @@ static struct k_heap sof_heap; #if CONFIG_L3_HEAP static struct k_heap l3_heap; +static struct k_heap l3_heap_copy __imrdata; /** * Returns the start of L3 memory heap. @@ -158,6 +159,16 @@ static inline size_t get_l3_heap_size(void) return ROUND_DOWN(IMR_L3_HEAP_SIZE, L3_MEM_PAGE_SIZE); } +void l3_heap_save(void) +{ + l3_heap_copy = l3_heap; + LOG_DBG("L3 heap copy: %p", (void *)l3_heap_copy.heap.heap); + dcache_writeback_region((__sparse_force void __sparse_cache *)&l3_heap_copy, + sizeof(l3_heap_copy)); + dcache_writeback_region((__sparse_force void __sparse_cache *)get_l3_heap_start(), + get_l3_heap_size()); +} + /** * Checks whether pointer is from L3 heap memory range. * @param ptr Pointer to memory being checked. @@ -395,7 +406,7 @@ void *rmalloc(enum mem_zone zone, uint32_t flags, uint32_t caps, size_t bytes) if (caps & SOF_MEM_CAPS_L3) { #if CONFIG_L3_HEAP heap = &l3_heap; - /* Uncached L3_HEAP should be not used */ + /* Uncached L3_HEAP should not be used */ if (!zone_is_cached(zone)) { tr_err(&zephyr_tr, "L3_HEAP available for cached zones only!"); return NULL; @@ -553,7 +564,11 @@ static int heap_init(void) sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE); #if CONFIG_L3_HEAP - sys_heap_init(&l3_heap.heap, UINT_TO_POINTER(get_l3_heap_start()), get_l3_heap_size()); + if (l3_heap_copy.heap.heap) + l3_heap = l3_heap_copy; + else + sys_heap_init(&l3_heap.heap, UINT_TO_POINTER(get_l3_heap_start()), + get_l3_heap_size()); #endif return 0; diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index c2129347241e..2e302f22c94b 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -20,6 +21,9 @@ #include #include #include +#if CONFIG_IPC_MAJOR_4 +#include +#endif /* Zephyr includes */ #include @@ -187,8 +191,25 @@ static int boot_complete(void) */ return 0; #else + unsigned int status; + int ret = llext_manager_restore_from_dram(); + + switch (ret) { +#if CONFIG_IPC_MAJOR_4 + case 0: + status = SOF_IPC4_FW_READY_LIB_RESTORED; + break; +#endif + case -ENOSYS: + status = 0; + break; + default: + status = 0; + LOG_ERR("LLEXT restore failed: %d", ret); + } + /* let host know DSP boot is complete */ - return platform_boot_complete(0); + return platform_boot_complete(status); #endif /* CONFIG_IMX93_A55 */ }