From ee61ce18f2615e8ca13dd016baf3591fa6c0e32e Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 16 May 2025 09:33:49 +0200 Subject: [PATCH 1/8] lib-manager: make authentication context local The authentication context, used by the library manager to check module signature, is fully reinitialised and released for each loaded module, no need to store it permanently. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 4 -- src/library_manager/lib_manager.c | 64 +++++++++++++++---------------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index cc0b73e0d7e7..c6a78664cbb9 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -126,10 +126,6 @@ struct ext_library { uint32_t lib_notif_count; void *runtime_data; -#if CONFIG_LIBRARY_AUTH_SUPPORT - struct auth_api_ctx auth_ctx; - void *auth_buffer; -#endif }; /* lib manager context, used by lib_notification */ diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 7040e283db03..eb5292aec1b7 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -36,6 +36,8 @@ #if CONFIG_LIBRARY_AUTH_SUPPORT #include +#else +struct auth_api_ctx; #endif #include @@ -60,49 +62,40 @@ struct lib_manager_dma_ext { static struct ext_library loader_ext_lib; #if CONFIG_LIBRARY_AUTH_SUPPORT -static int lib_manager_auth_init(void) +static int lib_manager_auth_init(struct auth_api_ctx *auth_ctx, void **auth_buffer) { - struct ext_library *ext_lib = ext_lib_get(); int ret; if (auth_api_version().major != AUTH_API_VERSION_MAJOR) return -EINVAL; - ext_lib->auth_buffer = rballoc_align(0, SOF_MEM_CAPS_RAM, - AUTH_SCRATCH_BUFF_SZ, CONFIG_MM_DRV_PAGE_SIZE); - if (!ext_lib->auth_buffer) + *auth_buffer = rballoc_align(0, SOF_MEM_CAPS_RAM, + AUTH_SCRATCH_BUFF_SZ, CONFIG_MM_DRV_PAGE_SIZE); + if (!*auth_buffer) return -ENOMEM; - ret = auth_api_init(&ext_lib->auth_ctx, ext_lib->auth_buffer, - AUTH_SCRATCH_BUFF_SZ, IMG_TYPE_LIB); + ret = auth_api_init(auth_ctx, *auth_buffer, AUTH_SCRATCH_BUFF_SZ, IMG_TYPE_LIB); if (ret != 0) { tr_err(&lib_manager_tr, "auth_api_init() failed with error: %d", ret); - rfree(ext_lib->auth_buffer); - ret = -EACCES; + rfree(*auth_buffer); + return -EACCES; } - return ret; + return 0; } -static void lib_manager_auth_deinit(void) +static void lib_manager_auth_deinit(struct auth_api_ctx *auth_ctx, void *auth_buffer) { - struct ext_library *ext_lib = ext_lib_get(); - - if (ext_lib->auth_buffer) - memset(ext_lib->auth_buffer, 0, AUTH_SCRATCH_BUFF_SZ); - - rfree(ext_lib->auth_buffer); - ext_lib->auth_buffer = NULL; - memset(&ext_lib->auth_ctx, 0, sizeof(struct auth_api_ctx)); + ARG_UNUSED(auth_ctx); + rfree(auth_buffer); } -static int lib_manager_auth_proc(const void *buffer_data, - size_t buffer_size, enum auth_phase phase) +static int lib_manager_auth_proc(const void *buffer_data, size_t buffer_size, + enum auth_phase phase, struct auth_api_ctx *auth_ctx) { - struct ext_library *ext_lib = ext_lib_get(); int ret; - ret = auth_api_init_auth_proc(&ext_lib->auth_ctx, buffer_data, buffer_size, phase); + ret = auth_api_init_auth_proc(auth_ctx, buffer_data, buffer_size, phase); if (ret != 0) { tr_err(&lib_manager_tr, "auth_api_init_auth_proc() failed with error: %d", ret); @@ -110,10 +103,10 @@ static int lib_manager_auth_proc(const void *buffer_data, } /* The auth_api_busy() will timeouts internally in case of failure */ - while (auth_api_busy(&ext_lib->auth_ctx)) + while (auth_api_busy(auth_ctx)) ; - ret = auth_api_result(&ext_lib->auth_ctx); + ret = auth_api_result(auth_ctx); if (ret != AUTH_IMAGE_TRUSTED) { tr_err(&lib_manager_tr, "Untrusted library!"); @@ -121,7 +114,7 @@ static int lib_manager_auth_proc(const void *buffer_data, } if (phase == AUTH_PHASE_LAST) - auth_api_cleanup(&ext_lib->auth_ctx); + auth_api_cleanup(auth_ctx); return 0; } @@ -819,7 +812,7 @@ static void __sparse_cache *lib_manager_allocate_store_mem(uint32_t size, static int lib_manager_store_library(struct lib_manager_dma_ext *dma_ext, const void __sparse_cache *man_buffer, - uint32_t lib_id) + uint32_t lib_id, struct auth_api_ctx *auth_ctx) { void __sparse_cache *library_base_address; const struct sof_man_fw_desc *man_desc = (struct sof_man_fw_desc *) @@ -850,7 +843,7 @@ static int lib_manager_store_library(struct lib_manager_dma_ext *dma_ext, #if CONFIG_LIBRARY_AUTH_SUPPORT /* AUTH_PHASE_FIRST - checks library manifest only. */ ret = lib_manager_auth_proc((__sparse_force const void *)man_buffer, - MAN_MAX_SIZE_V1_8, AUTH_PHASE_FIRST); + MAN_MAX_SIZE_V1_8, AUTH_PHASE_FIRST, auth_ctx); if (ret < 0) { rfree((__sparse_force void *)library_base_address); return ret; @@ -872,7 +865,7 @@ static int lib_manager_store_library(struct lib_manager_dma_ext *dma_ext, #if CONFIG_LIBRARY_AUTH_SUPPORT /* AUTH_PHASE_LAST - do final library authentication checks */ ret = lib_manager_auth_proc((__sparse_force void *)library_base_address, - preload_size - MAN_MAX_SIZE_V1_8, AUTH_PHASE_LAST); + preload_size - MAN_MAX_SIZE_V1_8, AUTH_PHASE_LAST, auth_ctx); if (ret < 0) { rfree((__sparse_force void *)library_base_address); return ret; @@ -1003,16 +996,19 @@ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type) goto stop_dma; #if CONFIG_LIBRARY_AUTH_SUPPORT + struct auth_api_ctx auth_ctx; + void *auth_buffer; + /* Initialize authentication support */ - ret = lib_manager_auth_init(); + ret = lib_manager_auth_init(&auth_ctx, &auth_buffer); if (ret < 0) goto stop_dma; -#endif /* CONFIG_LIBRARY_AUTH_SUPPORT */ - ret = lib_manager_store_library(dma_ext, man_tmp_buffer, lib_id); + ret = lib_manager_store_library(dma_ext, man_tmp_buffer, lib_id, &auth_ctx); -#if CONFIG_LIBRARY_AUTH_SUPPORT - lib_manager_auth_deinit(); + lib_manager_auth_deinit(&auth_ctx, auth_buffer); +#else + ret = lib_manager_store_library(dma_ext, man_tmp_buffer, lib_id, NULL); #endif /* CONFIG_LIBRARY_AUTH_SUPPORT */ stop_dma: From f8f3af919b9b2d8fbe05fdf6a5d56c88ef9666ba Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 23 May 2025 15:00:03 +0200 Subject: [PATCH 2/8] heap: enable saving L3 heap over DSP reset L3 heap allocates memory in DRAM. Usually this is done to preserve contents over DSP reset. This patch adds a method to do that. Signed-off-by: Guennadi Liakhovetski --- posix/include/rtos/alloc.h | 2 ++ xtos/include/rtos/alloc.h | 2 ++ zephyr/include/rtos/alloc.h | 5 +++++ zephyr/lib/alloc.c | 19 +++++++++++++++++-- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/posix/include/rtos/alloc.h b/posix/include/rtos/alloc.h index d64ea319f48e..1ceaef3e8b8f 100644 --- a/posix/include/rtos/alloc.h +++ b/posix/include/rtos/alloc.h @@ -167,6 +167,8 @@ int rstrlen(const char *s); */ int rstrcmp(const char *s1, const char *s2); +static inline void l3_heap_save(void) {} + /** @}*/ #endif /* __SOF_LIB_ALLOC_H__ */ diff --git a/xtos/include/rtos/alloc.h b/xtos/include/rtos/alloc.h index d64ea319f48e..1ceaef3e8b8f 100644 --- a/xtos/include/rtos/alloc.h +++ b/xtos/include/rtos/alloc.h @@ -167,6 +167,8 @@ int rstrlen(const char *s); */ int rstrcmp(const char *s1, const char *s2); +static inline void l3_heap_save(void) {} + /** @}*/ #endif /* __SOF_LIB_ALLOC_H__ */ diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index 69228ea8f9c2..18351ea44d1c 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -136,6 +136,11 @@ static inline void *rbrealloc(void *ptr, uint32_t flags, uint32_t caps, */ void rfree(void *ptr); +/** + * Save L3 heap over DSP reset + */ +void l3_heap_save(void); + /* TODO: remove - debug only - only needed for linking */ static inline void heap_trace_all(int force) {} diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index 3cfc6eb97296..3225fbce4f69 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -126,6 +126,7 @@ static struct k_heap sof_heap; #if CONFIG_L3_HEAP static struct k_heap l3_heap; +static struct k_heap l3_heap_copy __imrdata; /** * Returns the start of L3 memory heap. @@ -158,6 +159,16 @@ static inline size_t get_l3_heap_size(void) return ROUND_DOWN(IMR_L3_HEAP_SIZE, L3_MEM_PAGE_SIZE); } +void l3_heap_save(void) +{ + l3_heap_copy = l3_heap; + LOG_DBG("L3 heap copy: %p", (void *)l3_heap_copy.heap.heap); + dcache_writeback_region((__sparse_force void __sparse_cache *)&l3_heap_copy, + sizeof(l3_heap_copy)); + dcache_writeback_region((__sparse_force void __sparse_cache *)get_l3_heap_start(), + get_l3_heap_size()); +} + /** * Checks whether pointer is from L3 heap memory range. * @param ptr Pointer to memory being checked. @@ -395,7 +406,7 @@ void *rmalloc(enum mem_zone zone, uint32_t flags, uint32_t caps, size_t bytes) if (caps & SOF_MEM_CAPS_L3) { #if CONFIG_L3_HEAP heap = &l3_heap; - /* Uncached L3_HEAP should be not used */ + /* Uncached L3_HEAP should not be used */ if (!zone_is_cached(zone)) { tr_err(&zephyr_tr, "L3_HEAP available for cached zones only!"); return NULL; @@ -553,7 +564,11 @@ static int heap_init(void) sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE); #if CONFIG_L3_HEAP - sys_heap_init(&l3_heap.heap, UINT_TO_POINTER(get_l3_heap_start()), get_l3_heap_size()); + if (l3_heap_copy.heap.heap) + l3_heap = l3_heap_copy; + else + sys_heap_init(&l3_heap.heap, UINT_TO_POINTER(get_l3_heap_start()), + get_l3_heap_size()); #endif return 0; From 91783f3a7dafa1dff35ae4d80250893ad543b12c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 23 May 2025 16:40:42 +0200 Subject: [PATCH 3/8] library-manager: fix compilation errors in certain builds lib_manager.h uses some types without including respective headers or declaring them. Fix two such omissions. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index c6a78664cbb9..24e5b7bddb52 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -67,6 +67,7 @@ #if CONFIG_LIBRARY_AUTH_SUPPORT #include #endif +#include #define LIB_MANAGER_MAX_LIBS 16 #define LIB_MANAGER_LIB_ID_SHIFT 12 @@ -125,6 +126,7 @@ struct ext_library { struct ipc_lib_msg *lib_notif_pool; uint32_t lib_notif_count; + /* Only needed from SOF_IPC4_GLB_LOAD_LIBRARY_PREPARE to SOF_IPC4_GLB_LOAD_LIBRARY */ void *runtime_data; }; @@ -183,6 +185,7 @@ int lib_manager_register_module(const uint32_t component_id); const struct sof_man_fw_desc *lib_manager_get_library_manifest(int module_id); struct processing_module; +struct comp_ipc_config; /* * \brief Allocate module * From c9810d894d5dbe66620ade62aecc50e07646ffa0 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 23 May 2025 16:48:35 +0200 Subject: [PATCH 4/8] library-manager: fix a potential NULL dereference If module registration fails, lib_manager_load_library() will jump to "cleanup" where it will attempt to check whether a module is an LLEXT but in such cases the "mod" pointer can be NULL, which will cause an exception. Check the error code before checking the module type. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/lib_manager.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index eb5292aec1b7..25d719aac108 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -1030,7 +1030,7 @@ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type) uint32_t module_id = lib_id << LIB_MANAGER_LIB_ID_SHIFT; const struct sof_man_module *mod = lib_manager_get_module_manifest(module_id); - if (module_is_llext(mod) && !ret) + if (!ret && module_is_llext(mod)) /* Auxiliary LLEXT libraries need to be linked upon loading */ ret = llext_manager_add_library(module_id); @@ -1039,7 +1039,7 @@ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type) #endif if (!ret) - tr_info(&ipc_tr, "loaded library id: %u", lib_id); + tr_info(&lib_manager_tr, "loaded library id: %u", lib_id); return ret; } From 2c36914feefea447b87fcc173574d9f59f9b0755 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 23 May 2025 16:54:57 +0200 Subject: [PATCH 5/8] llext-manager: check llext_get_section_info() return code llext_get_section_info() shouldn't fail when called with a valid section number, but it's still cleaner to check its success and skip the section in case of an error. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 213d1722c5ad..6824ec702278 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -113,8 +113,12 @@ static int llext_manager_load_data_from_storage(const struct llext_loader *ldr, const elf_shdr_t *shdr; enum llext_mem s_region = LLEXT_MEM_COUNT; size_t s_offset = 0; + int ret = llext_get_section_info(ldr, ext, i, &shdr, &s_region, &s_offset); - llext_get_section_info(ldr, ext, i, &shdr, &s_region, &s_offset); + if (ret < 0) { + tr_err(lib_manager_tr, "no section info: %d", ret); + continue; + } /* skip sections not in the requested region */ if (s_region != region) From 6a4e0f49a3a9550c2eae25a5c5862bd9596e2cba Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 23 May 2025 16:56:55 +0200 Subject: [PATCH 6/8] llext-manager: fix an error case in llext_manager_link_single() When failing llext_manager_link_single() should return a negative error code, not 0. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 6824ec702278..0272b0280f8e 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -532,7 +532,7 @@ static int llext_manager_link_single(uint32_t module_id, const struct sof_man_fw sizeof(struct llext_buf_loader)); if (!mctx->ebl) { tr_err(&lib_manager_tr, "loader alloc failed"); - return 0; + return -ENOMEM; } uint8_t *dram_base = (uint8_t *)desc - SOF_MAN_ELF_TEXT_OFFSET; From e5ebed21ffcf8ad6b8f878d7f2894c73c4e839f9 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 11 Jun 2025 10:05:32 +0200 Subject: [PATCH 7/8] west.yml: update zephyr to 9075d5335596 Updpate Zephyr to include 4de0c9abc0e SoC: Intel: ADSP: ACE30: add .imrdata to MMU definitions 5f4177b47c9 llext: add context save and restore d8195c05c78 llext: rename _llext_list to llext_list Signed-off-by: Guennadi Liakhovetski --- west.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/west.yml b/west.yml index 09d91a0a1453..ae60859bc0a8 100644 --- a/west.yml +++ b/west.yml @@ -43,7 +43,7 @@ manifest: - name: zephyr repo-path: zephyr - revision: 8c6b2c8c88773810d64a47ed3b6206b7d6e8290f + revision: 9075d5335596080291ebcc448819fed2110dcb9a remote: zephyrproject # Import some projects listed in zephyr/west.yml@revision From 1ebde9709258095b9ea0a311eaf8f15cddb0de98 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 23 May 2025 16:18:30 +0200 Subject: [PATCH 8/8] llext: avoid reloading libraries during resume Currently when resuming from D3 we lose the complete LLEXT context, which forces SOF to reload all the libraries, which costs time. This isn't necessary, because the libraries are stored in DRAM, which preserves its contents across DSP reset. Instead of reloading save LLEXT context before power down and reload it when resuming. Signed-off-by: Guennadi Liakhovetski --- app/boards/intel_adsp_ace15_mtpm.conf | 1 + app/boards/intel_adsp_ace20_lnl.conf | 1 + app/boards/intel_adsp_ace30_ptl.conf | 1 + app/boards/intel_adsp_ace30_wcl.conf | 1 + src/include/ipc4/notification.h | 2 + src/include/sof/llext_manager.h | 8 + src/ipc/ipc4/handler.c | 12 + src/library_manager/CMakeLists.txt | 3 + src/library_manager/llext_manager_dram.c | 334 +++++++++++++++++++++++ src/platform/intel/ace/platform.c | 1 + zephyr/wrapper.c | 23 +- 11 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 src/library_manager/llext_manager_dram.c diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index 26a9680e92f1..d2cc58c13e9d 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -68,6 +68,7 @@ CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_MIN=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y CONFIG_TIMING_FUNCTIONS=y CONFIG_WATCHDOG=y diff --git a/app/boards/intel_adsp_ace20_lnl.conf b/app/boards/intel_adsp_ace20_lnl.conf index a8888c9557d6..6aa76f72f62b 100644 --- a/app/boards/intel_adsp_ace20_lnl.conf +++ b/app/boards/intel_adsp_ace20_lnl.conf @@ -52,6 +52,7 @@ CONFIG_COUNTER=y CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y CONFIG_TIMING_FUNCTIONS=y diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 7af406131d76..4aa8a5bf122d 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -53,6 +53,7 @@ CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_L3_HEAP=y CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y # Zephyr / device drivers diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index c33c0e2127bb..8da2738668a0 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -48,6 +48,7 @@ CONFIG_HEAP_MEM_POOL_SIZE=8192 CONFIG_L3_HEAP=y CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y +CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y # Zephyr / device drivers diff --git a/src/include/ipc4/notification.h b/src/include/ipc4/notification.h index 0581159d90c7..207dbfb7139c 100644 --- a/src/include/ipc4/notification.h +++ b/src/include/ipc4/notification.h @@ -114,6 +114,8 @@ enum sof_ipc4_resource_type { (((SOF_IPC4_NOTIFY_FW_READY) << (SOF_IPC4_GLB_NOTIFY_TYPE_SHIFT)) |\ ((SOF_IPC4_GLB_NOTIFICATION) << (SOF_IPC4_GLB_NOTIFY_MSG_TYPE_SHIFT))) +#define SOF_IPC4_FW_READY_LIB_RESTORED BIT(15) + #define SOF_IPC4_NOTIF_HEADER(notif_type) \ ((notif_type) << (SOF_IPC4_GLB_NOTIFY_TYPE_SHIFT) | \ ((SOF_IPC4_GLB_NOTIFICATION) << (SOF_IPC4_GLB_NOTIFY_MSG_TYPE_SHIFT))) diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index 7394e62725cf..44928c5a124f 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -38,4 +38,12 @@ bool comp_is_llext(struct comp_dev *comp); #define comp_is_llext(comp) false #endif +#if CONFIG_LLEXT_EXPERIMENTAL && !CONFIG_ADSP_IMR_CONTEXT_SAVE +int llext_manager_store_to_dram(void); +int llext_manager_restore_from_dram(void); +#else +#define llext_manager_store_to_dram() 0 +#define llext_manager_restore_from_dram() -ENOSYS +#endif + #endif diff --git a/src/ipc/ipc4/handler.c b/src/ipc/ipc4/handler.c index d2199be999ce..16156a29b6db 100644 --- a/src/ipc/ipc4/handler.c +++ b/src/ipc/ipc4/handler.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1485,6 +1486,17 @@ __cold static int ipc4_module_process_dx(struct ipc4_message_request *ipc4) return IPC4_BUSY; } +#if !CONFIG_ADSP_IMR_CONTEXT_SAVE + ret = llext_manager_store_to_dram(); + if (ret < 0) + ipc_cmd_err(&ipc_tr, "Error %d saving LLEXT context. Resume might fail.", + ret); + +#if CONFIG_L3_HEAP + l3_heap_save(); +#endif +#endif + #if defined(CONFIG_PM) ipc_get()->task_mask |= IPC_TASK_POWERDOWN; #endif diff --git a/src/library_manager/CMakeLists.txt b/src/library_manager/CMakeLists.txt index 31e2fc4a24be..e69ca4cca30c 100644 --- a/src/library_manager/CMakeLists.txt +++ b/src/library_manager/CMakeLists.txt @@ -5,5 +5,8 @@ if(CONFIG_LIBRARY_MANAGER) if (CONFIG_MM_DRV AND CONFIG_LLEXT) add_local_sources(sof llext_manager.c) + if(CONFIG_LLEXT_EXPERIMENTAL AND NOT CONFIG_ADSP_IMR_CONTEXT_SAVE) + add_local_sources_ifdef(CONFIG_L3_HEAP sof llext_manager_dram.c) + endif() endif() endif() diff --git a/src/library_manager/llext_manager_dram.c b/src/library_manager/llext_manager_dram.c new file mode 100644 index 000000000000..7629da427462 --- /dev/null +++ b/src/library_manager/llext_manager_dram.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2025 Intel Corporation. + +#include + +#include +#include + +#include +#include +#include +#include + +LOG_MODULE_DECLARE(lib_manager, CONFIG_SOF_LOG_LEVEL); + +struct lib_manager_dram_storage { + struct ext_library ext_lib; + struct lib_manager_mod_ctx *ctx; + struct lib_manager_module *mod; + struct llext *llext; + struct llext_buf_loader *bldr; + struct llext_elf_sect_map *sect; + struct llext_symbol *sym; + unsigned int n_llext; +}; + +/* + * This holds the complete LLEXT manager context in DRAM over DSP shut down to + * be restored during the next boot + */ +__imrdata static struct lib_manager_dram_storage lib_manager_dram; + +/* Store LLEXT manager context in DRAM to be restored during the next boot. */ +int llext_manager_store_to_dram(void) +{ + struct ext_library *_ext_lib = ext_lib_get(); + unsigned int i, j, k, l, n_lib, n_mod, n_llext, n_sect, n_sym; + size_t buf_size; + + if (lib_manager_dram.n_llext) { + tr_err(&lib_manager_tr, "context already saved"); + return 0; + } + + /* + * Count libraries, modules, instantiated extensions, sections and exported + * symbols in them. Allocate a buffer of required size. + */ + lib_manager_dram.ext_lib = *_ext_lib; + for (i = 0, n_lib = 0, n_mod = 0, n_llext = 0, n_sect = 0, n_sym = 0; + i < ARRAY_SIZE(_ext_lib->desc); i++) + if (_ext_lib->desc[i]) { + n_lib++; + n_mod += _ext_lib->desc[i]->n_mod; + for (k = 0; k < _ext_lib->desc[i]->n_mod; k++) + if (_ext_lib->desc[i]->mod[k].ebl) { + n_llext++; + n_sect += _ext_lib->desc[i]->mod[k].llext->sect_cnt; + n_sym += _ext_lib->desc[i]->mod[k].llext->exp_tab.sym_cnt; + tr_dbg(&lib_manager_tr, "add %u exported syms", + _ext_lib->desc[i]->mod[k].llext->exp_tab.sym_cnt); + } + } + + buf_size = sizeof(lib_manager_dram.ctx[0]) * n_lib + + sizeof(lib_manager_dram.mod[0]) * n_mod + + sizeof(lib_manager_dram.sect[0]) * n_sect + + sizeof(lib_manager_dram.sym[0]) * n_sym + + (sizeof(lib_manager_dram.llext[0]) + sizeof(lib_manager_dram.bldr[0])) * n_llext; + + lib_manager_dram.ctx = rmalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_L3, + buf_size); + if (!lib_manager_dram.ctx) + return -ENOMEM; + + /* Save pointers to buffer parts, holding parts of the context */ + lib_manager_dram.mod = (struct lib_manager_module *)(lib_manager_dram.ctx + n_lib); + lib_manager_dram.sect = (struct llext_elf_sect_map *)(lib_manager_dram.mod + n_mod); + lib_manager_dram.llext = (struct llext *)(lib_manager_dram.sect + n_sect); + lib_manager_dram.bldr = (struct llext_buf_loader *)(lib_manager_dram.llext + n_llext); + lib_manager_dram.sym = (struct llext_symbol *)(lib_manager_dram.bldr + n_llext); + + tr_dbg(&lib_manager_tr, "backup %u libs of %u modules with %u LLEXT with %u sections", + n_lib, n_mod, n_llext, n_sect); + + tr_dbg(&lib_manager_tr, "backup %p to %p, mod %p, loader %p", + lib_manager_dram.ctx, (void *)((uint8_t *)lib_manager_dram.ctx + buf_size), + lib_manager_dram.mod, lib_manager_dram.bldr); + + /* Walk all libraries */ + for (i = 0, j = 0, l = 0, n_mod = 0, n_sect = 0, n_sym = 0; + i < ARRAY_SIZE(_ext_lib->desc); i++) { + if (!_ext_lib->desc[i]) + continue; + + struct lib_manager_module *mod = _ext_lib->desc[i]->mod; + + /* Copy all modules in each library */ + lib_manager_dram.ctx[j++] = *_ext_lib->desc[i]; + memcpy(lib_manager_dram.mod + n_mod, mod, + sizeof(lib_manager_dram.mod[0]) * _ext_lib->desc[i]->n_mod); + tr_dbg(&lib_manager_tr, "lib %u base %p", j - 1, + lib_manager_dram.ctx[j - 1].base_addr); + n_mod += _ext_lib->desc[i]->n_mod; + + /* + * Copy instantiated extensions. Note that only modules, that + * were used, have their LLEXT context instantiated. + */ + for (k = 0; k < _ext_lib->desc[i]->n_mod; k++) { + if (!mod[k].llext) + continue; + + tr_dbg(&lib_manager_tr, "mod %u of %u sections", k, + mod[k].llext->sect_cnt); + + /* Copy the extension and the loader */ + lib_manager_dram.llext[l] = *mod[k].llext; + lib_manager_dram.bldr[l] = *mod[k].ebl; + + /* Copy the section map */ + memcpy(lib_manager_dram.sect + n_sect, mod[k].ebl->loader.sect_map, + mod[k].llext->sect_cnt * sizeof(lib_manager_dram.sect[0])); + n_sect += mod[k].llext->sect_cnt; + + /* Copy exported symbols */ + if (mod[k].llext->exp_tab.sym_cnt) { + memcpy(lib_manager_dram.sym + n_sym, mod[k].llext->exp_tab.syms, + mod[k].llext->exp_tab.sym_cnt * + sizeof(lib_manager_dram.sym[0])); + lib_manager_dram.llext[l].exp_tab.syms = lib_manager_dram.sym + + n_sym; + n_sym += mod[k].llext->exp_tab.sym_cnt; + } + + l++; + } + } + + /* Also flatten dependency lists */ + int ret = llext_relink_dependency(lib_manager_dram.llext, n_llext); + + if (ret < 0) { + tr_err(&lib_manager_tr, "Inconsistent dependencies!"); + return ret; + } + + lib_manager_dram.n_llext = n_llext; + /* Make sure, that the data is actually in the DRAM, not just in data cache */ + dcache_writeback_region((__sparse_force void __sparse_cache *)&lib_manager_dram, + sizeof(lib_manager_dram)); + dcache_writeback_region((__sparse_force void __sparse_cache *)lib_manager_dram.ctx, + buf_size); + + return 0; +} + +int llext_manager_restore_from_dram(void) +{ + lib_manager_init(); + + struct ext_library *_ext_lib = ext_lib_get(); + unsigned int i, j, k, n_mod, n_llext, n_sect, n_sym; + + if (!lib_manager_dram.n_llext || !lib_manager_dram.ctx) { + tr_dbg(&lib_manager_tr, "No modules saved"); + dcache_writeback_region((__sparse_force void __sparse_cache *)&lib_manager_dram, + sizeof(lib_manager_dram)); + return 0; + } + + /* arrays of pointers for llext_restore() */ + void **ptr_array = rmalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, + sizeof(*ptr_array) * lib_manager_dram.n_llext * 2); + + if (!ptr_array) + return -ENOMEM; + + struct llext_loader **ldr = (struct llext_loader **)ptr_array; + struct llext **llext = (struct llext **)(ptr_array + lib_manager_dram.n_llext); + + *_ext_lib = lib_manager_dram.ext_lib; + + /* The external loop walks all the libraries */ + for (i = 0, j = 0, n_mod = 0, n_llext = 0, n_sect = 0, n_sym = 0; + i < ARRAY_SIZE(_ext_lib->desc); i++) { + if (!lib_manager_dram.ext_lib.desc[i]) { + _ext_lib->desc[i] = NULL; + continue; + } + + /* Panics on failure - use the same zone as during the first boot */ + struct lib_manager_mod_ctx *ctx = rmalloc(SOF_MEM_ZONE_SYS, SOF_MEM_FLAG_COHERENT, + SOF_MEM_CAPS_RAM, sizeof(*ctx)); + + /* Restore the library context */ + *ctx = lib_manager_dram.ctx[j++]; + + /* Allocate and restore all the modules in the library */ + struct lib_manager_module *mod = rmalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + SOF_MEM_FLAG_COHERENT, SOF_MEM_CAPS_RAM, + ctx->n_mod * sizeof(ctx->mod[0])); + + if (!mod) { + tr_err(&lib_manager_tr, "module allocation failure"); + goto nomem; + } + tr_dbg(&lib_manager_tr, "%u modules alloc %p base %p copy %#zx", + ctx->n_mod, (void *)mod, ctx->base_addr, ctx->n_mod * sizeof(ctx->mod[0])); + + memcpy(mod, lib_manager_dram.mod + n_mod, sizeof(mod[0]) * ctx->n_mod); + n_mod += ctx->n_mod; + ctx->mod = mod; + + /* Second level: enumerate modules in each library */ + for (k = 0; k < ctx->n_mod; k++) { + if (!mod[k].llext) + /* Not instantiated - nothing to restore */ + continue; + + /* Loaders are supplied by the caller */ + struct llext_buf_loader *bldr = rmalloc(SOF_MEM_ZONE_RUNTIME_SHARED, + 0, SOF_MEM_CAPS_RAM, sizeof(*bldr)); + + if (!bldr) { + tr_err(&lib_manager_tr, "loader allocation failure"); + goto nomem; + } + + /* Extensions have to be restored by Zephyr, collect pointers first */ + llext[n_llext] = lib_manager_dram.llext + n_llext; + + *bldr = lib_manager_dram.bldr[n_llext]; + + bldr->loader.sect_map = lib_manager_dram.sect + n_sect; + + n_sect += llext[n_llext]->sect_cnt; + if (llext[n_llext]->exp_tab.sym_cnt) { + /* + * Just a check, that we're restoring exported + * symbols correctly + */ + tr_dbg(&lib_manager_tr, "got %u exported symbols", + llext[n_llext]->exp_tab.sym_cnt); + + if (llext[n_llext]->exp_tab.syms != lib_manager_dram.sym + n_sym) { + tr_err(&lib_manager_tr, + "bug detected! pointer mismatch %p vs. %p", + (void *)llext[n_llext]->exp_tab.syms, + (void *)(lib_manager_dram.sym + n_sym)); + goto nomem; + } + + n_sym += llext[n_llext]->exp_tab.sym_cnt; + } + + mod[k].ebl = bldr; + + ldr[n_llext++] = &bldr->loader; + } + + _ext_lib->desc[i] = ctx; + } + + /* Let Zephyr restore extensions and its own internal bookkeeping */ + int ret = llext_restore(llext, ldr, lib_manager_dram.n_llext); + + if (ret < 0) { + tr_err(&lib_manager_tr, "Zephyr failed to restore: %d", ret); + goto nomem; + } + + /* Rewrite to correct LLEXT pointers, created by Zephyr */ + for (i = 0, n_llext = 0; i < ARRAY_SIZE(_ext_lib->desc); i++) { + struct lib_manager_mod_ctx *ctx = _ext_lib->desc[i]; + + if (!ctx) + continue; + + struct lib_manager_module *mod = ctx->mod; + + for (k = 0; k < ctx->n_mod; k++) { + if (mod[k].llext) + mod[k].llext = llext[n_llext++]; + } + } + + tr_info(&lib_manager_tr, "restored %u modules with %u LLEXT", n_mod, n_llext); + + rfree(lib_manager_dram.ctx); + lib_manager_dram.ctx = NULL; + lib_manager_dram.sect = NULL; + lib_manager_dram.llext = NULL; + lib_manager_dram.bldr = NULL; + lib_manager_dram.sym = NULL; + rfree(ldr); + + lib_manager_dram.n_llext = 0; + + return 0; + +nomem: + tr_err(&lib_manager_tr, "Restore failed"); + for (i = 0; i < ARRAY_SIZE(_ext_lib->desc); i++) { + struct lib_manager_mod_ctx *ctx = _ext_lib->desc[i]; + + if (!ctx) + continue; + + struct lib_manager_module *mod = ctx->mod; + + if (!mod) + continue; + + for (k = 0; k < ctx->n_mod; k++) { + if (mod[k].llext) + llext_unload(&mod[k].llext); + + if (mod[k].ebl) + rfree(mod[k].ebl); + } + + rfree(mod); + rfree(ctx); + } + + /* at least create a sane empty lib-manager context */ + memset(_ext_lib->desc, 0, sizeof(_ext_lib->desc)); + + rfree(ldr); + + return -ENOMEM; +} diff --git a/src/platform/intel/ace/platform.c b/src/platform/intel/ace/platform.c index edc36ab5dcf8..1fde2713049d 100644 --- a/src/platform/intel/ace/platform.c +++ b/src/platform/intel/ace/platform.c @@ -67,6 +67,7 @@ __cold int platform_boot_complete(uint32_t boot_message) /* get any IPC specific boot message and optional data */ ipc_boot_complete_msg(&header, 0); + header.pri |= boot_message; struct ipc_msg msg = { .header = header.pri, diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index c2129347241e..2e302f22c94b 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -20,6 +21,9 @@ #include #include #include +#if CONFIG_IPC_MAJOR_4 +#include +#endif /* Zephyr includes */ #include @@ -187,8 +191,25 @@ static int boot_complete(void) */ return 0; #else + unsigned int status; + int ret = llext_manager_restore_from_dram(); + + switch (ret) { +#if CONFIG_IPC_MAJOR_4 + case 0: + status = SOF_IPC4_FW_READY_LIB_RESTORED; + break; +#endif + case -ENOSYS: + status = 0; + break; + default: + status = 0; + LOG_ERR("LLEXT restore failed: %d", ret); + } + /* let host know DSP boot is complete */ - return platform_boot_complete(0); + return platform_boot_complete(status); #endif /* CONFIG_IMX93_A55 */ }