From c8ddbd7d15ec561a8516d16653104ac5e7de1f5a Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 25 Jul 2025 12:03:12 +0200 Subject: [PATCH 1/9] audio: dp: create the thread early We need to move IPC processing for DP scheduled components into their thread context. For that the thread has to be started early. Create it immediately when creating DP task context. Note, that k_thread_create() never fails, so no need to check for error. Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_dp_schedule.c | 38 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 6181a34a1a4e..18c67fdc387f 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -481,16 +481,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return -EINVAL; } - /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)pdata->p_stack, - pdata->stack_size, dp_thread_fn, task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, task->flags, K_FOREVER); - if (!pdata->thread_id) { - tr_err(&dp_tr, "DP thread creation failed"); - scheduler_dp_unlock(lock_key); - return -ECHILD; - } - k_thread_access_grant(pdata->thread_id, pdata->sem); scheduler_dp_grant(pdata->thread_id, cpu_get_id()); /* pin the thread to specific core */ @@ -622,22 +612,24 @@ int scheduler_dp_task_init(struct task **task, goto err; } + struct task_dp_pdata *pdata = &task_memory->pdata; + /* Point to ksem semaphore for kernel threads synchronization */ /* It will be overwritten for K_USER threads to dynamic ones. */ - task_memory->pdata.sem = &task_memory->pdata.sem_struct; - task_memory->pdata.thread = &task_memory->pdata.thread_struct; + pdata->sem = &pdata->sem_struct; + pdata->thread = &pdata->thread_struct; #ifdef CONFIG_USERSPACE if (options & K_USER) { - task_memory->pdata.sem = k_object_alloc(K_OBJ_SEM); - if (!task_memory->pdata.sem) { + pdata->sem = k_object_alloc(K_OBJ_SEM); + if (!pdata->sem) { tr_err(&dp_tr, "Semaphore object allocation failed"); ret = -ENOMEM; goto err; } - task_memory->pdata.thread = k_object_alloc(K_OBJ_THREAD); - if (!task_memory->pdata.thread) { + pdata->thread = k_object_alloc(K_OBJ_THREAD); + if (!pdata->thread) { tr_err(&dp_tr, "Thread object allocation failed"); ret = -ENOMEM; goto err; @@ -650,15 +642,21 @@ int scheduler_dp_task_init(struct task **task, task_memory->task.ops.get_deadline = ops->get_deadline; task_memory->task.state = SOF_TASK_STATE_INIT; task_memory->task.core = core; + task_memory->task.priv_data = pdata; /* success, fill the structures */ - task_memory->task.priv_data = &task_memory->pdata; - task_memory->pdata.p_stack = p_stack; - task_memory->pdata.stack_size = stack_size; - task_memory->pdata.mod = mod; + pdata->p_stack = p_stack; + pdata->stack_size = stack_size; + pdata->mod = mod; *task = &task_memory->task; + /* create a zephyr thread for the task */ + pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, + stack_size, dp_thread_fn, *task, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, (*task)->flags, K_FOREVER); + return 0; + err: /* cleanup - free all allocated resources */ if (user_stack_free((__sparse_force void *)p_stack)) From acdb39b7ca73354a247cfd085909f454d9b63c89 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 30 Jul 2025 16:58:21 +0200 Subject: [PATCH 2/9] schedule: dp: start the task early We need to run all module callbacks in DP thread context, for this the thread has to be started early - before the first module callback is called. Signed-off-by: Guennadi Liakhovetski --- src/audio/module_adapter/module_adapter.c | 21 +++++---- src/schedule/zephyr_dp_schedule.c | 56 +++++++++++------------ 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index cf36da4e1034..4a7f53f78cff 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -183,6 +183,15 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, if (!mod) return NULL; + struct comp_dev *dev = mod->dev; + +#if CONFIG_ZEPHYR_DP_SCHEDULER + /* create a task for DP processing */ + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { + /* All data allocated, create a thread */ + pipeline_comp_dp_task_init(dev); + } +#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ module_set_private_data(mod, mod_priv); list_init(&mod->raw_data_buffers_list); @@ -190,8 +199,6 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, mod->user_ctx = user_ctx; #endif /* CONFIG_USERSPACE */ - struct comp_dev *dev = mod->dev; - dst = &mod->priv.cfg; ret = module_adapter_init_data(dev, dst, config, spec); if (ret) { @@ -238,12 +245,6 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, goto err; } -#if CONFIG_ZEPHYR_DP_SCHEDULER - /* create a task for DP processing */ - if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) - pipeline_comp_dp_task_init(dev); -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ - module_adapter_reset_data(dst); dev->state = COMP_STATE_READY; @@ -269,6 +270,10 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, return dev; err: +#if CONFIG_ZEPHYR_DP_SCHEDULER + if (dev->task) + schedule_task_free(dev->task); +#endif module_adapter_mem_free(mod); return NULL; diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 18c67fdc387f..34e30fe78d51 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -470,7 +470,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; unsigned int lock_key; - int ret; lock_key = scheduler_dp_lock(cpu_get_id()); @@ -481,29 +480,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return -EINVAL; } - k_thread_access_grant(pdata->thread_id, pdata->sem); - scheduler_dp_grant(pdata->thread_id, cpu_get_id()); - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(pdata->thread_id, task->core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto err; - } - -#ifdef CONFIG_USERSPACE - if (task->flags & K_USER) { - ret = user_memory_init_shared(pdata->thread_id, pdata->mod); - if (ret < 0) { - tr_err(&dp_tr, "user_memory_init_shared() failed"); - goto err; - } - } -#endif /* CONFIG_USERSPACE */ - - /* start the thread, it should immediately stop at a semaphore, so clean it */ - k_sem_init(pdata->sem, 0, 1); - k_thread_start(pdata->thread_id); - /* if there's no DP tasks scheduled yet, run ll tick source task */ if (list_is_empty(&dp_sch->tasks)) schedule_task(&dp_sch->ll_tick_src, 0, 0); @@ -517,12 +493,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta tr_dbg(&dp_tr, "DP task scheduled with period %u [us]", (uint32_t)period); return 0; - -err: - /* cleanup - unlock and free all allocated resources */ - scheduler_dp_unlock(lock_key); - k_thread_abort(pdata->thread_id); - return ret; } static struct scheduler_ops schedule_dp_ops = { @@ -655,8 +625,34 @@ int scheduler_dp_task_init(struct task **task, stack_size, dp_thread_fn, *task, NULL, NULL, CONFIG_DP_THREAD_PRIORITY, (*task)->flags, K_FOREVER); + k_thread_access_grant(pdata->thread_id, pdata->sem); + scheduler_dp_grant(pdata->thread_id, cpu_get_id()); + + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(pdata->thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } + +#ifdef CONFIG_USERSPACE + if ((*task)->flags & K_USER) { + ret = user_memory_init_shared(pdata->thread_id, pdata->mod); + if (ret < 0) { + tr_err(&dp_tr, "user_memory_init_shared() failed"); + goto e_thread; + } + } +#endif /* CONFIG_USERSPACE */ + + /* start the thread, it should immediately stop at a semaphore, so clean it */ + k_sem_init(pdata->sem, 0, 1); + k_thread_start(pdata->thread_id); + return 0; +e_thread: + k_thread_abort(pdata->thread_id); err: /* cleanup - free all allocated resources */ if (user_stack_free((__sparse_force void *)p_stack)) From b0690f8ae7ae6075c77516acb73cdba3d91ae368 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 7 Aug 2025 09:00:32 +0200 Subject: [PATCH 3/9] audio: dp: terminate component tasks late DP processing threads should have as long as life time as possible to process all the relevant IPCs in the thread context. Move thread termination to be called immediately before freeing module data. Signed-off-by: Guennadi Liakhovetski --- src/audio/module_adapter/module_adapter.c | 3 +++ src/include/sof/audio/component_ext.h | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 4a7f53f78cff..c64a650e0b38 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -1354,6 +1354,9 @@ void module_adapter_free(struct comp_dev *dev) comp_dbg(dev, "start"); + if (dev->task) + schedule_task_cancel(dev->task); + ret = module_free(mod); if (ret) comp_err(dev, "failed with error: %d", ret); diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index b5ce2075b2f0..0fac007cfe36 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -164,7 +164,6 @@ static inline int comp_trigger_local(struct comp_dev *dev, int cmd) case COMP_TRIGGER_XRUN: case COMP_TRIGGER_PAUSE: case COMP_TRIGGER_STOP: - schedule_task_cancel(dev->task); break; } } From 80dcb942a4db12e8899b23cd5cf88b6476c67f95 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 15 Aug 2025 10:39:35 +0200 Subject: [PATCH 4/9] audio: src: remove unused functions src_set_config() and src_get_config() aren't used, they would return an error if ever called. It's easier to just remove them. Signed-off-by: Guennadi Liakhovetski --- src/audio/src/src.c | 2 -- src/audio/src/src_common.c | 18 ------------------ src/audio/src/src_common.h | 6 ------ src/audio/src/src_lite.c | 2 -- 4 files changed, 28 deletions(-) diff --git a/src/audio/src/src.c b/src/audio/src/src.c index 3b1eca197b6a..9870ad911ca8 100644 --- a/src/audio/src/src.c +++ b/src/audio/src/src.c @@ -78,8 +78,6 @@ static const struct module_interface src_interface = { .prepare = src_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, - .set_configuration = src_set_config, - .get_configuration = src_get_config, .reset = src_reset, .free = src_free, }; diff --git a/src/audio/src/src_common.c b/src/audio/src/src_common.c index 5d25413193a8..f62df9deca24 100644 --- a/src/audio/src/src_common.c +++ b/src/audio/src/src_common.c @@ -669,24 +669,6 @@ int src_process(struct processing_module *mod, return cd->src_func(cd, sources[0], sinks[0]); } -__cold int src_set_config(struct processing_module *mod, uint32_t config_id, - enum module_cfg_fragment_position pos, uint32_t data_offset_size, - const uint8_t *fragment, size_t fragment_size, uint8_t *response, - size_t response_size) -{ - assert_can_be_cold(); - - return -EINVAL; -} - -__cold int src_get_config(struct processing_module *mod, uint32_t config_id, - uint32_t *data_offset_size, uint8_t *fragment, size_t fragment_size) -{ - assert_can_be_cold(); - - return -EINVAL; -} - int src_reset(struct processing_module *mod) { struct comp_data *cd = module_get_private_data(mod); diff --git a/src/audio/src/src_common.h b/src/audio/src/src_common.h index 6a8028662ffd..ae3c60574eec 100644 --- a/src/audio/src/src_common.h +++ b/src/audio/src/src_common.h @@ -242,12 +242,6 @@ int src_process(struct processing_module *mod, struct sof_source **sources, int num_of_sources, struct sof_sink **sinks, int num_of_sinks); -int src_set_config(struct processing_module *mod, uint32_t config_id, - enum module_cfg_fragment_position pos, uint32_t data_offset_size, - const uint8_t *fragment, size_t fragment_size, uint8_t *response, - size_t response_size); -int src_get_config(struct processing_module *mod, uint32_t config_id, - uint32_t *data_offset_size, uint8_t *fragment, size_t fragment_size); int src_free(struct processing_module *mod); int src_reset(struct processing_module *mod); extern struct tr_ctx src_tr; diff --git a/src/audio/src/src_lite.c b/src/audio/src/src_lite.c index cefe60367bfd..9d5593ff34ca 100644 --- a/src/audio/src/src_lite.c +++ b/src/audio/src/src_lite.c @@ -61,8 +61,6 @@ const struct module_interface src_lite_interface = { .prepare = src_lite_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, - .set_configuration = src_set_config, - .get_configuration = src_get_config, .reset = src_reset, .free = src_free, }; From 3ecabe3c13c341b43f00dd197bbdca92ba0ba7a5 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 26 Sep 2025 16:58:38 +0200 Subject: [PATCH 5/9] llext: add support for adding LLEXT memory to a memory domain Add functions for adding LLEXT partitions to a memory domain for user-space modules. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/llext_manager.h | 4 ++ src/library_manager/llext_manager.c | 61 +++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index 44928c5a124f..78c00e89d2a3 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -15,6 +15,7 @@ struct comp_dev; struct comp_driver; struct comp_ipc_config; +struct k_mem_domain; static inline bool module_is_llext(const struct sof_man_module *mod) { @@ -29,12 +30,15 @@ int llext_manager_free_module(const uint32_t component_id); int llext_manager_add_library(uint32_t module_id); +int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain); + bool comp_is_llext(struct comp_dev *comp); #else #define module_is_llext(mod) false #define llext_manager_allocate_module(ipc_config, ipc_specific_config) 0 #define llext_manager_free_module(component_id) 0 #define llext_manager_add_library(module_id) 0 +#define llext_manager_add_domain(component_id, domain) 0 #define comp_is_llext(comp) false #endif diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 9c032319f82d..a21891728037 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -716,6 +717,66 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config return mod_manifest->module.entry_point; } +#ifdef CONFIG_USERSPACE +static int llext_manager_add_partition(struct k_mem_domain *domain, + uintptr_t addr, size_t size, + k_mem_partition_attr_t attr) +{ + size_t pre_pad_size = addr & (PAGE_SZ - 1); + struct k_mem_partition part = { + .start = addr - pre_pad_size, + .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), + .attr = attr, + }; + + tr_dbg(&lib_manager_tr, "add %#zx @ %lx partition", part.size, part.start); + return k_mem_domain_add_partition(domain, &part); +} + +int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain) +{ + const uint32_t module_id = IPC4_MOD_ID(component_id); + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); + const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); + const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + struct lib_manager_module *mctx = ctx->mod + mod_idx; + int ret; + + /* Executable code (.text) */ + uintptr_t va_base_text = mctx->segment[LIB_MANAGER_TEXT].addr; + size_t text_size = mctx->segment[LIB_MANAGER_TEXT].size; + + /* Read-only data (.rodata and others) */ + uintptr_t va_base_rodata = mctx->segment[LIB_MANAGER_RODATA].addr; + size_t rodata_size = mctx->segment[LIB_MANAGER_RODATA].size; + + /* Writable data (.data, .bss and others) */ + uintptr_t va_base_data = mctx->segment[LIB_MANAGER_DATA].addr; + size_t data_size = mctx->segment[LIB_MANAGER_DATA].size; + + ret = llext_manager_add_partition(domain, va_base_text, text_size, + K_MEM_PARTITION_P_RX_U_RX); + if (ret < 0) + return ret; + + if (rodata_size) { + ret = llext_manager_add_partition(domain, va_base_rodata, rodata_size, + K_MEM_PARTITION_P_RO_U_RO); + if (ret < 0) + return ret; + } + + if (data_size) { + ret = llext_manager_add_partition(domain, va_base_data, data_size, + K_MEM_PARTITION_P_RW_U_RW); + if (ret < 0) + return ret; + } + + return 0; +} +#endif + int llext_manager_free_module(const uint32_t component_id) { const uint32_t module_id = IPC4_MOD_ID(component_id); From 984ed50b3798e0a9817002eb25e47e9d85668fe8 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 26 Sep 2025 13:55:54 +0200 Subject: [PATCH 6/9] ring-buffer: allocate on module heap Allocate the ring-buffer object on module heap too for DP access. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/ring_buffer.c | 11 ++++++++--- src/include/sof/audio/audio_buffer.h | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index 183894daf456..e6334f55a45c 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -97,7 +98,7 @@ static void ring_buffer_free(struct sof_audio_buffer *audio_buffer) container_of(audio_buffer, struct ring_buffer, audio_buffer); rfree((__sparse_force void *)ring_buffer->_data_buffer); - rfree(ring_buffer); + sof_heap_free(audio_buffer->heap, ring_buffer); } static void ring_buffer_reset(struct sof_audio_buffer *audio_buffer) @@ -286,14 +287,17 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl uint32_t id) { struct ring_buffer *ring_buffer; + struct k_heap *heap = dev->mod->priv.resources.heap; int memory_flags = (is_shared ? SOF_MEM_FLAG_COHERENT : 0) | user_get_buffer_memory_region(dev->drv); /* allocate ring_buffer structure */ - ring_buffer = rzalloc(memory_flags, sizeof(*ring_buffer)); + ring_buffer = sof_heap_alloc(heap, memory_flags, sizeof(*ring_buffer), 0); if (!ring_buffer) return NULL; + memset(ring_buffer, 0, sizeof(*ring_buffer)); + /* init base structure. The audio_stream_params is NULL because ring_buffer * is currently used as a secondary buffer for DP only * @@ -303,6 +307,7 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl audio_buffer_init(&ring_buffer->audio_buffer, BUFFER_TYPE_RING_BUFFER, is_shared, &ring_buffer_source_ops, &ring_buffer_sink_ops, &audio_buffer_ops, NULL); + ring_buffer->audio_buffer.heap = heap; /* set obs/ibs in sink/source interfaces */ sink_set_min_free_space(audio_buffer_get_sink(&ring_buffer->audio_buffer), @@ -374,6 +379,6 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl return ring_buffer; err: tr_err(&ring_buffer_tr, "Ring buffer creation failure"); - rfree(ring_buffer); + sof_heap_free(heap, ring_buffer); return NULL; } diff --git a/src/include/sof/audio/audio_buffer.h b/src/include/sof/audio/audio_buffer.h index e19937be156f..f9506a29b41f 100644 --- a/src/include/sof/audio/audio_buffer.h +++ b/src/include/sof/audio/audio_buffer.h @@ -110,6 +110,8 @@ struct sof_audio_buffer { * should not be in struct sof_audio_buffer at all, kept for pipeline2.0 transition */ bool walking; /**< indicates if the buffer is being walked */ + + struct k_heap *heap; }; #if CONFIG_PIPELINE_2_0 From 93b8fd902d472a79dbf22ceb9768d876ec16341c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 26 Sep 2025 13:59:09 +0200 Subject: [PATCH 7/9] ring-buffer: allocate the data buffer on module heap The ring-buffer data buffer has to be accessible to user-space DP modules, allocate it on module heap. Signed-off-by: Guennadi Liakhovetski --- src/audio/buffers/ring_buffer.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index e6334f55a45c..11ebd736dfe4 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -94,10 +94,10 @@ static void ring_buffer_free(struct sof_audio_buffer *audio_buffer) if (!audio_buffer) return; - struct ring_buffer *ring_buffer = - container_of(audio_buffer, struct ring_buffer, audio_buffer); + struct ring_buffer *ring_buffer = container_of(audio_buffer, + struct ring_buffer, audio_buffer); - rfree((__sparse_force void *)ring_buffer->_data_buffer); + sof_heap_free(audio_buffer->heap, (__sparse_force void *)ring_buffer->_data_buffer); sof_heap_free(audio_buffer->heap, ring_buffer); } @@ -362,12 +362,11 @@ struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_availabl ring_buffer->data_buffer_size = 3 * max_ibs_obs; /* allocate data buffer - always in cached memory alias */ - ring_buffer->data_buffer_size = - ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); - ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *) - rballoc_align(user_get_buffer_memory_region(dev->drv), - ring_buffer->data_buffer_size, - PLATFORM_DCACHE_ALIGN); + ring_buffer->data_buffer_size = ALIGN_UP(ring_buffer->data_buffer_size, + PLATFORM_DCACHE_ALIGN); + ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *)sof_heap_alloc(heap, + user_get_buffer_memory_region(dev->drv), + ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); if (!ring_buffer->_data_buffer) goto err; From 298a9db9244e90d342c5556835b2887d8b49c054 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 26 Sep 2025 15:54:43 +0200 Subject: [PATCH 8/9] fast-get: allocate on specific heap Pass a "heap" argument to fast_get() and fast_put() for user-space DP allocations. Signed-off-by: Guennadi Liakhovetski --- src/audio/module_adapter/module/generic.c | 4 +- src/include/sof/lib/fast-get.h | 5 ++- test/cmocka/src/lib/fast-get/fast-get-tests.c | 22 +++++----- test/ztest/unit/fast-get/CMakeLists.txt | 2 +- .../ztest/unit/fast-get/test_fast_get_ztest.c | 42 ++++++++++++------- zephyr/lib/fast-get.c | 8 ++-- 6 files changed, 49 insertions(+), 34 deletions(-) diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 1f0862d2a590..5b8eb8e43ee6 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -326,7 +326,7 @@ const void *mod_fast_get(struct processing_module *mod, const void * const dram_ if (!container) return NULL; - ptr = fast_get(dram_ptr, size); + ptr = fast_get(res->heap, dram_ptr, size); if (!ptr) { container_put(mod, container); return NULL; @@ -358,7 +358,7 @@ static int free_contents(struct processing_module *mod, struct module_resource * #endif #if CONFIG_FAST_GET case MOD_RES_FAST_GET: - fast_put(container->sram_ptr); + fast_put(res->heap, container->sram_ptr); return 0; #endif default: diff --git a/src/include/sof/lib/fast-get.h b/src/include/sof/lib/fast-get.h index ffbac19cf1b8..1748b6be222e 100644 --- a/src/include/sof/lib/fast-get.h +++ b/src/include/sof/lib/fast-get.h @@ -10,7 +10,8 @@ #include -const void *fast_get(const void * const dram_ptr, size_t size); -void fast_put(const void *sram_ptr); +struct k_heap; +const void *fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size); +void fast_put(struct k_heap *heap, const void *sram_ptr); #endif /* __SOF_LIB_FAST_GET_H__ */ diff --git a/test/cmocka/src/lib/fast-get/fast-get-tests.c b/test/cmocka/src/lib/fast-get/fast-get-tests.c index b7041abd827b..557b2c243425 100644 --- a/test/cmocka/src/lib/fast-get/fast-get-tests.c +++ b/test/cmocka/src/lib/fast-get/fast-get-tests.c @@ -72,12 +72,12 @@ static void test_simple_fast_get_put(void **state) (void)state; /* unused */ - ret = fast_get(testdata[0], sizeof(testdata[0])); + ret = fast_get(NULL, testdata[0], sizeof(testdata[0])); assert(ret); assert(!memcmp(ret, testdata[0], sizeof(testdata[0]))); - fast_put(ret); + fast_put(NULL, ret); } static void test_fast_get_size_missmatch_test(void **state) @@ -86,15 +86,15 @@ static void test_fast_get_size_missmatch_test(void **state) (void)state; /* unused */ - ret[0] = fast_get(testdata[0], sizeof(testdata[0])); + ret[0] = fast_get(NULL, testdata[0], sizeof(testdata[0])); assert(ret[0]); assert(!memcmp(ret[0], testdata[0], sizeof(testdata[0]))); - ret[1] = fast_get(testdata[0], sizeof(testdata[0]) + 1); + ret[1] = fast_get(NULL, testdata[0], sizeof(testdata[0]) + 1); assert(!ret[1]); - fast_put(ret); + fast_put(NULL, ret); } static void test_over_32_fast_gets_and_puts(void **state) @@ -105,13 +105,13 @@ static void test_over_32_fast_gets_and_puts(void **state) (void)state; /* unused */ for (i = 0; i < ARRAY_SIZE(copy); i++) - copy[i] = fast_get(testdata[i], sizeof(testdata[0])); + copy[i] = fast_get(NULL, testdata[i], sizeof(testdata[0])); for (i = 0; i < ARRAY_SIZE(copy); i++) assert(!memcmp(copy[i], testdata[i], sizeof(testdata[0]))); for (i = 0; i < ARRAY_SIZE(copy); i++) - fast_put(copy[i]); + fast_put(NULL, copy[i]); } static void test_fast_get_refcounting(void **state) @@ -121,10 +121,10 @@ static void test_fast_get_refcounting(void **state) (void)state; /* unused */ for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - copy[0][i] = fast_get(testdata[i], sizeof(testdata[0])); + copy[0][i] = fast_get(NULL, testdata[i], sizeof(testdata[0])); for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - copy[1][i] = fast_get(testdata[i], sizeof(testdata[0])); + copy[1][i] = fast_get(NULL, testdata[i], sizeof(testdata[0])); for (i = 0; i < ARRAY_SIZE(copy[0]); i++) assert(copy[0][i] == copy[1][i]); @@ -133,13 +133,13 @@ static void test_fast_get_refcounting(void **state) assert(!memcmp(copy[0][i], testdata[i], sizeof(testdata[0]))); for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - fast_put(copy[0][i]); + fast_put(NULL, copy[0][i]); for (i = 0; i < ARRAY_SIZE(copy[0]); i++) assert(!memcmp(copy[1][i], testdata[i], sizeof(testdata[0]))); for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - fast_put(copy[1][i]); + fast_put(NULL, copy[1][i]); } int main(void) diff --git a/test/ztest/unit/fast-get/CMakeLists.txt b/test/ztest/unit/fast-get/CMakeLists.txt index 45de96da341c..2e0b036c0358 100644 --- a/test/ztest/unit/fast-get/CMakeLists.txt +++ b/test/ztest/unit/fast-get/CMakeLists.txt @@ -25,7 +25,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/zephyr/lib/fast-get.c ) -target_link_libraries(app PRIVATE "-Wl,--wrap=rzalloc,--wrap=rmalloc,--wrap=rfree") +target_link_libraries(app PRIVATE "-Wl,--wrap=rzalloc,--wrap=rfree,--wrap=sof_heap_alloc,--wrap=sof_heap_free") # Add RELATIVE_FILE definitions for SOF trace functionality sof_append_relative_path_definitions(app) diff --git a/test/ztest/unit/fast-get/test_fast_get_ztest.c b/test/ztest/unit/fast-get/test_fast_get_ztest.c index f43f9c1bbd14..dedcc3f8630e 100644 --- a/test/ztest/unit/fast-get/test_fast_get_ztest.c +++ b/test/ztest/unit/fast-get/test_fast_get_ztest.c @@ -6,6 +6,7 @@ // generative artificial intelligence solutions. #include +#include #include #include @@ -71,20 +72,33 @@ void *__wrap_rzalloc(uint32_t flags, size_t bytes) return ret; } -void *__wrap_rmalloc(uint32_t flags, size_t bytes) +void __wrap_rfree(void *ptr) +{ + free(ptr); +} + +struct k_heap; +void *__wrap_sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, size_t alignment) { void *ret; + (void)flags; + (void)heap; - ret = malloc(bytes); + if (alignment) + ret = aligned_alloc(alignment, ALIGN_UP(bytes, alignment)); + else + ret = malloc(bytes); zassert_not_null(ret, "Memory allocation should not fail"); return ret; } -void __wrap_rfree(void *ptr) +void __wrap_sof_heap_free(struct k_heap *heap, void *ptr) { + (void)heap; + free(ptr); } @@ -98,13 +112,13 @@ ZTEST(fast_get_suite, test_simple_fast_get_put) { const void *ret; - ret = fast_get(testdata[0], sizeof(testdata[0])); + ret = fast_get(NULL, testdata[0], sizeof(testdata[0])); zassert_not_null(ret, "fast_get should return valid pointer"); zassert_mem_equal(ret, testdata[0], sizeof(testdata[0]), "Returned data should match original data"); - fast_put(ret); + fast_put(NULL, ret); } /** @@ -117,16 +131,16 @@ ZTEST(fast_get_suite, test_fast_get_size_missmatch_test) { const void *ret[2]; - ret[0] = fast_get(testdata[0], sizeof(testdata[0])); + ret[0] = fast_get(NULL, testdata[0], sizeof(testdata[0])); zassert_not_null(ret[0], "First fast_get should succeed"); zassert_mem_equal(ret[0], testdata[0], sizeof(testdata[0]), "Returned data should match original data"); - ret[1] = fast_get(testdata[0], sizeof(testdata[0]) + 1); + ret[1] = fast_get(NULL, testdata[0], sizeof(testdata[0]) + 1); zassert_is_null(ret[1], "fast_get with different size should return NULL"); - fast_put(ret[0]); + fast_put(NULL, ret[0]); } /** @@ -141,14 +155,14 @@ ZTEST(fast_get_suite, test_over_32_fast_gets_and_puts) int i; for (i = 0; i < ARRAY_SIZE(copy); i++) - copy[i] = fast_get(testdata[i], sizeof(testdata[0])); + copy[i] = fast_get(NULL, testdata[i], sizeof(testdata[0])); for (i = 0; i < ARRAY_SIZE(copy); i++) zassert_mem_equal(copy[i], testdata[i], sizeof(testdata[0]), "Data at index %d should match original", i); for (i = 0; i < ARRAY_SIZE(copy); i++) - fast_put(copy[i]); + fast_put(NULL, copy[i]); } /** @@ -164,10 +178,10 @@ ZTEST(fast_get_suite, test_fast_get_refcounting) int i; for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - copy[0][i] = fast_get(testdata[i], sizeof(testdata[0])); + copy[0][i] = fast_get(NULL, testdata[i], sizeof(testdata[0])); for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - copy[1][i] = fast_get(testdata[i], sizeof(testdata[0])); + copy[1][i] = fast_get(NULL, testdata[i], sizeof(testdata[0])); for (i = 0; i < ARRAY_SIZE(copy[0]); i++) zassert_equal_ptr(copy[0][i], copy[1][i], @@ -179,7 +193,7 @@ ZTEST(fast_get_suite, test_fast_get_refcounting) /* Release first set of references */ for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - fast_put(copy[0][i]); + fast_put(NULL, copy[0][i]); /* Data should still be valid through second set of references */ for (i = 0; i < ARRAY_SIZE(copy[0]); i++) @@ -188,7 +202,7 @@ ZTEST(fast_get_suite, test_fast_get_refcounting) /* Release second set of references */ for (i = 0; i < ARRAY_SIZE(copy[0]); i++) - fast_put(copy[1][i]); + fast_put(NULL, copy[1][i]); } /** diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index 3be9475ff90c..cb5e5a4823fc 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -80,7 +80,7 @@ static struct sof_fast_get_entry *fast_get_find_entry(struct sof_fast_get_data * return NULL; } -const void *fast_get(const void *dram_ptr, size_t size) +const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) { struct sof_fast_get_data *data = &fast_get_data; struct sof_fast_get_entry *entry; @@ -116,7 +116,7 @@ const void *fast_get(const void *dram_ptr, size_t size) goto out; } - ret = rmalloc(SOF_MEM_FLAG_USER, size); + ret = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, size, 0); if (!ret) goto out; entry->size = size; @@ -146,7 +146,7 @@ static struct sof_fast_get_entry *fast_put_find_entry(struct sof_fast_get_data * return NULL; } -void fast_put(const void *sram_ptr) +void fast_put(struct k_heap *heap, const void *sram_ptr) { struct sof_fast_get_data *data = &fast_get_data; struct sof_fast_get_entry *entry; @@ -160,7 +160,7 @@ void fast_put(const void *sram_ptr) } entry->refcount--; if (!entry->refcount) { - rfree(entry->sram_ptr); + sof_heap_free(heap, entry->sram_ptr); memset(entry, 0, sizeof(*entry)); } out: From 879ba45307d318ea17f91cb955a379e275d0c72a Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 26 Sep 2025 16:32:30 +0200 Subject: [PATCH 9/9] fast-get: convert to syscalls Convert fast_get() and fast_put() to syscalls. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib/fast-get.h | 15 +++++++++++++-- zephyr/CMakeLists.txt | 2 ++ zephyr/lib/fast-get.c | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/include/sof/lib/fast-get.h b/src/include/sof/lib/fast-get.h index 1748b6be222e..72e60a8a68a3 100644 --- a/src/include/sof/lib/fast-get.h +++ b/src/include/sof/lib/fast-get.h @@ -11,7 +11,18 @@ #include struct k_heap; -const void *fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size); -void fast_put(struct k_heap *heap, const void *sram_ptr); + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF) +#include + +__syscall const void *fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size); +__syscall void fast_put(struct k_heap *heap, const void *sram_ptr); +#include +#else +const void *z_impl_fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size); +void z_impl_fast_put(struct k_heap *heap, const void *sram_ptr); +#define fast_get z_impl_fast_get +#define fast_put z_impl_fast_put +#endif /* __ZEPHYR__ */ #endif /* __SOF_LIB_FAST_GET_H__ */ diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 4ef481d8c52d..d39bbf7e88e8 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -529,6 +529,8 @@ zephyr_library_sources_ifdef(CONFIG_SHELL sof_shell.c ) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) + zephyr_library_link_libraries(SOF) target_link_libraries(SOF INTERFACE zephyr_interface) diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index cb5e5a4823fc..d23616541745 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -80,7 +80,7 @@ static struct sof_fast_get_entry *fast_get_find_entry(struct sof_fast_get_data * return NULL; } -const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) +const void *z_impl_fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) { struct sof_fast_get_data *data = &fast_get_data; struct sof_fast_get_entry *entry; @@ -131,7 +131,7 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) return ret; } -EXPORT_SYMBOL(fast_get); +EXPORT_SYMBOL(z_impl_fast_get); static struct sof_fast_get_entry *fast_put_find_entry(struct sof_fast_get_data *data, const void *sram_ptr) @@ -146,7 +146,7 @@ static struct sof_fast_get_entry *fast_put_find_entry(struct sof_fast_get_data * return NULL; } -void fast_put(struct k_heap *heap, const void *sram_ptr) +void z_impl_fast_put(struct k_heap *heap, const void *sram_ptr) { struct sof_fast_get_data *data = &fast_get_data; struct sof_fast_get_entry *entry; @@ -168,4 +168,30 @@ void fast_put(struct k_heap *heap, const void *sram_ptr) entry ? entry->size : 0, entry ? entry->refcount : 0); k_spin_unlock(&data->lock, key); } -EXPORT_SYMBOL(fast_put); +EXPORT_SYMBOL(z_impl_fast_put); + +#ifdef CONFIG_USERSPACE +void z_vrfy_fast_put(struct k_heap *heap, const void *sram_ptr) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(heap, sizeof(*heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(heap->heap.heap, sizeof(*heap->heap.heap))); + /* + * FIXME: we don't know how much SRAM has been allocated, so cannot + * check. Should fast_put() be changed to pass a size argument? + */ + + z_impl_fast_put(heap, sram_ptr); +} +#include + +const void *z_vrfy_fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(heap, sizeof(*heap))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(heap->heap.heap, sizeof(*heap->heap.heap))); + /* We cannot (easily) verify the actual heapp memory */ + K_OOPS(K_SYSCALL_MEMORY_READ(dram_ptr, size)); + + return z_impl_fast_get(heap, dram_ptr, size); +} +#include +#endif