diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index f05f2d97d20b..398003300e90 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -83,7 +83,6 @@ CONFIG_WATCHDOG=y CONFIG_TIMESLICE_PER_THREAD=y CONFIG_THREAD_RUNTIME_STATS=y CONFIG_SCHED_THREAD_USAGE=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 # Zephyr / device drivers CONFIG_CLOCK_CONTROL=y @@ -103,6 +102,7 @@ CONFIG_INTEL_ADSP_IPC=y CONFIG_INTEL_ADSP_TIMER=y CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=38400000 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace20_lnl.conf b/app/boards/intel_adsp_ace20_lnl.conf index 53cc9e1a1b0b..578d3e4ad225 100644 --- a/app/boards/intel_adsp_ace20_lnl.conf +++ b/app/boards/intel_adsp_ace20_lnl.conf @@ -63,7 +63,6 @@ CONFIG_LLEXT_STORAGE_WRITABLE=y CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y CONFIG_TIMING_FUNCTIONS=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 # Zephyr / device drivers CONFIG_CLOCK_CONTROL=y @@ -82,6 +81,7 @@ CONFIG_INTEL_ADSP_IPC=y CONFIG_INTEL_ADSP_TIMER=y CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=38400000 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index f84fff54b51d..f7f7296bbce9 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -63,7 +63,6 @@ CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 # Zephyr / device drivers CONFIG_CLOCK_CONTROL=y @@ -83,6 +82,7 @@ CONFIG_INTEL_ADSP_IPC=y CONFIG_INTEL_ADSP_TIMER=y CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings @@ -95,6 +95,7 @@ CONFIG_PM_DEVICE_SYSTEM_MANAGED=y CONFIG_PM_PREWAKEUP_CONV_MODE_CEIL=y CONFIG_POWER_DOMAIN=y CONFIG_POWER_DOMAIN_INTEL_ADSP=y +CONFIG_USERSPACE=y # Zephyr / logging CONFIG_LOG=y diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index f96bfa913677..504836c40f06 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -50,7 +50,6 @@ CONFIG_LLEXT=y CONFIG_LLEXT_STORAGE_WRITABLE=y CONFIG_LLEXT_EXPERIMENTAL=y CONFIG_MODULES=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 # Zephyr / device drivers CONFIG_CLOCK_CONTROL=y @@ -68,6 +67,7 @@ CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_INTEL_ADSP_IPC=y CONFIG_INTEL_ADSP_TIMER=y CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings diff --git a/app/prj.conf b/app/prj.conf index e072f1df2fd2..5ee884d43302 100644 --- a/app/prj.conf +++ b/app/prj.conf @@ -49,3 +49,8 @@ CONFIG_SCHED_CPU_MASK=y CONFIG_SYS_CLOCK_TICKS_PER_SEC=15000 CONFIG_DAI=y CONFIG_HEAP_MEM_POOL_SIZE=2048 + +# Support dynamic thread stack allocation +CONFIG_DYNAMIC_THREAD=y +CONFIG_DYNAMIC_THREAD_ALLOC=y +CONFIG_DYNAMIC_THREAD_PREFER_ALLOC=y diff --git a/src/audio/aria/aria.c b/src/audio/aria/aria.c index 2f558eb2d6d9..5981171369fe 100644 --- a/src/audio/aria/aria.c +++ b/src/audio/aria/aria.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -126,7 +125,7 @@ static int aria_init(struct processing_module *mod) list_init(&dev->bsource_list); list_init(&dev->bsink_list); - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = mod_zalloc(mod, sizeof(*cd)); if (!cd) { return -ENOMEM; } @@ -145,10 +144,9 @@ static int aria_init(struct processing_module *mod) } mod_data->private = cd; - buf = rballoc(SOF_MEM_FLAG_USER, req_mem); + buf = mod_alloc(mod, req_mem); if (!buf) { - rfree(cd); comp_err(dev, "allocation failed for size %d", req_mem); return -ENOMEM; } @@ -158,10 +156,6 @@ static int aria_init(struct processing_module *mod) static int aria_free(struct processing_module *mod) { - struct aria_data *cd = module_get_private_data(mod); - - rfree(cd->data_addr); - rfree(cd); return 0; } @@ -320,10 +314,8 @@ static const struct module_interface aria_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(aria, &aria_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("ARIA", aria_llext_entry, 1, SOF_REG_UUID(aria), 8); + SOF_LLEXT_MODULE_MANIFEST("ARIA", &aria_interface, 1, SOF_REG_UUID(aria), 8); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/asrc/asrc.c b/src/audio/asrc/asrc.c index 2d77a04c380f..9d13dfb55e92 100644 --- a/src/audio/asrc/asrc.c +++ b/src/audio/asrc/asrc.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -217,7 +216,7 @@ static int asrc_init(struct processing_module *mod) return -EINVAL; } - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = mod_alloc(mod, sizeof(*cd)); if (!cd) return -ENOMEM; @@ -242,7 +241,7 @@ static int asrc_init(struct processing_module *mod) return 0; } -static int asrc_initialize_buffers(struct asrc_farrow *src_obj) +static int asrc_initialize_buffers(struct processing_module *mod, struct asrc_farrow *src_obj) { int32_t *buf_32; int16_t *buf_16; @@ -261,7 +260,7 @@ static int asrc_initialize_buffers(struct asrc_farrow *src_obj) buffer_size = src_obj->buffer_length * sizeof(int32_t); for (ch = 0; ch < src_obj->num_channels; ch++) { - buf_32 = rzalloc(SOF_MEM_FLAG_USER, buffer_size); + buf_32 = mod_zalloc(mod, buffer_size); if (!buf_32) return -ENOMEM; @@ -272,7 +271,7 @@ static int asrc_initialize_buffers(struct asrc_farrow *src_obj) buffer_size = src_obj->buffer_length * sizeof(int16_t); for (ch = 0; ch < src_obj->num_channels; ch++) { - buf_16 = rzalloc(SOF_MEM_FLAG_USER, buffer_size); + buf_16 = mod_zalloc(mod, buffer_size); if (!buf_16) return -ENOMEM; @@ -284,7 +283,7 @@ static int asrc_initialize_buffers(struct asrc_farrow *src_obj) return 0; } -static void asrc_release_buffers(struct asrc_farrow *src_obj) +static void asrc_release_buffers(struct processing_module *mod, struct asrc_farrow *src_obj) { int32_t *buf_32; int16_t *buf_16; @@ -299,7 +298,7 @@ static void asrc_release_buffers(struct asrc_farrow *src_obj) if (buf_32) { src_obj->ring_buffers32[ch] = NULL; - rfree(buf_32); + mod_free(mod, buf_32); } } else @@ -308,23 +307,17 @@ static void asrc_release_buffers(struct asrc_farrow *src_obj) if (buf_16) { src_obj->ring_buffers16[ch] = NULL; - rfree(buf_16); + mod_free(mod, buf_16); } } } static int asrc_free(struct processing_module *mod) { - struct comp_data *cd = module_get_private_data(mod); struct comp_dev *dev = mod->dev; comp_dbg(dev, "asrc_free()"); - rfree(cd->buf); - asrc_release_buffers(cd->asrc_obj); - asrc_free_polyphase_filter(cd->asrc_obj); - rfree(cd->asrc_obj); - rfree(cd); return 0; } @@ -614,8 +607,7 @@ static int asrc_prepare(struct processing_module *mod, cd->buf_size = (cd->source_frames_max + cd->sink_frames_max) * frame_bytes; - cd->buf = rzalloc(SOF_MEM_FLAG_USER, - cd->buf_size); + cd->buf = mod_zalloc(mod, cd->buf_size); if (!cd->buf) { cd->buf_size = 0; comp_err(dev, "asrc_prepare(), allocation fail for size %d", @@ -632,7 +624,7 @@ static int asrc_prepare(struct processing_module *mod, /* Get required size and allocate memory for ASRC */ sample_bits = sample_bytes * 8; - ret = asrc_get_required_size(dev, &cd->asrc_size, + ret = asrc_get_required_size(mod, &cd->asrc_size, audio_stream_get_channels(&sourceb->stream), sample_bits); if (ret) { @@ -640,8 +632,7 @@ static int asrc_prepare(struct processing_module *mod, goto err_free_buf; } - cd->asrc_obj = rzalloc(SOF_MEM_FLAG_USER, - cd->asrc_size); + cd->asrc_obj = mod_zalloc(mod, cd->asrc_size); if (!cd->asrc_obj) { comp_err(dev, "asrc_prepare(), allocation fail for size %d", cd->asrc_size); @@ -659,7 +650,7 @@ static int asrc_prepare(struct processing_module *mod, fs_sec = cd->source_rate; } - ret = asrc_initialise(dev, cd->asrc_obj, audio_stream_get_channels(&sourceb->stream), + ret = asrc_initialise(mod, cd->asrc_obj, audio_stream_get_channels(&sourceb->stream), fs_prim, fs_sec, ASRC_IOF_INTERLEAVED, ASRC_IOF_INTERLEAVED, ASRC_BM_LINEAR, cd->frames, sample_bits, @@ -670,7 +661,7 @@ static int asrc_prepare(struct processing_module *mod, } /* Allocate ring buffers */ - ret = asrc_initialize_buffers(cd->asrc_obj); + ret = asrc_initialize_buffers(mod, cd->asrc_obj); /* check for errors */ if (ret) { @@ -698,12 +689,12 @@ static int asrc_prepare(struct processing_module *mod, return 0; err_free_asrc: - asrc_release_buffers(cd->asrc_obj); - rfree(cd->asrc_obj); + asrc_release_buffers(mod, cd->asrc_obj); + mod_free(mod, cd->asrc_obj); cd->asrc_obj = NULL; err_free_buf: - rfree(cd->buf); + mod_free(mod, cd->buf); cd->buf = NULL; err: @@ -865,10 +856,10 @@ static int asrc_reset(struct processing_module *mod) asrc_dai_stop_timestamp(cd); /* Free the allocations those were done in prepare() */ - asrc_release_buffers(cd->asrc_obj); - asrc_free_polyphase_filter(cd->asrc_obj); - rfree(cd->asrc_obj); - rfree(cd->buf); + asrc_release_buffers(mod, cd->asrc_obj); + asrc_free_polyphase_filter(mod, cd->asrc_obj); + mod_free(mod, cd->asrc_obj); + mod_free(mod, cd->buf); cd->asrc_obj = NULL; cd->buf = NULL; @@ -892,10 +883,8 @@ static const struct module_interface asrc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(asrc, &asrc_interface); - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("ASRC", asrc_llext_entry, 1, SOF_REG_UUID(asrc4), 2), + SOF_LLEXT_MODULE_MANIFEST("ASRC", &asrc_interface, 1, SOF_REG_UUID(asrc4), 2), }; SOF_LLEXT_BUILDINFO; diff --git a/src/audio/asrc/asrc_farrow.c b/src/audio/asrc/asrc_farrow.c index 987a11408af0..e840a8082a3b 100644 --- a/src/audio/asrc/asrc_farrow.c +++ b/src/audio/asrc/asrc_farrow.c @@ -11,9 +11,9 @@ #include #include #include -#include #include #include +#include #include "asrc_farrow.h" LOG_MODULE_DECLARE(asrc, CONFIG_SOF_LOG_LEVEL); @@ -243,7 +243,7 @@ static const struct asrc_filter_params c_filter_params[CR_NUM] = { * Initialise the pointers to the filters, set the number of filters * and their length */ -static enum asrc_error_code initialise_filter(struct comp_dev *dev, +static enum asrc_error_code initialise_filter(struct processing_module *mod, struct asrc_farrow *src_obj); /* @@ -268,11 +268,12 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, * Pointers to each channels data. Buffers are allocated externally. */ -enum asrc_error_code asrc_get_required_size(struct comp_dev *dev, +enum asrc_error_code asrc_get_required_size(struct processing_module *mod, int *required_size, int num_channels, int bit_depth) { + struct comp_dev *dev = mod->dev; int size; /* check for parameter errors */ @@ -318,7 +319,7 @@ enum asrc_error_code asrc_get_required_size(struct comp_dev *dev, return ASRC_EC_OK; } -enum asrc_error_code asrc_initialise(struct comp_dev *dev, +enum asrc_error_code asrc_initialise(struct processing_module *mod, struct asrc_farrow *src_obj, int num_channels, int32_t fs_prim, @@ -331,6 +332,7 @@ enum asrc_error_code asrc_initialise(struct comp_dev *dev, enum asrc_control_mode control_mode, enum asrc_operation_mode operation_mode) { + struct comp_dev *dev = mod->dev; enum asrc_error_code error_code; /* check for parameter errors */ @@ -410,7 +412,7 @@ enum asrc_error_code asrc_initialise(struct comp_dev *dev, * also sets the pointer to the corresponding * calc_impulse_response_nX function. */ - error_code = initialise_filter(dev, src_obj); + error_code = initialise_filter(mod, src_obj); /* check for errors */ if (error_code != ASRC_EC_OK) { @@ -438,10 +440,12 @@ enum asrc_error_code asrc_initialise(struct comp_dev *dev, return ASRC_EC_OK; } -enum asrc_error_code asrc_set_fs_ratio(struct comp_dev *dev, +enum asrc_error_code asrc_set_fs_ratio(struct processing_module *mod, struct asrc_farrow *src_obj, int32_t fs_prim, int32_t fs_sec) { + struct comp_dev *dev = mod->dev; + /* Check for parameter errors */ if (!src_obj) { comp_err(dev, "asrc_set_fs_ratio(), null src_obj"); @@ -490,7 +494,7 @@ enum asrc_error_code asrc_set_fs_ratio(struct comp_dev *dev, /* See initialise_asrc(...) for further information * Update the filters accordingly */ - enum asrc_error_code error_code = initialise_filter(dev, src_obj); + enum asrc_error_code error_code = initialise_filter(mod, src_obj); /* check for errors */ if (error_code != ASRC_EC_OK) { comp_err(dev, "asrc_set_fs_ratio(), failed filter initialise"); @@ -554,28 +558,29 @@ enum asrc_error_code asrc_set_output_format(struct comp_dev *dev, return ASRC_EC_OK; } -static const int32_t *__get_polyphase_filter(const int32_t *filter, size_t size) +static const int32_t *__get_polyphase_filter(struct processing_module *mod, + const int32_t *filter, size_t size) { #if CONFIG_FAST_GET - return fast_get(filter, size); + return mod_fast_get(mod, filter, size); #else return filter; #endif } -#define get_polyphase_filter(f) __get_polyphase_filter(f, sizeof(f)) +#define get_polyphase_filter(m, f) __get_polyphase_filter(m, f, sizeof(f)) -static void put_polyphase_filter(const int32_t *filter) +static void put_polyphase_filter(struct processing_module *mod, const int32_t *filter) { #if CONFIG_FAST_GET - fast_put(filter); + mod_fast_put(mod, filter); #endif } -void asrc_free_polyphase_filter(struct asrc_farrow *src_obj) +void asrc_free_polyphase_filter(struct processing_module *mod, struct asrc_farrow *src_obj) { if (src_obj && src_obj->polyphase_filters) { - put_polyphase_filter(src_obj->polyphase_filters); + put_polyphase_filter(mod, src_obj->polyphase_filters); src_obj->polyphase_filters = NULL; } } @@ -583,9 +588,10 @@ void asrc_free_polyphase_filter(struct asrc_farrow *src_obj) /* * FILTER FUNCTIONS */ -static enum asrc_error_code initialise_filter(struct comp_dev *dev, +static enum asrc_error_code initialise_filter(struct processing_module *mod, struct asrc_farrow *src_obj) { + struct comp_dev *dev = mod->dev; int fs_in; int fs_out; @@ -606,7 +612,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, /* Reset coefficients for possible exit with error. */ src_obj->filter_length = 0; src_obj->num_filters = 0; - asrc_free_polyphase_filter(src_obj); + asrc_free_polyphase_filter(mod, src_obj); if (fs_in == 0 || fs_out == 0) { /* Avoid possible divisions by zero. */ @@ -622,7 +628,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO48000].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO48000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to48000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to48000); } else if (fs_in <= fs_out) { /* All upsampling use cases can share the same set of * filter coefficients. @@ -631,7 +637,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_44100TO48000].filter_length; src_obj->num_filters = c_filter_params[CR_44100TO48000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff44100to48000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff44100to48000); } else if (fs_in == 48000) { switch (fs_out) { #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_08000) @@ -640,7 +646,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO08000].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO08000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to08000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to08000); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_11025) @@ -649,7 +655,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO11025].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO11025].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to11025); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to11025); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_12000) @@ -658,7 +664,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO12000].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO12000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to12000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to12000); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_16000) @@ -667,7 +673,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO16000].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO16000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to16000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to16000); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_22050) @@ -676,7 +682,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO22050].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO22050].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to22050); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to22050); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_24000) @@ -685,7 +691,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO24000].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO24000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to24000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to24000); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_32000) @@ -694,7 +700,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO32000].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO32000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to32000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to32000); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_48000_TO_44100) @@ -703,7 +709,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_48000TO44100].filter_length; src_obj->num_filters = c_filter_params[CR_48000TO44100].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff48000to44100); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff48000to44100); break; #endif default: @@ -719,7 +725,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_24000TO08000].filter_length; src_obj->num_filters = c_filter_params[CR_24000TO08000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff24000to08000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff24000to08000); break; #endif #if (CONFIG_ASRC_SUPPORT_CONVERSION_24000_TO_16000) @@ -728,7 +734,7 @@ static enum asrc_error_code initialise_filter(struct comp_dev *dev, c_filter_params[CR_24000TO16000].filter_length; src_obj->num_filters = c_filter_params[CR_24000TO16000].num_filters; - src_obj->polyphase_filters = get_polyphase_filter(coeff24000to16000); + src_obj->polyphase_filters = get_polyphase_filter(mod, coeff24000to16000); break; #endif default: diff --git a/src/audio/asrc/asrc_farrow.h b/src/audio/asrc/asrc_farrow.h index 766ead172d58..48e564c2d1e1 100644 --- a/src/audio/asrc/asrc_farrow.h +++ b/src/audio/asrc/asrc_farrow.h @@ -231,7 +231,7 @@ struct asrc_farrow { * @param[in] bit_depth The wordlength that will be used for representing * the PCM samples, must be 16 or 32. */ -enum asrc_error_code asrc_get_required_size(struct comp_dev *dev, +enum asrc_error_code asrc_get_required_size(struct processing_module *mod, int *required_size, int num_channels, int bit_depth); @@ -268,7 +268,7 @@ enum asrc_error_code asrc_get_required_size(struct comp_dev *dev, * @param[in] operation_mode Choose 'push' or 'pull', depending on the mode * you want your ASRC to operate in. */ -enum asrc_error_code asrc_initialise(struct comp_dev *dev, +enum asrc_error_code asrc_initialise(struct processing_module *mod, struct asrc_farrow *src_obj, int num_channels, int32_t fs_prim, @@ -286,7 +286,7 @@ enum asrc_error_code asrc_initialise(struct comp_dev *dev, * * @param[in] src_obj Pointer to the ias_src_farrow. */ -void asrc_free_polyphase_filter(struct asrc_farrow *src_obj); +void asrc_free_polyphase_filter(struct processing_module *mod, struct asrc_farrow *src_obj); /* * @brief Process the sample rate converter for one frame; the frame @@ -591,7 +591,7 @@ enum asrc_error_code asrc_update_fs_ratio(struct comp_dev *dev, * @param[in] fs_prim Primary sampling rate. * @param[in] fs_sec Secondary sampling rate. */ -enum asrc_error_code asrc_set_fs_ratio(struct comp_dev *dev, +enum asrc_error_code asrc_set_fs_ratio(struct processing_module *mod, struct asrc_farrow *src_obj, int32_t fs_prim, int32_t fs_sec); diff --git a/src/audio/asrc/asrc_ipc3.c b/src/audio/asrc/asrc_ipc3.c index 30dfbecddc6b..ca917318d7e5 100644 --- a/src/audio/asrc/asrc_ipc3.c +++ b/src/audio/asrc/asrc_ipc3.c @@ -4,7 +4,6 @@ #include #include -#include #include #include #include diff --git a/src/audio/asrc/asrc_ipc4.c b/src/audio/asrc/asrc_ipc4.c index 3e5d8bf7818f..1047799f207b 100644 --- a/src/audio/asrc/asrc_ipc4.c +++ b/src/audio/asrc/asrc_ipc4.c @@ -4,7 +4,6 @@ #include #include -#include #include #include #include diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8b21b06c454d..484ea878234c 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -157,8 +158,16 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) /* In case some listeners didn't unregister from buffer's callbacks */ notifier_unregister_all(NULL, buffer); + struct k_heap *heap = buffer->audio_buffer.heap; + rfree(buffer->stream.addr); - rfree(buffer); + sof_heap_free(heap, buffer); + if (heap) { + struct dp_heap_user *mod_heap_user = CONTAINER_OF(heap, struct dp_heap_user, heap); + + if (!--mod_heap_user->client_count) + rfree(mod_heap_user); + } } static const struct source_ops comp_buffer_source_ops = { @@ -187,7 +196,8 @@ static const struct audio_buffer_ops audio_buffer_ops = { .set_alignment_constants = comp_buffer_set_alignment_constants }; -static struct comp_buffer *buffer_alloc_struct(void *stream_addr, size_t size, +static struct comp_buffer *buffer_alloc_struct(struct k_heap *heap, + void *stream_addr, size_t size, uint32_t flags, bool is_shared) { struct comp_buffer *buffer; @@ -198,13 +208,14 @@ static struct comp_buffer *buffer_alloc_struct(void *stream_addr, size_t size, if (is_shared) flags |= SOF_MEM_FLAG_COHERENT; - buffer = rzalloc(flags, sizeof(*buffer)); - + buffer = sof_heap_alloc(heap, flags, sizeof(*buffer), 0); if (!buffer) { tr_err(&buffer_tr, "could not alloc structure"); return NULL; } + memset(buffer, 0, sizeof(*buffer)); + buffer->flags = flags; /* Force channels to 2 for init to prevent bad call to clz in buffer_init_stream */ buffer->stream.runtime_stream_params.channels = 2; @@ -219,6 +230,7 @@ static struct comp_buffer *buffer_alloc_struct(void *stream_addr, size_t size, audio_stream_set_underrun(&buffer->stream, !!(flags & SOF_BUF_UNDERRUN_PERMITTED)); audio_stream_set_overrun(&buffer->stream, !!(flags & SOF_BUF_OVERRUN_PERMITTED)); + buffer->audio_buffer.heap = heap; comp_buffer_reset_source_list(buffer); comp_buffer_reset_sink_list(buffer); @@ -226,7 +238,7 @@ static struct comp_buffer *buffer_alloc_struct(void *stream_addr, size_t size, return buffer; } -struct comp_buffer *buffer_alloc(size_t size, uint32_t flags, uint32_t align, +struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flags, uint32_t align, bool is_shared) { struct comp_buffer *buffer; @@ -247,7 +259,7 @@ struct comp_buffer *buffer_alloc(size_t size, uint32_t flags, uint32_t align, return NULL; } - buffer = buffer_alloc_struct(stream_addr, size, flags, is_shared); + buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared); if (!buffer) { tr_err(&buffer_tr, "could not alloc buffer structure"); rfree(stream_addr); @@ -256,7 +268,8 @@ struct comp_buffer *buffer_alloc(size_t size, uint32_t flags, uint32_t align, return buffer; } -struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_size, +struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_size, + size_t minimum_size, uint32_t flags, uint32_t align, bool is_shared) { struct comp_buffer *buffer; @@ -277,7 +290,7 @@ struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_siz preferred_size += minimum_size - preferred_size % minimum_size; for (size = preferred_size; size >= minimum_size; size -= minimum_size) { - stream_addr = rballoc_align(flags, size, align); + stream_addr = sof_heap_alloc(heap, flags, size, align); if (stream_addr) break; } @@ -290,7 +303,7 @@ struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_siz return NULL; } - buffer = buffer_alloc_struct(stream_addr, size, flags, is_shared); + buffer = buffer_alloc_struct(heap, stream_addr, size, flags, is_shared); if (!buffer) { tr_err(&buffer_tr, "could not alloc buffer structure"); rfree(stream_addr); diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index b6cbc23060e2..5111414047b2 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -95,8 +95,8 @@ static void ring_buffer_free(struct sof_audio_buffer *audio_buffer) struct ring_buffer *ring_buffer = container_of(audio_buffer, struct ring_buffer, audio_buffer); - rfree((__sparse_force void *)ring_buffer->_data_buffer); - rfree(ring_buffer); + sof_heap_free(audio_buffer->heap, (__sparse_force void *)ring_buffer->_data_buffer); + sof_heap_free(audio_buffer->heap, ring_buffer); } static void ring_buffer_reset(struct sof_audio_buffer *audio_buffer) @@ -279,20 +279,17 @@ static const struct audio_buffer_ops audio_buffer_ops = { .reset = ring_buffer_reset }; -struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, bool is_shared, - uint32_t id) +struct ring_buffer *ring_buffer_create(struct k_heap *heap, size_t min_available, + size_t min_free_space, bool is_shared, uint32_t id) { - struct ring_buffer *ring_buffer; - - /* allocate ring_buffer structure */ - if (is_shared) - ring_buffer = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ring_buffer)); - else - ring_buffer = rzalloc(SOF_MEM_FLAG_USER, sizeof(*ring_buffer)); + uint32_t flags = is_shared ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; + struct ring_buffer *ring_buffer = sof_heap_alloc(heap, flags, sizeof(*ring_buffer), 0); + if (!ring_buffer) return NULL; + memset(ring_buffer, 0, sizeof(*ring_buffer)); + /* init base structure. The audio_stream_params is NULL because ring_buffer * is currently used as a secondary buffer for DP only * @@ -302,6 +299,7 @@ struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_spa audio_buffer_init(&ring_buffer->audio_buffer, BUFFER_TYPE_RING_BUFFER, is_shared, &ring_buffer_source_ops, &ring_buffer_sink_ops, &audio_buffer_ops, NULL); + ring_buffer->audio_buffer.heap = heap; /* set obs/ibs in sink/source interfaces */ sink_set_min_free_space(audio_buffer_get_sink(&ring_buffer->audio_buffer), @@ -356,10 +354,11 @@ struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_spa ring_buffer->data_buffer_size = 3 * max_ibs_obs; /* allocate data buffer - always in cached memory alias */ - ring_buffer->data_buffer_size = - ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); + ring_buffer->data_buffer_size = ALIGN_UP(ring_buffer->data_buffer_size, + PLATFORM_DCACHE_ALIGN); ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *) - rballoc_align(SOF_MEM_FLAG_USER, ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); + sof_heap_alloc(heap, SOF_MEM_FLAG_USER, ring_buffer->data_buffer_size, + PLATFORM_DCACHE_ALIGN); if (!ring_buffer->_data_buffer) goto err; @@ -371,6 +370,6 @@ struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_spa return ring_buffer; err: tr_err(&ring_buffer_tr, "Ring buffer creation failure"); - rfree(ring_buffer); + sof_heap_free(heap, ring_buffer); return NULL; } diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c index 29353428e846..3717536abbe1 100644 --- a/src/audio/chain_dma.c +++ b/src/audio/chain_dma.c @@ -591,7 +591,7 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align); /* allocate not shared buffer */ - cd->dma_buffer = buffer_alloc(fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, false); if (!cd->dma_buffer) { diff --git a/src/audio/codec/dts/dts.c b/src/audio/codec/dts/dts.c index cae6aeb0938b..0bc590017004 100644 --- a/src/audio/codec/dts/dts.c +++ b/src/audio/codec/dts/dts.c @@ -473,10 +473,8 @@ static const struct module_interface dts_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(dts, &dts_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("DTS", dts_llext_entry, 1, SOF_REG_UUID(dts), 40); + SOF_LLEXT_MODULE_MANIFEST("DTS", &dts_interface, 1, SOF_REG_UUID(dts), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/copier/copier.c b/src/audio/copier/copier.c index 9aab0858a305..c4a1b1097429 100644 --- a/src/audio/copier/copier.c +++ b/src/audio/copier/copier.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -82,13 +81,12 @@ static void mic_privacy_event(void *arg, enum notify_id type, void *data) } } -static int mic_privacy_configure(struct comp_dev *dev, struct copier_data *cd) +static int mic_privacy_configure(struct processing_module *mod, struct copier_data *cd) { struct mic_privacy_data *mic_priv_data; int ret; - mic_priv_data = rzalloc(SOF_MEM_FLAG_USER, - sizeof(struct mic_privacy_data)); + mic_priv_data = mod_zalloc(mod, sizeof(struct mic_privacy_data)); if (!mic_priv_data) return -ENOMEM; @@ -100,7 +98,7 @@ static int mic_privacy_configure(struct comp_dev *dev, struct copier_data *cd) uint32_t zeroing_wait_time = (mic_privacy_get_dma_zeroing_wait_time() * 1000) / ADSP_RTC_FREQUENCY; - ret = copier_gain_set_params(dev, &mic_priv_data->mic_priv_gain_params, + ret = copier_gain_set_params(mod->dev, &mic_priv_data->mic_priv_gain_params, zeroing_wait_time, SOF_DAI_INTEL_NONE); if (ret != 0) { rfree(mic_priv_data); @@ -144,7 +142,7 @@ __cold static int copier_init(struct processing_module *mod) assert_can_be_cold(); - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = mod_zalloc(mod, sizeof(*cd)); if (!cd) return -ENOMEM; @@ -154,10 +152,8 @@ __cold static int copier_init(struct processing_module *mod) * store it, it's only used during IPC processing, besides we haven't * allocated space for it, so don't "fix" this! */ - if (memcpy_s(&cd->config, sizeof(cd->config), copier, sizeof(*copier)) < 0) { - ret = -EINVAL; - goto error_cd; - } + if (memcpy_s(&cd->config, sizeof(cd->config), copier, sizeof(*copier)) < 0) + return -EINVAL; /* Allocate memory and store gateway_cfg in runtime. Gateway cfg has to * be kept even after copier is created e.g. during SET_PIPELINE_STATE @@ -166,18 +162,15 @@ __cold static int copier_init(struct processing_module *mod) */ if (copier->gtw_cfg.config_length) { gtw_cfg_size = copier->gtw_cfg.config_length << 2; - gtw_cfg = rmalloc(SOF_MEM_FLAG_USER, - gtw_cfg_size); - if (!gtw_cfg) { - ret = -ENOMEM; - goto error_cd; - } + gtw_cfg = mod_alloc(mod, gtw_cfg_size); + if (!gtw_cfg) + return -ENOMEM; ret = memcpy_s(gtw_cfg, gtw_cfg_size, &copier->gtw_cfg.config_data, gtw_cfg_size); if (ret) { comp_err(dev, "Unable to copy gateway config from copier blob"); - goto error; + return ret; } cd->gtw_cfg = gtw_cfg; @@ -191,8 +184,7 @@ __cold static int copier_init(struct processing_module *mod) IPC_COMP_IGNORE_REMOTE); if (!ipc_pipe) { comp_err(dev, "pipeline %d is not existed", config->pipeline_id); - ret = -EPIPE; - goto error; + return -EPIPE; } dev->pipeline = ipc_pipe->pipeline; @@ -208,15 +200,15 @@ __cold static int copier_init(struct processing_module *mod) ret = copier_host_create(dev, cd, copier, ipc_pipe->pipeline); if (ret < 0) { comp_err(dev, "unable to create host"); - goto error; + return ret; } #if CONFIG_INTEL_ADSP_MIC_PRIVACY if (cd->direction == SOF_IPC_STREAM_CAPTURE && node_id.f.dma_type == ipc4_hda_host_output_class) { - ret = mic_privacy_configure(dev, cd); + ret = mic_privacy_configure(mod, cd); if (ret < 0) { comp_err(dev, "unable to configure mic privacy"); - goto error; + return ret; } } #endif @@ -231,14 +223,14 @@ __cold static int copier_init(struct processing_module *mod) ret = copier_dai_create(dev, cd, copier, ipc_pipe->pipeline); if (ret < 0) { comp_err(dev, "unable to create dai"); - goto error; + return ret; } #if CONFIG_INTEL_ADSP_MIC_PRIVACY if (cd->direction == SOF_IPC_STREAM_CAPTURE) { - ret = mic_privacy_configure(dev, cd); + ret = mic_privacy_configure(mod, cd); if (ret < 0) { comp_err(dev, "unable to configure mic privacy"); - goto error; + return ret; } } #endif @@ -249,14 +241,13 @@ __cold static int copier_init(struct processing_module *mod) ret = copier_ipcgtw_create(dev, cd, copier, ipc_pipe->pipeline); if (ret < 0) { comp_err(dev, "unable to create IPC gateway"); - goto error; + return ret; } break; #endif default: comp_err(dev, "unsupported dma type %x", (uint32_t)node_id.f.dma_type); - ret = -EINVAL; - goto error; + return -EINVAL; }; dev->direction_set = true; @@ -270,11 +261,6 @@ __cold static int copier_init(struct processing_module *mod) dev->direction = cd->direction; dev->state = COMP_STATE_READY; return 0; -error: - rfree(gtw_cfg); -error_cd: - rfree(cd); - return ret; } __cold static int copier_free(struct processing_module *mod) @@ -303,10 +289,6 @@ __cold static int copier_free(struct processing_module *mod) break; } - if (cd) - rfree(cd->gtw_cfg); - rfree(cd); - return 0; } diff --git a/src/audio/copier/copier_generic.c b/src/audio/copier/copier_generic.c index dfed78e33738..ef21ad6fa4d7 100644 --- a/src/audio/copier/copier_generic.c +++ b/src/audio/copier/copier_generic.c @@ -428,7 +428,7 @@ __cold int create_multi_endpoint_buffer(struct comp_dev *dev, ipc_buf.comp.pipeline_id = config->pipeline_id; ipc_buf.comp.core = config->core; /* allocate not shared buffer */ - buffer = buffer_new(&ipc_buf, false); + buffer = buffer_new(NULL, &ipc_buf, false); if (!buffer) return -ENOMEM; diff --git a/src/audio/crossover/crossover.c b/src/audio/crossover/crossover.c index 7ae8dd47c6a2..cd820a2d1d50 100644 --- a/src/audio/crossover/crossover.c +++ b/src/audio/crossover/crossover.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -47,12 +46,13 @@ DECLARE_TR_CTX(crossover_tr, SOF_UUID(crossover_uuid), LOG_LEVEL_INFO); * \brief Reset the state (coefficients and delay) of the crossover filter * across all channels */ -static void crossover_reset_state(struct comp_data *cd) +static void crossover_reset_state(struct processing_module *mod, + struct comp_data *cd) { int i; for (i = 0; i < PLATFORM_MAX_CHANNELS; i++) - crossover_reset_state_ch(&cd->state[i]); + crossover_reset_state_ch(mod, &cd->state[i]); } /** @@ -156,7 +156,8 @@ static int crossover_assign_sinks(struct processing_module *mod, * high/low pass filter. * \param[out] lr4 initialized struct */ -static int crossover_init_coef_lr4(struct sof_eq_iir_biquad *coef, +static int crossover_init_coef_lr4(struct processing_module *mod, + struct sof_eq_iir_biquad *coef, struct iir_state_df1 *lr4) { int ret; @@ -169,8 +170,7 @@ static int crossover_init_coef_lr4(struct sof_eq_iir_biquad *coef, * in series due to identity. To maintain the structure of * iir_state_df1, it requires two copies of coefficients in a row. */ - lr4->coef = rzalloc(SOF_MEM_FLAG_USER, - sizeof(struct sof_eq_iir_biquad) * 2); + lr4->coef = mod_zalloc(mod, sizeof(struct sof_eq_iir_biquad) * 2); if (!lr4->coef) return -ENOMEM; @@ -189,8 +189,7 @@ static int crossover_init_coef_lr4(struct sof_eq_iir_biquad *coef, * delay[0..1] -> state for first biquad * delay[2..3] -> state for second biquad */ - lr4->delay = rzalloc(SOF_MEM_FLAG_USER, - sizeof(uint64_t) * CROSSOVER_NUM_DELAYS_LR4); + lr4->delay = mod_zalloc(mod, sizeof(uint64_t) * CROSSOVER_NUM_DELAYS_LR4); if (!lr4->delay) return -ENOMEM; @@ -203,7 +202,8 @@ static int crossover_init_coef_lr4(struct sof_eq_iir_biquad *coef, /** * \brief Initializes the crossover coefficients for one channel */ -int crossover_init_coef_ch(struct sof_eq_iir_biquad *coef, +int crossover_init_coef_ch(struct processing_module *mod, + struct sof_eq_iir_biquad *coef, struct crossover_state *ch_state, int32_t num_sinks) { @@ -214,12 +214,12 @@ int crossover_init_coef_ch(struct sof_eq_iir_biquad *coef, for (i = 0; i < num_lr4s; i++) { /* Get the low pass coefficients */ - err = crossover_init_coef_lr4(&coef[j], + err = crossover_init_coef_lr4(mod, &coef[j], &ch_state->lowpass[i]); if (err < 0) return -EINVAL; /* Get the high pass coefficients */ - err = crossover_init_coef_lr4(&coef[j + 1], + err = crossover_init_coef_lr4(mod, &coef[j + 1], &ch_state->highpass[i]); if (err < 0) return -EINVAL; @@ -259,13 +259,13 @@ static int crossover_init_coef(struct processing_module *mod, int nch) /* Collect the coef array and assign it to every channel */ crossover = config->coef; for (ch = 0; ch < nch; ch++) { - err = crossover_init_coef_ch(crossover, &cd->state[ch], + err = crossover_init_coef_ch(mod, crossover, &cd->state[ch], config->num_sinks); /* Free all previously allocated blocks in case of an error */ if (err < 0) { comp_err(mod->dev, "crossover_init_coef(), could not assign coefficients to ch %d", ch); - crossover_reset_state(cd); + crossover_reset_state(mod, cd); return err; } } @@ -282,7 +282,7 @@ static int crossover_setup(struct processing_module *mod, int nch) int ret = 0; /* Reset any previous state */ - crossover_reset_state(cd); + crossover_reset_state(mod, cd); /* Assign LR4 coefficients from config */ ret = crossover_init_coef(mod, nch); @@ -312,40 +312,34 @@ static int crossover_init(struct processing_module *mod) return -ENOMEM; } - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = mod_zalloc(mod, sizeof(*cd)); if (!cd) return -ENOMEM; md->private = cd; /* Handler for configuration data */ - cd->model_handler = comp_data_blob_handler_new(dev); + cd->model_handler = mod_data_blob_handler_new(mod); if (!cd->model_handler) { comp_err(dev, "comp_data_blob_handler_new() failed."); - ret = -ENOMEM; - goto cd_fail; + return -ENOMEM; } /* Get configuration data and reset Crossover state */ ret = comp_init_data_blob(cd->model_handler, bs, ipc_crossover->data); if (ret < 0) { comp_err(dev, "comp_init_data_blob() failed."); - goto cd_fail; + return ret; } ret = crossover_output_pin_init(mod); if (ret < 0) { comp_err(dev, "crossover_init_output_pins() failed."); - goto cd_fail; + return ret; } - crossover_reset_state(cd); + crossover_reset_state(mod, cd); return 0; - -cd_fail: - comp_data_blob_handler_free(cd->model_handler); - rfree(cd); - return ret; } /** @@ -357,11 +351,8 @@ static int crossover_free(struct processing_module *mod) comp_info(mod->dev, "crossover_free()"); - comp_data_blob_handler_free(cd->model_handler); - - crossover_reset_state(cd); + crossover_reset_state(mod, cd); - rfree(cd); return 0; } @@ -616,7 +607,7 @@ static int crossover_reset(struct processing_module *mod) comp_info(mod->dev, "crossover_reset()"); - crossover_reset_state(cd); + crossover_reset_state(mod, cd); cd->crossover_process = NULL; cd->crossover_split = NULL; @@ -642,10 +633,8 @@ static const struct module_interface crossover_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(crossover, &crossover_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("XOVER", crossover_llext_entry, 1, SOF_REG_UUID(crossover), 40); + SOF_LLEXT_MODULE_MANIFEST("XOVER", &crossover_interface, 1, SOF_REG_UUID(crossover), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/dai-legacy.c b/src/audio/dai-legacy.c index 7d98985d3a46..c91f5bf0a4b9 100644 --- a/src/audio/dai-legacy.c +++ b/src/audio/dai-legacy.c @@ -561,7 +561,8 @@ int dai_common_params(struct dai_data *dd, struct comp_dev *dev, return err; } } else { - dd->dma_buffer = buffer_alloc(buffer_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + dd->dma_buffer = buffer_alloc(NULL, buffer_size, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, false); if (!dd->dma_buffer) { comp_err(dev, "failed to alloc dma buffer"); diff --git a/src/audio/dai-zephyr.c b/src/audio/dai-zephyr.c index 0512b92abc87..443a0621b112 100644 --- a/src/audio/dai-zephyr.c +++ b/src/audio/dai-zephyr.c @@ -1019,7 +1019,7 @@ static int dai_set_dma_buffer(struct dai_data *dd, struct comp_dev *dev, return err; } } else { - dd->dma_buffer = buffer_alloc_range(buffer_size_preferred, buffer_size, + dd->dma_buffer = buffer_alloc_range(NULL, buffer_size_preferred, buffer_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, false); if (!dd->dma_buffer) { comp_err(dev, "failed to alloc dma buffer"); diff --git a/src/audio/dcblock/dcblock.c b/src/audio/dcblock/dcblock.c index 1576835d9ee1..d266f357554f 100644 --- a/src/audio/dcblock/dcblock.c +++ b/src/audio/dcblock/dcblock.c @@ -265,10 +265,8 @@ static const struct module_interface dcblock_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(dcblock, &dcblock_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("DCBLOCK", dcblock_llext_entry, 1, SOF_REG_UUID(dcblock), 40); + SOF_LLEXT_MODULE_MANIFEST("DCBLOCK", &dcblock_interface, 1, SOF_REG_UUID(dcblock), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/drc/drc.c b/src/audio/drc/drc.c index 5a6397291eda..734636f59337 100644 --- a/src/audio/drc/drc.c +++ b/src/audio/drc/drc.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -43,11 +42,11 @@ extern const struct sof_uuid drc_uuid; extern struct tr_ctx drc_tr; /* Called from drc_setup() from drc_process(), so cannot be __cold */ -void drc_reset_state(struct drc_state *state) +void drc_reset_state(struct processing_module *mod, struct drc_state *state) { int i; - rfree(state->pre_delay_buffers[0]); + mod_free(mod, state->pre_delay_buffers[0]); for (i = 0; i < PLATFORM_MAX_CHANNELS; ++i) { state->pre_delay_buffers[i] = NULL; } @@ -67,7 +66,8 @@ void drc_reset_state(struct drc_state *state) state->max_attack_compression_diff_db = INT32_MIN; } -int drc_init_pre_delay_buffers(struct drc_state *state, +int drc_init_pre_delay_buffers(struct processing_module *mod, + struct drc_state *state, size_t sample_bytes, int channels) { @@ -76,7 +76,7 @@ int drc_init_pre_delay_buffers(struct drc_state *state, int i; /* Allocate pre-delay (lookahead) buffers */ - state->pre_delay_buffers[0] = rballoc(SOF_MEM_FLAG_USER, bytes_total); + state->pre_delay_buffers[0] = mod_alloc(mod, bytes_total); if (!state->pre_delay_buffers[0]) return -ENOMEM; @@ -121,16 +121,17 @@ int drc_set_pre_delay_time(struct drc_state *state, } /* Called from drc_process(), so cannot be __cold */ -static int drc_setup(struct drc_comp_data *cd, uint16_t channels, uint32_t rate) +static int drc_setup(struct processing_module *mod, uint16_t channels, uint32_t rate) { + struct drc_comp_data *cd = module_get_private_data(mod); uint32_t sample_bytes = get_sample_bytes(cd->source_format); int ret; /* Reset any previous state */ - drc_reset_state(&cd->state); + drc_reset_state(mod, &cd->state); /* Allocate pre-delay buffers */ - ret = drc_init_pre_delay_buffers(&cd->state, (size_t)sample_bytes, (int)channels); + ret = drc_init_pre_delay_buffers(mod, &cd->state, (size_t)sample_bytes, (int)channels); if (ret < 0) return ret; @@ -164,28 +165,27 @@ __cold static int drc_init(struct processing_module *mod) return -EINVAL; } - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = mod_zalloc(mod, sizeof(*cd)); if (!cd) return -ENOMEM; md->private = cd; /* Handler for configuration data */ - cd->model_handler = comp_data_blob_handler_new(dev); + cd->model_handler = mod_data_blob_handler_new(mod); if (!cd->model_handler) { comp_err(dev, "comp_data_blob_handler_new() failed."); - ret = -ENOMEM; - goto cd_fail; + return -ENOMEM; } /* Get configuration data and reset DRC state */ ret = comp_init_data_blob(cd->model_handler, bs, cfg->data); if (ret < 0) { comp_err(dev, "comp_init_data_blob() failed."); - goto cd_fail; + return ret; } - drc_reset_state(&cd->state); + drc_reset_state(mod, &cd->state); /* Initialize DRC to enabled. If defined by topology, a control may set * enabled to false before prepare() or during streaming with the switch @@ -194,21 +194,12 @@ __cold static int drc_init(struct processing_module *mod) cd->enabled = true; cd->enable_switch = true; return 0; - -cd_fail: - comp_data_blob_handler_free(cd->model_handler); - rfree(cd); - return ret; } __cold static int drc_free(struct processing_module *mod) { - struct drc_comp_data *cd = module_get_private_data(mod); - assert_can_be_cold(); - comp_data_blob_handler_free(cd->model_handler); - rfree(cd); return 0; } @@ -284,7 +275,7 @@ static int drc_process(struct processing_module *mod, /* Check for changed configuration */ if (comp_is_new_data_blob_available(cd->model_handler)) { cd->config = comp_get_data_blob(cd->model_handler, NULL, NULL); - ret = drc_setup(cd, audio_stream_get_channels(source), + ret = drc_setup(mod, audio_stream_get_channels(source), audio_stream_get_rate(source)); if (ret < 0) { comp_err(dev, "drc_copy(), failed DRC setup"); @@ -370,7 +361,7 @@ static int drc_prepare(struct processing_module *mod, comp_info(dev, "drc_prepare(), source_format=%d", cd->source_format); cd->config = comp_get_data_blob(cd->model_handler, NULL, NULL); if (cd->config) { - ret = drc_setup(cd, channels, rate); + ret = drc_setup(mod, channels, rate); if (ret < 0) { comp_err(dev, "drc_prepare() error: drc_setup failed."); return ret; @@ -403,7 +394,7 @@ static int drc_reset(struct processing_module *mod) { struct drc_comp_data *cd = module_get_private_data(mod); - drc_reset_state(&cd->state); + drc_reset_state(mod, &cd->state); return 0; } @@ -424,10 +415,8 @@ static const struct module_interface drc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(drc, &drc_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("DRC", drc_llext_entry, 1, SOF_REG_UUID(drc), 40); + SOF_LLEXT_MODULE_MANIFEST("DRC", &drc_interface, 1, SOF_REG_UUID(drc), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/drc/drc_algorithm.h b/src/audio/drc/drc_algorithm.h index c0a942b09622..8d9d759eb3a8 100644 --- a/src/audio/drc/drc_algorithm.h +++ b/src/audio/drc/drc_algorithm.h @@ -14,10 +14,11 @@ #include "drc.h" /* drc reset function */ -void drc_reset_state(struct drc_state *state); +void drc_reset_state(struct processing_module *mod, struct drc_state *state); /* drc init functions */ -int drc_init_pre_delay_buffers(struct drc_state *state, +int drc_init_pre_delay_buffers(struct processing_module *mod, + struct drc_state *state, size_t sample_bytes, int channels); int drc_set_pre_delay_time(struct drc_state *state, diff --git a/src/audio/eq_fir/eq_fir.c b/src/audio/eq_fir/eq_fir.c index 0269c783a331..ed946157ed4c 100644 --- a/src/audio/eq_fir/eq_fir.c +++ b/src/audio/eq_fir/eq_fir.c @@ -490,10 +490,8 @@ static const struct module_interface eq_fir_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(eq_fir, &eq_fir_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("EQFIR", eq_fir_llext_entry, 1, SOF_REG_UUID(eq_fir), 40); + SOF_LLEXT_MODULE_MANIFEST("EQFIR", &eq_fir_interface, 1, SOF_REG_UUID(eq_fir), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/eq_iir/eq_iir.c b/src/audio/eq_iir/eq_iir.c index 0e7d3a9b541d..4b70fe355c77 100644 --- a/src/audio/eq_iir/eq_iir.c +++ b/src/audio/eq_iir/eq_iir.c @@ -260,10 +260,8 @@ static const struct module_interface eq_iir_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(eq_iir, &eq_iir_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("EQIIR", eq_iir_llext_entry, 1, SOF_REG_UUID(eq_iir), 40); + SOF_LLEXT_MODULE_MANIFEST("EQIIR", &eq_iir_interface, 1, SOF_REG_UUID(eq_iir), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/google/google_ctc_audio_processing.c b/src/audio/google/google_ctc_audio_processing.c index 3e3e12224457..4b01feea1efa 100644 --- a/src/audio/google/google_ctc_audio_processing.c +++ b/src/audio/google/google_ctc_audio_processing.c @@ -466,10 +466,8 @@ static const struct module_interface google_ctc_audio_processing_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(google_ctc_audio_processing, &google_ctc_audio_processing_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("CTC", google_ctc_audio_processing_llext_entry, + SOF_LLEXT_MODULE_MANIFEST("CTC", &google_ctc_audio_processing_interface, 1, SOF_REG_UUID(google_ctc_audio_processing), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/google/google_rtc_audio_processing.c b/src/audio/google/google_rtc_audio_processing.c index f98b2de7322c..4e9a55f39516 100644 --- a/src/audio/google/google_rtc_audio_processing.c +++ b/src/audio/google/google_rtc_audio_processing.c @@ -856,10 +856,8 @@ static const struct module_interface google_rtc_audio_processing_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(google_rtc_audio_processing, &google_rtc_audio_processing_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("RTC_AEC", google_rtc_audio_processing_llext_entry, + SOF_LLEXT_MODULE_MANIFEST("RTC_AEC", &google_rtc_audio_processing_interface, 7, SOF_REG_UUID(google_rtc_audio_processing), 1); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/host-legacy.c b/src/audio/host-legacy.c index 192686da3fdd..ac21aa38d9df 100644 --- a/src/audio/host-legacy.c +++ b/src/audio/host-legacy.c @@ -759,7 +759,8 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, goto out; } } else { - hd->dma_buffer = buffer_alloc(buffer_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + hd->dma_buffer = buffer_alloc(NULL, buffer_size, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, false); if (!hd->dma_buffer) { comp_err(dev, "failed to alloc dma buffer"); diff --git a/src/audio/host-zephyr.c b/src/audio/host-zephyr.c index b7f203eea3a0..4c99a04e8170 100644 --- a/src/audio/host-zephyr.c +++ b/src/audio/host-zephyr.c @@ -920,8 +920,9 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, } } else { /* allocate not shared buffer */ - hd->dma_buffer = buffer_alloc_range(buffer_size_preferred, buffer_size, - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, false); + hd->dma_buffer = buffer_alloc_range(NULL, buffer_size_preferred, buffer_size, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + addr_align, false); if (!hd->dma_buffer) { comp_err(dev, "failed to alloc dma buffer"); return -ENOMEM; diff --git a/src/audio/igo_nr/igo_nr.c b/src/audio/igo_nr/igo_nr.c index 5ab5424cdff3..fd6d900eaf82 100644 --- a/src/audio/igo_nr/igo_nr.c +++ b/src/audio/igo_nr/igo_nr.c @@ -895,10 +895,8 @@ static const struct module_interface igo_nr_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(igo_nr, &igo_nr_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("IGO_NR", igo_nr_llext_entry, 1, SOF_REG_UUID(igo_nr), 40); + SOF_LLEXT_MODULE_MANIFEST("IGO_NR", &igo_nr_interface, 1, SOF_REG_UUID(igo_nr), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/mfcc/mfcc.c b/src/audio/mfcc/mfcc.c index 6437c187e29e..da1fa9f3414e 100644 --- a/src/audio/mfcc/mfcc.c +++ b/src/audio/mfcc/mfcc.c @@ -267,10 +267,8 @@ static const struct module_interface mfcc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(mfcc, &mfcc_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("MFCC", mfcc_llext_entry, 1, SOF_REG_UUID(mfcc), 40); + SOF_LLEXT_MODULE_MANIFEST("MFCC", &mfcc_interface, 1, SOF_REG_UUID(mfcc), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/mixin_mixout/mixin_mixout.c b/src/audio/mixin_mixout/mixin_mixout.c index 650d790dff7b..5043d02920f0 100644 --- a/src/audio/mixin_mixout/mixin_mixout.c +++ b/src/audio/mixin_mixout/mixin_mixout.c @@ -1020,13 +1020,10 @@ static const struct module_interface mixout_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(mixin, &mixin_interface); -SOF_LLEXT_MOD_ENTRY(mixout, &mixout_interface); - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("MIXIN", mixin_llext_entry, 1, SOF_REG_UUID(mixin), 30), - SOF_LLEXT_MODULE_MANIFEST("MIXOUT", mixout_llext_entry, 1, SOF_REG_UUID(mixout), 30), + SOF_LLEXT_MODULE_MANIFEST("MIXIN", &mixin_interface, 1, SOF_REG_UUID(mixin), 30), + SOF_LLEXT_MODULE_MANIFEST("MIXOUT", &mixout_interface, 1, SOF_REG_UUID(mixout), 30), }; SOF_LLEXT_BUILDINFO; diff --git a/src/audio/module_adapter/Kconfig b/src/audio/module_adapter/Kconfig index bb9081812271..3c762c73f7f8 100644 --- a/src/audio/module_adapter/Kconfig +++ b/src/audio/module_adapter/Kconfig @@ -3,6 +3,27 @@ menu "Processing modules" visible if COMP_MODULE_ADAPTER + config MODULE_MEMORY_API_CONTAINER_CHUNK_SIZE + int "Number of memory containers to allocate at once" + default 16 + help + The per module resource containers are allocated in + chunks. The unused containers are kept in free + list. When the free list is empty the amount of + containers to allocate at once is selected by this + config option. + + config MODULE_MEMORY_API_DEBUG + bool "Turn on memory API thread safety checks" + default y if DEBUG + help + The Module Memory API structures are not protected + by locks. This is because the initialization, + allocation, and freeing of resources should always + be done in the same thread. This option adds an + assert to make sure no other thread makes such + operations. + config CADENCE_CODEC bool "Cadence codec" default n diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 03a345552f71..1a0ea3277530 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -12,8 +12,24 @@ */ #include - #include +#include +#include +#include +#if CONFIG_IPC_MAJOR_4 +#include +#include +#include +#endif + +/* The __ZEPHYR__ condition is to keep cmocka tests working */ +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) +#define MEM_API_CHECK_THREAD(res) if ((res)->rsrc_mngr != k_current_get()) \ + LOG_ERR("mngr %p != cur %p", (res)->rsrc_mngr, k_current_get()); else \ + LOG_INF("mngr %p == cur %p", (res)->rsrc_mngr, k_current_get()) +#else +#define MEM_API_CHECK_THREAD(res) +#endif LOG_MODULE_DECLARE(module_adapter, CONFIG_SOF_LOG_LEVEL); @@ -61,10 +77,24 @@ int module_load_config(struct comp_dev *dev, const void *cfg, size_t size) return ret; } +void mod_resource_init(struct processing_module *mod) +{ + struct module_resources *rsrc = &mod->priv.resources; + + /* Init memory list */ + list_init(&rsrc->mem_list); + list_init(&rsrc->free_cont_list); + list_init(&rsrc->cont_chunk_list); + rsrc->heap_usage = 0; + rsrc->heap_high_water_mark = 0; +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) + rsrc->rsrc_mngr = k_current_get(); +#endif +} + int module_init(struct processing_module *mod) { int ret; - struct module_data *md = &mod->priv; struct comp_dev *dev = mod->dev; const struct module_interface *const interface = dev->drv->adapter_ops; @@ -91,11 +121,14 @@ int module_init(struct processing_module *mod) return -EIO; } - /* Init memory list */ - list_init(&md->memory.mem_list); - /* Now we can proceed with module specific initialization */ - ret = interface->init(mod); +#if CONFIG_IPC_MAJOR_4 + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + ret = scheduler_dp_thread_ipc(mod, SOF_IPC4_MOD_INIT_INSTANCE, NULL); + else +#endif + ret = interface->init(mod); + if (ret) { comp_err(dev, "module_init() error %d: module specific init failed, comp id %d", ret, dev_comp_id(dev)); @@ -104,116 +137,217 @@ int module_init(struct processing_module *mod) comp_dbg(dev, "module_init() done"); #if CONFIG_IPC_MAJOR_3 - md->state = MODULE_INITIALIZED; + mod->priv.state = MODULE_INITIALIZED; #endif return 0; } +struct container_chunk { + struct list_item chunk_list; + struct module_memory containers[CONFIG_MODULE_MEMORY_API_CONTAINER_CHUNK_SIZE]; +}; + +static struct module_memory *container_get(struct processing_module *mod) +{ + struct module_resources *res = &mod->priv.resources; + struct k_heap *mod_heap = res->heap; + struct module_memory *container; + + if (list_is_empty(&res->free_cont_list)) { + struct container_chunk *chunk = sof_heap_alloc(mod_heap, 0, sizeof(*chunk), 0); + int i; + + if (!chunk) { + comp_err(mod->dev, "allocating more containers failed"); + return NULL; + } + + memset(chunk, 0, sizeof(*chunk)); + + list_item_append(&chunk->chunk_list, &res->cont_chunk_list); + for (i = 0; i < ARRAY_SIZE(chunk->containers); i++) + list_item_append(&chunk->containers[i].mem_list, &res->free_cont_list); + } + + container = list_first_item(&res->free_cont_list, struct module_memory, mem_list); + list_item_del(&container->mem_list); + return container; +} + +static void container_put(struct processing_module *mod, struct module_memory *container) +{ + struct module_resources *res = &mod->priv.resources; + + list_item_append(&container->mem_list, &res->free_cont_list); +} + +void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start) +{ + struct module_resources *res = &mod->priv.resources; + + if (size) + *size = res->heap_size; + + if (start) + *start = (uintptr_t)res->heap_mem; +} + /** - * Allocates aligned memory block for module. + * Allocates aligned memory block with flags for module. * @param mod Pointer to the module this memory block is allocatd for. + * @param flags Allocator flags. * @param bytes Size in bytes. * @param alignment Alignment in bytes. * @return Pointer to the allocated memory or NULL if failed. * * The allocated memory is automatically freed when the module is unloaded. */ -void *mod_alloc_align(struct processing_module *mod, uint32_t size, uint32_t alignment) +void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, + size_t alignment) { - struct comp_dev *dev = mod->dev; - struct module_memory *container; + struct module_memory *container = container_get(mod); + struct module_resources *res = &mod->priv.resources; + struct k_heap *mod_heap = res->heap; void *ptr; - if (!size) { - comp_err(dev, "mod_alloc: requested allocation of 0 bytes."); + MEM_API_CHECK_THREAD(res); + if (!container) return NULL; - } - /* Allocate memory container */ - container = rzalloc(SOF_MEM_FLAG_USER, - sizeof(struct module_memory)); - if (!container) { - comp_err(dev, "mod_alloc: failed to allocate memory container."); + if (!size) { + comp_err(mod->dev, "mod_alloc: requested allocation of 0 bytes."); + container_put(mod, container); return NULL; } /* Allocate memory for module */ - if (alignment) - ptr = rballoc_align(SOF_MEM_FLAG_USER, size, alignment); - else - ptr = rballoc(SOF_MEM_FLAG_USER, size); - + ptr = sof_heap_alloc(mod_heap, flags, size, alignment); if (!ptr) { - comp_err(dev, "mod_alloc: failed to allocate memory for comp %x.", - dev_comp_id(dev)); - rfree(container); + comp_err(mod->dev, "mod_alloc: failed to allocate memory for comp %x.", + dev_comp_id(mod->dev)); + container_put(mod, container); return NULL; } /* Store reference to allocated memory */ container->ptr = ptr; container->size = size; - list_item_prepend(&container->mem_list, &mod->priv.memory.mem_list); + container->free = NULL; + list_item_prepend(&container->mem_list, &res->mem_list); + + res->heap_usage += size; + if (res->heap_usage > res->heap_high_water_mark) + res->heap_high_water_mark = res->heap_usage; return ptr; } -EXPORT_SYMBOL(mod_alloc_align); +EXPORT_SYMBOL(z_impl_mod_alloc_ext); /** - * Allocates memory block for module. + * Creates a blob handler and releases it when the module is unloaded * @param mod Pointer to module this memory block is allocated for. - * @param bytes Size in bytes. - * @return Pointer to the allocated memory or NULL if failed. + * @return Pointer to the created data blob handler * - * Like mod_alloc_align() but the alignment can not be specified. However, - * rballoc() will always aligns the memory to PLATFORM_DCACHE_ALIGN. + * Like comp_data_blob_handler_new() but the handler is automatically freed. */ -void *mod_alloc(struct processing_module *mod, uint32_t size) +#if CONFIG_COMP_BLOB +static void blob_free_wrapper(struct k_heap *heap, struct comp_data_blob_handler *blob_handler) +{ + ARG_UNUSED(heap); + comp_data_blob_handler_free(blob_handler); +} + +struct comp_data_blob_handler * +mod_data_blob_handler_new(struct processing_module *mod) { - return mod_alloc_align(mod, size, 0); + struct module_resources *res = &mod->priv.resources; + struct module_memory *container = container_get(mod); + struct comp_data_blob_handler *dbh; + + MEM_API_CHECK_THREAD(res); + if (!container) + return NULL; + + dbh = comp_data_blob_handler_new_ext(mod->dev, false, NULL, NULL); + if (!dbh) { + container_put(mod, container); + return NULL; + } + + container->ptr = dbh; + container->size = 0; + container->free = (void (*)(struct k_heap *, void *))blob_free_wrapper; + list_item_prepend(&container->mem_list, &res->mem_list); + + return dbh; } -EXPORT_SYMBOL(mod_alloc); +EXPORT_SYMBOL(mod_data_blob_handler_new); +#endif /** - * Allocates memory block for module and initializes it to zero. - * @param mod Pointer to module this memory block is allocated for. - * @param bytes Size in bytes. - * @return Pointer to the allocated memory or NULL if failed. + * Make a module associated shared SRAM copy of DRAM read-only data. + * @param mod Pointer to module this copy is allocated for. + * @return Pointer to the SRAM copy. * - * Like mod_alloc() but the allocated memory is initialized to zero. + * Like fast_get() but the handler is automatically freed. */ -void *mod_zalloc(struct processing_module *mod, uint32_t size) +#if CONFIG_FAST_GET +const void *z_impl_mod_fast_get(struct processing_module *mod, const void * const dram_ptr, + size_t size) { - void *ret = mod_alloc(mod, size); + struct module_resources *res = &mod->priv.resources; + struct module_memory *container = container_get(mod); + const void *ptr; - if (ret) - memset(ret, 0, size); + MEM_API_CHECK_THREAD(res); + if (!container) + return NULL; - return ret; + ptr = fast_get(res->heap, dram_ptr, size); + if (!ptr) { + container_put(mod, container); + return NULL; + } + + container->ptr = (void *)ptr; + container->size = 0; + container->free = (void (*)(struct k_heap *, void *))fast_put; + list_item_prepend(&container->mem_list, &res->mem_list); + + return ptr; } -EXPORT_SYMBOL(mod_zalloc); +EXPORT_SYMBOL(z_impl_mod_fast_get); +#endif /** * Frees the memory block removes it from module's book keeping. * @param mod Pointer to module this memory block was allocated for. * @param ptr Pointer to the memory block. */ -int mod_free(struct processing_module *mod, void *ptr) +int z_impl_mod_free(struct processing_module *mod, const void *ptr) { + struct module_resources *res = &mod->priv.resources; + struct k_heap *mod_heap = res->heap; struct module_memory *mem; struct list_item *mem_list; struct list_item *_mem_list; + MEM_API_CHECK_THREAD(res); if (!ptr) return 0; /* Find which container keeps this memory */ - list_for_item_safe(mem_list, _mem_list, &mod->priv.memory.mem_list) { + list_for_item_safe(mem_list, _mem_list, &res->mem_list) { mem = container_of(mem_list, struct module_memory, mem_list); if (mem->ptr == ptr) { - rfree(mem->ptr); + if (mem->free) { + mem->free(mod_heap, mem->ptr); + } else { + sof_heap_free(mod_heap, mem->ptr); + res->heap_usage -= mem->size; + } list_item_del(&mem->mem_list); - rfree(mem); + container_put(mod, mem); return 0; } } @@ -223,7 +357,29 @@ int mod_free(struct processing_module *mod, void *ptr) return -EINVAL; } -EXPORT_SYMBOL(mod_free); +EXPORT_SYMBOL(z_impl_mod_free); + +#ifdef CONFIG_USERSPACE +const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * const dram_ptr, + size_t size) +{ + return z_impl_mod_fast_get(mod, dram_ptr, size); +} +#include + +void *z_vrfy_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, + size_t alignment) +{ + return z_impl_mod_alloc_ext(mod, flags, size, alignment); +} +#include + +int z_vrfy_mod_free(struct processing_module *mod, const void *ptr) +{ + return z_impl_mod_free(mod, ptr); +} +#include +#endif int module_prepare(struct processing_module *mod, struct sof_source **sources, int num_of_sources, @@ -242,7 +398,27 @@ int module_prepare(struct processing_module *mod, return -EPERM; #endif if (ops->prepare) { - int ret = ops->prepare(mod, sources, num_of_sources, sinks, num_of_sinks); + int ret; + + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { +#if CONFIG_IPC_MAJOR_4 + union scheduler_dp_thread_ipc_param param = { + .pipeline_state = { + .trigger_cmd = COMP_TRIGGER_PREPARE, + .state = SOF_IPC4_PIPELINE_STATE_RUNNING, + .n_sources = num_of_sources, + .sources = sources, + .n_sinks = num_of_sinks, + .sinks = sinks, + }, + }; + ret = scheduler_dp_thread_ipc(mod, SOF_IPC4_GLB_SET_PIPELINE_STATE, ¶m); +#else + ret = 0; +#endif + } else { + ret = ops->prepare(mod, sources, num_of_sources, sinks, num_of_sinks); + } if (ret) { comp_err(dev, "module_prepare() error %d: module specific prepare failed, comp_id %d", @@ -366,11 +542,21 @@ int module_reset(struct processing_module *mod) if (md->state < MODULE_IDLE) return 0; #endif - /* cancel task if DP task*/ - if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP && mod->dev->task) - schedule_task_cancel(mod->dev->task); + if (ops->reset) { - ret = ops->reset(mod); + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { +#if CONFIG_IPC_MAJOR_4 + union scheduler_dp_thread_ipc_param param = { + .pipeline_state.trigger_cmd = COMP_TRIGGER_STOP, + }; + ret = scheduler_dp_thread_ipc(mod, SOF_IPC4_GLB_SET_PIPELINE_STATE, ¶m); +#else + ret = 0; +#endif + } else { + ret = ops->reset(mod); + } + if (ret) { if (ret != PPL_STATUS_PATH_STOP) comp_err(mod->dev, @@ -403,16 +589,29 @@ int module_reset(struct processing_module *mod) */ void mod_free_all(struct processing_module *mod) { - struct module_memory *mem; - struct list_item *mem_list; - struct list_item *_mem_list; + struct module_resources *res = &mod->priv.resources; + struct k_heap *mod_heap = res->heap; + struct list_item *list; + struct list_item *_list; + MEM_API_CHECK_THREAD(res); /* Find which container keeps this memory */ - list_for_item_safe(mem_list, _mem_list, &mod->priv.memory.mem_list) { - mem = container_of(mem_list, struct module_memory, mem_list); - rfree(mem->ptr); + list_for_item_safe(list, _list, &res->mem_list) { + struct module_memory *mem = container_of(list, struct module_memory, mem_list); + + if (mem->free) + mem->free(res->heap, mem->ptr); + else + sof_heap_free(mod_heap, mem->ptr); list_item_del(&mem->mem_list); - rfree(mem); + } + + list_for_item_safe(list, _list, &res->cont_chunk_list) { + struct container_chunk *chunk = + container_of(list, struct container_chunk, chunk_list); + + list_item_del(&chunk->chunk_list); + sof_heap_free(mod_heap, chunk); } } EXPORT_SYMBOL(mod_free_all); @@ -423,7 +622,7 @@ int module_free(struct processing_module *mod) struct module_data *md = &mod->priv; int ret = 0; - if (ops->free) { + if (ops->free && mod->dev->ipc_config.proc_domain != COMP_PROCESSING_DOMAIN_DP) { ret = ops->free(mod); if (ret) comp_warn(mod->dev, "error: %d for %d", @@ -570,8 +769,18 @@ int module_bind(struct processing_module *mod, struct bind_info *bind_data) if (ret) return ret; - if (ops->bind) - ret = ops->bind(mod, bind_data); + if (ops->bind) { + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { +#if CONFIG_IPC_MAJOR_4 + union scheduler_dp_thread_ipc_param param = { + .bind_data = bind_data, + }; + ret = scheduler_dp_thread_ipc(mod, SOF_IPC4_MOD_BIND, ¶m); +#endif + } else { + ret = ops->bind(mod, bind_data); + } + } return ret; } diff --git a/src/audio/module_adapter/module/waves/waves.c b/src/audio/module_adapter/module/waves/waves.c index 5bb6dcd1a99a..624b53f5f75c 100644 --- a/src/audio/module_adapter/module/waves/waves.c +++ b/src/audio/module_adapter/module/waves/waves.c @@ -913,10 +913,8 @@ static const struct module_interface waves_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(waves, &waves_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("WAVES", waves_llext_entry, 7, SOF_REG_UUID(waves), 8); + SOF_LLEXT_MODULE_MANIFEST("WAVES", &waves_interface, 7, SOF_REG_UUID(waves), 8); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 8b649cfe18c3..694f6db95e2f 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -18,10 +18,16 @@ #include #include #include +#include #include #include #include #include +#if CONFIG_IPC_MAJOR_4 +#include +#include +#include +#endif #include #include #include @@ -74,12 +80,34 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, return NULL; } - dev = comp_alloc(drv, sizeof(*dev)); - if (!dev) { - comp_cl_err(drv, "module_adapter_new(), failed to allocate memory for comp_dev"); - return NULL; + struct dp_heap_user *mod_heap_user; + const size_t heap_size = 8 * 1024; + uint8_t *mod_heap_mem; + struct k_heap *mod_heap; + int flags; + + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { + /* Keep uncached to match the default SOF heap! */ + mod_heap_mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + heap_size, 4096); + if (!mod_heap_mem) + return NULL; + + const size_t heap_pref_size = ALIGN_UP(sizeof(*mod_heap_user), 4); + void *mod_heap_buf = mod_heap_mem + heap_pref_size; + + mod_heap_user = (struct dp_heap_user *)mod_heap_mem; + mod_heap = &mod_heap_user->heap; + k_heap_init(mod_heap, mod_heap_buf, heap_size - heap_pref_size); + + flags = SOF_MEM_FLAG_COHERENT; + } else { + mod_heap_mem = NULL; + mod_heap = NULL; + mod_heap_user = NULL; + + flags = 0; } - dev->ipc_config = *config; /* allocate module information. * for DP shared modules this struct must be accessible from all cores @@ -87,21 +115,48 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, * will be bound to. So we need to allocate shared memory for each DP module * To be removed when pipeline 2.0 is ready */ - int flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - mod = rzalloc(flags, sizeof(*mod)); + mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0); if (!mod) { - comp_err(dev, "module_adapter_new(), failed to allocate memory for module"); + comp_cl_err(drv, "module_adapter_new(), failed to allocate memory for module"); + goto emod; + } + + memset(mod, 0, sizeof(*mod)); + mod->priv.resources.heap = mod_heap; + mod->priv.resources.heap_mem = mod_heap_mem; + mod->priv.resources.heap_size = heap_size; + mod_resource_init(mod); + + /* + * comp_alloc() always allocated dev uncached. Would be difficult to optimize. Only + * if the whole currently active topology is running on the primary core, then it + * can be cached. Effectively it can be only cached in single-core configurations. + */ + dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0); + if (!dev) { + comp_cl_err(drv, "module_adapter_new(), failed to allocate memory for comp_dev"); goto err; } + memset(dev, 0, sizeof(*dev)); + comp_init(drv, dev, sizeof(*dev)); + dev->ipc_config = *config; + dst = &mod->priv.cfg; module_set_private_data(mod, mod_priv); mod->dev = dev; dev->mod = mod; +#if CONFIG_ZEPHYR_DP_SCHEDULER + /* create a task for DP processing */ + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) { + /* All data allocated, create a thread */ + pipeline_comp_dp_task_init(dev); + } +#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ + list_init(&mod->raw_data_buffers_list); ret = module_adapter_init_data(dev, dst, config, spec); @@ -133,12 +188,6 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, goto err; } -#if CONFIG_ZEPHYR_DP_SCHEDULER - /* create a task for DP processing */ - if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP) - pipeline_comp_dp_task_init(dev); -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ - module_adapter_reset_data(dst); dev->state = COMP_STATE_READY; @@ -155,17 +204,28 @@ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, comp_err(dev, "module_adapter_new() %d: module params failed", ret); goto err; } + + if (mod_heap_user) + mod_heap_user->client_count++; #endif comp_dbg(dev, "module_adapter_new() done"); return dev; + err: +#if CONFIG_ZEPHYR_DP_SCHEDULER + if (dev->task) + schedule_task_free(dev->task); +#endif #if CONFIG_IPC_MAJOR_4 if (mod) rfree(mod->priv.cfg.input_pins); #endif - rfree(mod); - rfree(dev); + sof_heap_free(mod_heap, dev); + sof_heap_free(mod_heap, mod); +emod: + rfree(mod_heap_mem); + return NULL; } EXPORT_SYMBOL(module_adapter_new); @@ -450,7 +510,8 @@ int module_adapter_prepare(struct comp_dev *dev) if (list_is_empty(&mod->raw_data_buffers_list)) { for (i = 0; i < mod->num_of_sinks; i++) { /* allocate not shared buffer */ - struct comp_buffer *buffer = buffer_alloc(buff_size, SOF_MEM_FLAG_USER, + struct comp_buffer *buffer = buffer_alloc(md->resources.heap, + buff_size, SOF_MEM_FLAG_USER, PLATFORM_DCACHE_ALIGN, false); uint32_t flags; @@ -531,11 +592,9 @@ int module_adapter_params(struct comp_dev *dev, struct sof_ipc_stream_params *pa } /* allocate stream_params each time */ - if (mod->stream_params) - rfree(mod->stream_params); + mod_free(mod, mod->stream_params); - mod->stream_params = rzalloc(SOF_MEM_FLAG_USER, - sizeof(*mod->stream_params) + params->ext_data_length); + mod->stream_params = mod_alloc(mod, sizeof(*mod->stream_params) + params->ext_data_length); if (!mod->stream_params) return -ENOMEM; @@ -1200,8 +1259,20 @@ int module_adapter_trigger(struct comp_dev *dev, int cmd) dev->state = COMP_STATE_ACTIVE; return PPL_STATUS_PATH_STOP; } - if (interface->trigger) + + if (interface->trigger) { +#if CONFIG_IPC_MAJOR_4 + if (dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { + /* Process DP module's trigger */ + union scheduler_dp_thread_ipc_param param = { + .pipeline_state.trigger_cmd = cmd, + }; + return scheduler_dp_thread_ipc(mod, SOF_IPC4_GLB_SET_PIPELINE_STATE, + ¶m); + } +#endif return interface->trigger(mod, cmd); + } return module_adapter_set_state(mod, dev, cmd); } @@ -1246,7 +1317,7 @@ int module_adapter_reset(struct comp_dev *dev) buffer_zero(buffer); } - rfree(mod->stream_params); + mod_free(mod, mod->stream_params); mod->stream_params = NULL; comp_dbg(dev, "done"); @@ -1263,6 +1334,19 @@ void module_adapter_free(struct comp_dev *dev) comp_dbg(dev, "start"); + if (dev->task) { + /* + * Run DP module's .free() method in its thread context. + * Unlike with other IPCs we first run module's .free() in + * thread context, then cancel the thread, and then execute + * final clean up + */ +#if CONFIG_IPC_MAJOR_4 + scheduler_dp_thread_ipc(mod, SOF_IPC4_MOD_DELETE_INSTANCE, NULL); +#endif + schedule_task_free(dev->task); + } + ret = module_free(mod); if (ret) comp_err(dev, "failed with error: %d", ret); @@ -1284,24 +1368,31 @@ void module_adapter_free(struct comp_dev *dev) rfree(mod->priv.cfg.input_pins); #endif - rfree(mod->stream_params); - rfree(mod); - rfree(dev); + struct k_heap *mod_heap = mod->priv.resources.heap; + struct dp_heap_user *mod_heap_user = CONTAINER_OF(mod_heap, struct dp_heap_user, heap); + void *mem = mod->priv.resources.heap_mem; + + /* + * In principle it shouldn't even be needed to free individual objects + * on the module heap since we're freeing the heap itself too + */ + sof_heap_free(mod_heap, mod); + list_item_del(&dev->bsource_list); + list_item_del(&dev->bsink_list); + sof_heap_free(mod_heap, dev); + if (!mod_heap || !--mod_heap_user->client_count) + rfree(mem); } EXPORT_SYMBOL(module_adapter_free); -size_t module_adapter_heap_usage(struct processing_module *mod) +size_t module_adapter_heap_usage(struct processing_module *mod, size_t *hwm) { - struct list_item *mem_list, *_mem_list; - size_t size = 0; - - list_for_item_safe(mem_list, _mem_list, &mod->priv.memory.mem_list) { - struct module_memory *mem = container_of(mem_list, struct module_memory, mem_list); + struct module_resources *res = &mod->priv.resources; - size += mem->size; - } + if (hwm) + *hwm = res->heap_high_water_mark; - return size; + return res->heap_usage; } EXPORT_SYMBOL(module_adapter_heap_usage); diff --git a/src/audio/multiband_drc/multiband_drc.c b/src/audio/multiband_drc/multiband_drc.c index 74cdb0ebcdd4..e1c796b5440b 100644 --- a/src/audio/multiband_drc/multiband_drc.c +++ b/src/audio/multiband_drc/multiband_drc.c @@ -42,28 +42,30 @@ SOF_DEFINE_REG_UUID(multiband_drc); DECLARE_TR_CTX(multiband_drc_tr, SOF_UUID(multiband_drc_uuid), LOG_LEVEL_INFO); /* Called from multiband_drc_setup() from multiband_drc_process(), so cannot be __cold */ -static void multiband_drc_reset_state(struct multiband_drc_state *state) +static void multiband_drc_reset_state(struct processing_module *mod, + struct multiband_drc_state *state) { int i; /* Reset emphasis eq-iir state */ for (i = 0; i < PLATFORM_MAX_CHANNELS; i++) - multiband_drc_iir_reset_state_ch(&state->emphasis[i]); + multiband_drc_iir_reset_state_ch(mod, &state->emphasis[i]); /* Reset crossover state */ for (i = 0; i < PLATFORM_MAX_CHANNELS; i++) - crossover_reset_state_ch(&state->crossover[i]); + crossover_reset_state_ch(mod, &state->crossover[i]); /* Reset drc kernel state */ for (i = 0; i < SOF_MULTIBAND_DRC_MAX_BANDS; i++) - drc_reset_state(&state->drc[i]); + drc_reset_state(mod, &state->drc[i]); /* Reset deemphasis eq-iir state */ for (i = 0; i < PLATFORM_MAX_CHANNELS; i++) - multiband_drc_iir_reset_state_ch(&state->deemphasis[i]); + multiband_drc_iir_reset_state_ch(mod, &state->deemphasis[i]); } -static int multiband_drc_eq_init_coef_ch(struct sof_eq_iir_biquad *coef, +static int multiband_drc_eq_init_coef_ch(struct processing_module *mod, + struct sof_eq_iir_biquad *coef, struct iir_state_df1 *eq) { int ret; @@ -72,8 +74,7 @@ static int multiband_drc_eq_init_coef_ch(struct sof_eq_iir_biquad *coef, if (SOF_EMP_DEEMP_BIQUADS != SOF_IIR_DF1_4TH_NUM_BIQUADS) return -EINVAL; - eq->coef = rzalloc(SOF_MEM_FLAG_USER, - sizeof(struct sof_eq_iir_biquad) * SOF_EMP_DEEMP_BIQUADS); + eq->coef = mod_zalloc(mod, sizeof(struct sof_eq_iir_biquad) * SOF_EMP_DEEMP_BIQUADS); if (!eq->coef) return -ENOMEM; @@ -86,8 +87,7 @@ static int multiband_drc_eq_init_coef_ch(struct sof_eq_iir_biquad *coef, * delay[0..1] -> state for first biquad * delay[2..3] -> state for second biquad */ - eq->delay = rzalloc(SOF_MEM_FLAG_USER, - sizeof(uint64_t) * CROSSOVER_NUM_DELAYS_LR4); + eq->delay = mod_zalloc(mod, sizeof(uint64_t) * CROSSOVER_NUM_DELAYS_LR4); if (!eq->delay) return -ENOMEM; @@ -142,13 +142,13 @@ static int multiband_drc_init_coef(struct processing_module *mod, int16_t nch, u /* Crossover: collect the coef array and assign it to every channel */ crossover = config->crossover_coef; for (ch = 0; ch < nch; ch++) { - ret = crossover_init_coef_ch(crossover, &state->crossover[ch], + ret = crossover_init_coef_ch(mod, crossover, &state->crossover[ch], config->num_bands); /* Free all previously allocated blocks in case of an error */ if (ret < 0) { comp_err(dev, "multiband_drc_init_coef(), could not assign coeffs to ch %d", ch); - goto err; + return ret; } } @@ -157,12 +157,12 @@ static int multiband_drc_init_coef(struct processing_module *mod, int16_t nch, u /* Emphasis: collect the coef array and assign it to every channel */ emphasis = config->emp_coef; for (ch = 0; ch < nch; ch++) { - ret = multiband_drc_eq_init_coef_ch(emphasis, &state->emphasis[ch]); + ret = multiband_drc_eq_init_coef_ch(mod, emphasis, &state->emphasis[ch]); /* Free all previously allocated blocks in case of an error */ if (ret < 0) { comp_err(dev, "multiband_drc_init_coef(), could not assign coeffs to ch %d", ch); - goto err; + return ret; } } @@ -171,12 +171,12 @@ static int multiband_drc_init_coef(struct processing_module *mod, int16_t nch, u /* Deemphasis: collect the coef array and assign it to every channel */ deemphasis = config->deemp_coef; for (ch = 0; ch < nch; ch++) { - ret = multiband_drc_eq_init_coef_ch(deemphasis, &state->deemphasis[ch]); + ret = multiband_drc_eq_init_coef_ch(mod, deemphasis, &state->deemphasis[ch]); /* Free all previously allocated blocks in case of an error */ if (ret < 0) { comp_err(dev, "multiband_drc_init_coef(), could not assign coeffs to ch %d", ch); - goto err; + return ret; } } @@ -184,26 +184,23 @@ static int multiband_drc_init_coef(struct processing_module *mod, int16_t nch, u for (i = 0; i < num_bands; i++) { comp_info(dev, "multiband_drc_init_coef(), initializing drc band %d", i); - ret = drc_init_pre_delay_buffers(&state->drc[i], (size_t)sample_bytes, (int)nch); + ret = drc_init_pre_delay_buffers(mod, &state->drc[i], + (size_t)sample_bytes, (int)nch); if (ret < 0) { comp_err(dev, "multiband_drc_init_coef(), could not init pre delay buffers"); - goto err; + return ret; } ret = drc_set_pre_delay_time(&state->drc[i], cd->config->drc_coef[i].pre_delay_time, rate); if (ret < 0) { comp_err(dev, "multiband_drc_init_coef(), could not set pre delay time"); - goto err; + return ret; } } return 0; - -err: - multiband_drc_reset_state(state); - return ret; } /* Called from multiband_drc_process(), so cannot be __cold */ @@ -213,7 +210,7 @@ static int multiband_drc_setup(struct processing_module *mod, int16_t channels, struct multiband_drc_comp_data *cd = module_get_private_data(mod); /* Reset any previous state */ - multiband_drc_reset_state(&cd->state); + multiband_drc_reset_state(mod, &cd->state); /* Setup Crossover, Emphasis EQ, Deemphasis EQ, and DRC */ return multiband_drc_init_coef(mod, channels, rate); @@ -243,7 +240,7 @@ static int multiband_drc_init(struct processing_module *mod) return -EINVAL; } - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = mod_zalloc(mod, sizeof(*cd)); if (!cd) return -ENOMEM; @@ -258,40 +255,29 @@ static int multiband_drc_init(struct processing_module *mod) multiband_drc_process_enable(&cd->process_enabled); /* Handler for configuration data */ - cd->model_handler = comp_data_blob_handler_new(dev); + cd->model_handler = mod_data_blob_handler_new(mod); if (!cd->model_handler) { comp_err(dev, "comp_data_blob_handler_new() failed."); - ret = -ENOMEM; - goto cd_fail; + return -ENOMEM; } /* Get configuration data and reset DRC state */ ret = comp_init_data_blob(cd->model_handler, bs, cfg->data); if (ret < 0) { comp_err(dev, "comp_init_data_blob() failed."); - goto cd_fail; + return ret; } - multiband_drc_reset_state(&cd->state); + multiband_drc_reset_state(mod, &cd->state); return 0; - -cd_fail: - comp_data_blob_handler_free(cd->model_handler); - rfree(cd); - return ret; } __cold static int multiband_drc_free(struct processing_module *mod) { - struct multiband_drc_comp_data *cd = module_get_private_data(mod); - assert_can_be_cold(); comp_info(mod->dev, "multiband_drc_free()"); - comp_data_blob_handler_free(cd->model_handler); - - rfree(cd); return 0; } @@ -415,7 +401,7 @@ static int multiband_drc_reset(struct processing_module *mod) comp_info(mod->dev, "multiband_drc_reset()"); - multiband_drc_reset_state(&cd->state); + multiband_drc_reset_state(mod, &cd->state); cd->source_format = 0; cd->multiband_drc_func = NULL; @@ -441,10 +427,8 @@ static const struct module_interface multiband_drc_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(multiband_drc, &multiband_drc_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("MB_DRC", multiband_drc_llext_entry, 1, + SOF_LLEXT_MODULE_MANIFEST("MB_DRC", &multiband_drc_interface, 1, SOF_REG_UUID(multiband_drc), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/multiband_drc/multiband_drc.h b/src/audio/multiband_drc/multiband_drc.h index 20f939877209..6a99fda55cef 100644 --- a/src/audio/multiband_drc/multiband_drc.h +++ b/src/audio/multiband_drc/multiband_drc.h @@ -89,10 +89,11 @@ static inline multiband_drc_func multiband_drc_find_proc_func_pass(enum sof_ipc_ return NULL; } -static inline void multiband_drc_iir_reset_state_ch(struct iir_state_df1 *iir) +static inline void multiband_drc_iir_reset_state_ch(struct processing_module *mod, + struct iir_state_df1 *iir) { - rfree(iir->coef); - rfree(iir->delay); + mod_free(mod, iir->coef); + mod_free(mod, iir->delay); iir->coef = NULL; iir->delay = NULL; diff --git a/src/audio/mux/mux.c b/src/audio/mux/mux.c index a4313e45f4ea..bbcf1b544e90 100644 --- a/src/audio/mux/mux.c +++ b/src/audio/mux/mux.c @@ -479,19 +479,12 @@ static const struct module_interface demux_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(mux, &mux_interface); - -/* - * The demux entry is removed because mtl.toml doesn't have an entry - * for it. Once that is fixed, the manifest line below can be - * re-activated: - * SOF_LLEXT_MOD_ENTRY(demux, &demux_interface); - */ - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("MUX", mux_llext_entry, 1, SOF_REG_UUID(mux4), 15), + SOF_LLEXT_MODULE_MANIFEST("MUX", &mux_interface, 1, SOF_REG_UUID(mux4), 15), /* - * See comment above for a demux deactivation reason + * The demux entry is removed because mtl.toml doesn't have an entry + * for it. Once that is fixed, the manifest line below can be + * re-activated: * SOF_LLEXT_MODULE_MANIFEST("DEMUX", demux_llext_entry, 1, SOF_REG_UUID(demux), 15), */ }; diff --git a/src/audio/rtnr/rtnr.c b/src/audio/rtnr/rtnr.c index 0078ce275b33..541f985753a1 100644 --- a/src/audio/rtnr/rtnr.c +++ b/src/audio/rtnr/rtnr.c @@ -880,10 +880,8 @@ static const struct module_interface rtnr_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(rtnr, &rtnr_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("RTNR", rtnr_llext_entry, 1, SOF_REG_UUID(rtnr), 40); + SOF_LLEXT_MODULE_MANIFEST("RTNR", &rtnr_interface, 1, SOF_REG_UUID(rtnr), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/selector/selector.c b/src/audio/selector/selector.c index d0f52a92303b..201ed9fa6567 100644 --- a/src/audio/selector/selector.c +++ b/src/audio/selector/selector.c @@ -930,11 +930,8 @@ static const struct module_interface selector_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(selector, &selector_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("MICSEL", selector_llext_entry, 1, SOF_REG_UUID(selector4), - 8); + SOF_LLEXT_MODULE_MANIFEST("MICSEL", &selector_interface, 1, SOF_REG_UUID(selector4), 8); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/src/src.c b/src/audio/src/src.c index 7f744f64d43d..1b3d6b3d3ca8 100644 --- a/src/audio/src/src.c +++ b/src/audio/src/src.c @@ -78,8 +78,6 @@ static const struct module_interface src_interface = { .prepare = src_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, - .set_configuration = src_set_config, - .get_configuration = src_get_config, .reset = src_reset, .free = src_free, }; @@ -91,18 +89,14 @@ static const struct module_interface src_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(src, &src_interface); - #if CONFIG_COMP_SRC_LITE extern const struct module_interface src_lite_interface; -SOF_LLEXT_MOD_ENTRY(src_lite, &src_lite_interface); #endif static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("SRC", src_llext_entry, 1, SOF_REG_UUID(src4), 1), + SOF_LLEXT_MODULE_MANIFEST("SRC", &src_interface, 1, SOF_REG_UUID(src4), 1), #if CONFIG_COMP_SRC_LITE - SOF_LLEXT_MODULE_MANIFEST("SRC_LITE", src_lite_llext_entry, 1, SOF_REG_UUID(src_lite), - 1), + SOF_LLEXT_MODULE_MANIFEST("SRC_LITE", &src_lite_interface, 1, SOF_REG_UUID(src_lite), 1), #endif }; diff --git a/src/audio/src/src_common.c b/src/audio/src/src_common.c index f96b3bc6e358..eb40f211a222 100644 --- a/src/audio/src/src_common.c +++ b/src/audio/src/src_common.c @@ -627,12 +627,11 @@ int src_allocate_copy_stages(struct processing_module *mod, struct src_param *pr return -EINVAL; } - stage_dst[0].coefs = fast_get(stage_src1->coefs, coef_size[0]); - stage_dst[1].coefs = fast_get(stage_src2->coefs, coef_size[1]); + stage_dst[0].coefs = mod_fast_get(mod, stage_src1->coefs, coef_size[0]); + stage_dst[1].coefs = mod_fast_get(mod, stage_src2->coefs, coef_size[1]); if (!stage_dst[0].coefs || !stage_dst[1].coefs) { comp_err(mod->dev, "failed to allocate coefficients"); - fast_put(stage_dst[0].coefs); return -ENOMEM; } @@ -650,9 +649,7 @@ bool src_is_ready_to_process(struct processing_module *mod, struct sof_source **sources, int num_of_sources, struct sof_sink **sinks, int num_of_sinks) { - struct comp_data *cd = module_get_private_data(mod); - - return src_get_copy_limits(cd, sources[0], sinks[0]); + return sink_get_free_frames(sinks[0]) || source_get_data_frames_available(sources[0]); } int src_process(struct processing_module *mod, @@ -672,24 +669,6 @@ int src_process(struct processing_module *mod, return cd->src_func(cd, sources[0], sinks[0]); } -__cold int src_set_config(struct processing_module *mod, uint32_t config_id, - enum module_cfg_fragment_position pos, uint32_t data_offset_size, - const uint8_t *fragment, size_t fragment_size, uint8_t *response, - size_t response_size) -{ - assert_can_be_cold(); - - return -EINVAL; -} - -__cold int src_get_config(struct processing_module *mod, uint32_t config_id, - uint32_t *data_offset_size, uint8_t *fragment, size_t fragment_size) -{ - assert_can_be_cold(); - - return -EINVAL; -} - int src_reset(struct processing_module *mod) { struct comp_data *cd = module_get_private_data(mod); @@ -708,12 +687,5 @@ __cold int src_free(struct processing_module *mod) comp_info(mod->dev, "src_free()"); -#if CONFIG_FAST_GET - struct comp_data *cd = module_get_private_data(mod); - if (cd->param.stage1) { - fast_put(cd->param.stage1->coefs); - fast_put(cd->param.stage2->coefs); - } -#endif return 0; } diff --git a/src/audio/src/src_common.h b/src/audio/src/src_common.h index 6a8028662ffd..ae3c60574eec 100644 --- a/src/audio/src/src_common.h +++ b/src/audio/src/src_common.h @@ -242,12 +242,6 @@ int src_process(struct processing_module *mod, struct sof_source **sources, int num_of_sources, struct sof_sink **sinks, int num_of_sinks); -int src_set_config(struct processing_module *mod, uint32_t config_id, - enum module_cfg_fragment_position pos, uint32_t data_offset_size, - const uint8_t *fragment, size_t fragment_size, uint8_t *response, - size_t response_size); -int src_get_config(struct processing_module *mod, uint32_t config_id, - uint32_t *data_offset_size, uint8_t *fragment, size_t fragment_size); int src_free(struct processing_module *mod); int src_reset(struct processing_module *mod); extern struct tr_ctx src_tr; diff --git a/src/audio/src/src_lite.c b/src/audio/src/src_lite.c index 4cbedea76ad2..d6704374d8cc 100644 --- a/src/audio/src/src_lite.c +++ b/src/audio/src/src_lite.c @@ -61,8 +61,6 @@ const struct module_interface src_lite_interface = { .prepare = src_lite_prepare, .process = src_process, .is_ready_to_process = src_is_ready_to_process, - .set_configuration = src_set_config, - .get_configuration = src_get_config, .reset = src_reset, .free = src_free, }; diff --git a/src/audio/tdfb/tdfb.c b/src/audio/tdfb/tdfb.c index e849b1dd0781..9fc04820ff75 100644 --- a/src/audio/tdfb/tdfb.c +++ b/src/audio/tdfb/tdfb.c @@ -833,10 +833,8 @@ static const struct module_interface tdfb_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(tdfb, &tdfb_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TDFB", tdfb_llext_entry, 1, SOF_REG_UUID(tdfb), 40); + SOF_LLEXT_MODULE_MANIFEST("TDFB", &tdfb_interface, 1, SOF_REG_UUID(tdfb), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/template_comp/template.c b/src/audio/template_comp/template.c index 7b15f1fbf591..fcd3fa294d03 100644 --- a/src/audio/template_comp/template.c +++ b/src/audio/template_comp/template.c @@ -197,10 +197,8 @@ static const struct module_interface template_comp_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(template_comp, &template_comp_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TEMPLATE", template_comp_llext_entry, 1, + SOF_LLEXT_MODULE_MANIFEST("TEMPLATE", &template_comp_interface, 1, SOF_REG_UUID(template_comp), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/tensorflow/tflm-classify.c b/src/audio/tensorflow/tflm-classify.c index d24099757c2d..29b4b6a1af74 100644 --- a/src/audio/tensorflow/tflm-classify.c +++ b/src/audio/tensorflow/tflm-classify.c @@ -243,10 +243,8 @@ SOF_MODULE_INIT(tflmcly, sys_comp_module_tflmcly_interface_init); #include #include -SOF_LLEXT_MOD_ENTRY(tflmcly, &tflmcly_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TFLMCLY", tflmcly_llext_entry, 1, SOF_REG_UUID(tflmcly), 40); + SOF_LLEXT_MODULE_MANIFEST("TFLMCLY", &tflmcly_interface, 1, SOF_REG_UUID(tflmcly), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/audio/volume/volume.c b/src/audio/volume/volume.c index 50aa4a4750d5..9f8aff265226 100644 --- a/src/audio/volume/volume.c +++ b/src/audio/volume/volume.c @@ -812,20 +812,12 @@ static const struct module_interface gain_interface = { #include #include -#if CONFIG_COMP_PEAK_VOL -SOF_LLEXT_MOD_ENTRY(peakvol, &volume_interface); -#endif - -#if CONFIG_COMP_GAIN -SOF_LLEXT_MOD_ENTRY(gain, &gain_interface); -#endif - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { #if CONFIG_COMP_PEAK_VOL - SOF_LLEXT_MODULE_MANIFEST("PEAKVOL", peakvol_llext_entry, 1, SOF_REG_UUID(volume4), 10), + SOF_LLEXT_MODULE_MANIFEST("PEAKVOL", &volume_interface, 1, SOF_REG_UUID(volume4), 10), #endif #if CONFIG_COMP_GAIN - SOF_LLEXT_MODULE_MANIFEST("GAIN", gain_llext_entry, 1, SOF_REG_UUID(gain), 40), + SOF_LLEXT_MODULE_MANIFEST("GAIN", &gain_interface, 1, SOF_REG_UUID(gain), 40), #endif }; diff --git a/src/debug/tester/tester.c b/src/debug/tester/tester.c index 73686ff49652..869b7d9b2a7a 100644 --- a/src/debug/tester/tester.c +++ b/src/debug/tester/tester.c @@ -243,10 +243,8 @@ static const struct module_interface tester_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(tester, &tester_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("TESTER", tester_llext_entry, 1, SOF_REG_UUID(tester), 40); + SOF_LLEXT_MODULE_MANIFEST("TESTER", &tester_interface, 1, SOF_REG_UUID(tester), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/idc/zephyr_idc.c b/src/idc/zephyr_idc.c index a0b2840f4ca2..9f6556bd18fc 100644 --- a/src/idc/zephyr_idc.c +++ b/src/idc/zephyr_idc.c @@ -185,6 +185,11 @@ void idc_init_thread(void) k_p4wq_enable_static_thread(q_zephyr_idc + cpu, _p4threads_q_zephyr_idc + cpu, BIT(cpu)); + /* + * Assign SOF system heap to the IDC thread. Otherwise by default it + * uses the Zephyr heap for DP stack allocation + */ + k_thread_heap_assign(_p4threads_q_zephyr_idc + cpu, sof_sys_heap_get()); } #endif /* CONFIG_MULTICORE */ diff --git a/src/include/module/crossover/crossover_common.h b/src/include/module/crossover/crossover_common.h index 28c6e8aa7775..d238ff7c1008 100644 --- a/src/include/module/crossover/crossover_common.h +++ b/src/include/module/crossover/crossover_common.h @@ -8,6 +8,7 @@ #ifndef __SOF_CROSSOVER_COMMON_H__ #define __SOF_CROSSOVER_COMMON_H__ +#include #include #include @@ -39,17 +40,19 @@ typedef void (*crossover_split)(int32_t in, int32_t out[], extern const crossover_split crossover_split_fnmap[]; /* crossover init function */ -int crossover_init_coef_ch(struct sof_eq_iir_biquad *coef, +int crossover_init_coef_ch(struct processing_module *mod, + struct sof_eq_iir_biquad *coef, struct crossover_state *ch_state, int32_t num_sinks); /** * \brief Reset the state of an LR4 filter. */ -static inline void crossover_reset_state_lr4(struct iir_state_df1 *lr4) +static inline void crossover_reset_state_lr4(struct processing_module *mod, + struct iir_state_df1 *lr4) { - rfree(lr4->coef); - rfree(lr4->delay); + mod_free(mod, lr4->coef); + mod_free(mod, lr4->delay); lr4->coef = NULL; lr4->delay = NULL; @@ -59,13 +62,14 @@ static inline void crossover_reset_state_lr4(struct iir_state_df1 *lr4) * \brief Reset the state (coefficients and delay) of the crossover filter * of a single channel. */ -static inline void crossover_reset_state_ch(struct crossover_state *ch_state) +static inline void crossover_reset_state_ch(struct processing_module *mod, + struct crossover_state *ch_state) { int i; for (i = 0; i < CROSSOVER_MAX_LR4; i++) { - crossover_reset_state_lr4(&ch_state->lowpass[i]); - crossover_reset_state_lr4(&ch_state->highpass[i]); + crossover_reset_state_lr4(mod, &ch_state->lowpass[i]); + crossover_reset_state_lr4(mod, &ch_state->highpass[i]); } } diff --git a/src/include/module/module/base.h b/src/include/module/module/base.h index 2371d77124f5..47b9309f0518 100644 --- a/src/include/module/module/base.h +++ b/src/include/module/module/base.h @@ -16,6 +16,10 @@ #include "interface.h" #include "../ipc4/base-config.h" +#ifdef SOF_MODULE_API_PRIVATE +#include +#endif + #define module_get_private_data(mod) ((mod)->priv.private) #define module_set_private_data(mod, data) ((mod)->priv.private = data) @@ -59,7 +63,7 @@ struct module_data { enum module_state state; size_t new_cfg_size; /**< size of new module config data */ void *runtime_params; - struct module_memory memory; /**< memory allocated by module */ + struct module_resources resources; /**< resources allocated by module */ struct module_processing_data mpd; /**< shared data comp <-> module */ #endif /* SOF_MODULE_PRIVATE */ }; diff --git a/src/include/module/module/llext.h b/src/include/module/module/llext.h index 7b5d00feeb30..2f05cb6c692a 100644 --- a/src/include/module/module/llext.h +++ b/src/include/module/module/llext.h @@ -33,13 +33,6 @@ } \ } -#define SOF_LLEXT_MOD_ENTRY(name, interface) \ -static const struct module_interface *name##_llext_entry(void *mod_cfg, \ - void *parent_ppl, void **mod_ptr) \ -{ \ - return interface; \ -} - #define SOF_LLEXT_BUILDINFO \ static const struct sof_module_api_build_info buildinfo __section(".mod_buildinfo") __used = { \ .format = SOF_MODULE_API_BUILD_INFO_FORMAT, \ diff --git a/src/include/sof/audio/audio_buffer.h b/src/include/sof/audio/audio_buffer.h index a4b52248ce59..81b4563756dc 100644 --- a/src/include/sof/audio/audio_buffer.h +++ b/src/include/sof/audio/audio_buffer.h @@ -110,6 +110,8 @@ struct sof_audio_buffer { * should not be in struct sof_audio_buffer at all, kept for pipeline2.0 transition */ bool walking; /**< indicates if the buffer is being walked */ + + struct k_heap *heap; }; #if CONFIG_PIPELINE_2_0 diff --git a/src/include/sof/audio/buffer.h b/src/include/sof/audio/buffer.h index c8b98255ed4f..8b0e1bb521fb 100644 --- a/src/include/sof/audio/buffer.h +++ b/src/include/sof/audio/buffer.h @@ -207,12 +207,16 @@ struct buffer_cb_free { buffer->cb_type = type; \ } while (0) +struct k_heap; + /* pipeline buffer creation and destruction */ -struct comp_buffer *buffer_alloc(size_t size, uint32_t flags, uint32_t align, +struct comp_buffer *buffer_alloc(struct k_heap *heap, size_t size, uint32_t flags, uint32_t align, bool is_shared); -struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_size, +struct comp_buffer *buffer_alloc_range(struct k_heap *heap, size_t preferred_size, + size_t minimum_size, uint32_t flags, uint32_t align, bool is_shared); -struct comp_buffer *buffer_new(const struct sof_ipc_buffer *desc, bool is_shared); +struct comp_buffer *buffer_new(struct k_heap *heap, const struct sof_ipc_buffer *desc, + bool is_shared); int buffer_set_size(struct comp_buffer *buffer, uint32_t size, uint32_t alignment); int buffer_set_size_range(struct comp_buffer *buffer, size_t preferred_size, size_t minimum_size, diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 7f71af8f8025..da9edc490fc7 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -847,6 +847,24 @@ static inline enum sof_comp_type dev_comp_type(const struct comp_dev *dev) return dev->ipc_config.type; } +/** + * Initialize common part of a component device + * @param drv Parent component driver. + * @param dev Device. + * @param bytes Size of the component device in bytes. + */ +static inline void comp_init(const struct comp_driver *drv, + struct comp_dev *dev, size_t bytes) +{ + dev->size = bytes; + dev->drv = drv; + dev->state = COMP_STATE_INIT; + list_init(&dev->bsink_list); + list_init(&dev->bsource_list); + memcpy_s(&dev->tctx, sizeof(struct tr_ctx), + trace_comp_drv_get_tr_ctx(dev->drv), sizeof(struct tr_ctx)); +} + /** * Allocates memory for the component device and initializes common part. * @param drv Parent component driver. @@ -856,22 +874,16 @@ static inline enum sof_comp_type dev_comp_type(const struct comp_dev *dev) static inline struct comp_dev *comp_alloc(const struct comp_driver *drv, size_t bytes) { - struct comp_dev *dev = NULL; - /* * Use uncached address everywhere to access components to rule out * multi-core failures. TODO: verify if cached alias may be used in some cases */ - dev = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, bytes); + struct comp_dev *dev = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, bytes); + if (!dev) return NULL; - dev->size = bytes; - dev->drv = drv; - dev->state = COMP_STATE_INIT; - list_init(&dev->bsink_list); - list_init(&dev->bsource_list); - memcpy_s(&dev->tctx, sizeof(struct tr_ctx), - trace_comp_drv_get_tr_ctx(dev->drv), sizeof(struct tr_ctx)); + + comp_init(drv, dev, bytes); return dev; } diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 3370cc110d2e..ca42cc8503a1 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -46,14 +46,10 @@ static inline void comp_free(struct comp_dev *dev) { assert(dev->drv->ops.free); - /* free task if shared component or DP task*/ - if ((dev->is_shared || dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) && - dev->task) { - schedule_task_free(dev->task); - rfree(dev->task); - dev->task = NULL; - } - + /* + * In DP case this will run in DP thread context, so the task can only + * be freed after this. + */ dev->drv->ops.free(dev); } @@ -164,7 +160,6 @@ static inline int comp_trigger_local(struct comp_dev *dev, int cmd) case COMP_TRIGGER_XRUN: case COMP_TRIGGER_PAUSE: case COMP_TRIGGER_STOP: - schedule_task_cancel(dev->task); break; } } diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 1001c4ba450c..9b48e14dc901 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -19,6 +19,12 @@ #include #include "module_interface.h" +/* The __ZEPHYR__ condition is to keep cmocka tests working */ +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) +#include +#endif +#include + /* * helpers to determine processing type * Needed till all the modules use PROCESSING_MODE_SINK_SOURCE @@ -117,12 +123,33 @@ struct module_param { /** * \struct module_memory - * \brief module memory block - used for every memory allocated by module + * \brief module resources block - used for module allocation records + * The allocations are recorded so that they can be automatically freed + * when the module unloads. + */ +struct module_resources { + struct list_item mem_list; /**< Allocad memory containers */ + struct list_item free_cont_list; /**< Unused memory containers */ + struct list_item cont_chunk_list; /**< Memory container chunks */ + size_t heap_usage; + size_t heap_high_water_mark; + struct k_heap *heap; + void *heap_mem; + size_t heap_size; +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) + k_tid_t rsrc_mngr; +#endif +}; + +/** + * \struct module_memory + * \brief module memory container - used for every memory allocated by module */ struct module_memory { - void *ptr; /**< A pointr to particular memory block */ + void *ptr; /**< A pointer to particular memory block */ struct list_item mem_list; /**< list of memory allocated by module */ - size_t size; + size_t size; /**< Size of allocated heap memory, 0 if not from heap */ + void (*free)(struct k_heap *heap, void *buf); /**< Pointer to custom free function */ }; /** @@ -154,10 +181,76 @@ struct module_processing_data { /*****************************************************************************/ int module_load_config(struct comp_dev *dev, const void *cfg, size_t size); int module_init(struct processing_module *mod); -void *mod_alloc_align(struct processing_module *mod, uint32_t size, uint32_t alignment); -void *mod_alloc(struct processing_module *mod, uint32_t size); -void *mod_zalloc(struct processing_module *mod, uint32_t size); -int mod_free(struct processing_module *mod, void *ptr); +void mod_resource_init(struct processing_module *mod); +void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start); +__syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, + size_t alignment); +__syscall int mod_free(struct processing_module *mod, const void *ptr); +#include + +/** + * Allocates aligned memory block for module. + * @param mod Pointer to the module this memory block is allocatd for. + * @param bytes Size in bytes. + * @param alignment Alignment in bytes. + * @return Pointer to the allocated memory or NULL if failed. + * + * The allocated memory is automatically freed when the module is unloaded. + */ +static inline void *mod_alloc_align(struct processing_module *mod, size_t size, size_t alignment) +{ + return mod_alloc_ext(mod, 0, size, alignment); +} + +/** + * Allocates memory block for module. + * @param mod Pointer to module this memory block is allocated for. + * @param bytes Size in bytes. + * @return Pointer to the allocated memory or NULL if failed. + * + * Like mod_alloc_align() but the alignment cannot be specified. However, + * rballoc() will always aligns the memory to PLATFORM_DCACHE_ALIGN. + */ +static inline void *mod_alloc(struct processing_module *mod, size_t size) +{ + return mod_alloc_align(mod, size, 0); +} + +/** + * Allocates memory block for module and initializes it to zero. + * @param mod Pointer to module this memory block is allocated for. + * @param bytes Size in bytes. + * @return Pointer to the allocated memory or NULL if failed. + * + * Like mod_alloc() but the allocated memory is initialized to zero. + */ +static inline void *mod_zalloc(struct processing_module *mod, size_t size) +{ + void *ret = mod_alloc(mod, size); + + if (ret) + memset(ret, 0, size); + + return ret; +} + +#if CONFIG_COMP_BLOB +struct comp_data_blob_handler *mod_data_blob_handler_new(struct processing_module *mod); +static inline void mod_data_blob_handler_free(struct processing_module *mod, + struct comp_data_blob_handler *dbh) +{ + mod_free(mod, dbh); +} +#endif +#if CONFIG_FAST_GET +__syscall const void *mod_fast_get(struct processing_module *mod, const void * const dram_ptr, + size_t size); +static inline void mod_fast_put(struct processing_module *mod, const void *sram_ptr) +{ + mod_free(mod, sram_ptr); +} +#endif + void mod_free_all(struct processing_module *mod); int module_prepare(struct processing_module *mod, struct sof_source **sources, int num_of_sources, @@ -223,7 +316,7 @@ int module_adapter_trigger(struct comp_dev *dev, int cmd); void module_adapter_free(struct comp_dev *dev); int module_adapter_reset(struct comp_dev *dev); -size_t module_adapter_heap_usage(struct processing_module *mod); +size_t module_adapter_heap_usage(struct processing_module *mod, size_t *hwm); #if CONFIG_IPC_MAJOR_3 static inline diff --git a/src/include/sof/audio/ring_buffer.h b/src/include/sof/audio/ring_buffer.h index 222b102f2ae9..483783f22112 100644 --- a/src/include/sof/audio/ring_buffer.h +++ b/src/include/sof/audio/ring_buffer.h @@ -113,16 +113,15 @@ struct ring_buffer { }; /** - * + * @param heap Zephyr heap to allocate memory on * @param min_available minimum data available in queue required by the module using * ring_buffer's source api * @param min_free_space minimum buffer space in queue required by the module using * ring_buffer's sink api * @param is_shared indicates if the buffer will be shared between cores * @param id a stream ID, accessible later by sink_get_id/source_get_id - * */ -struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, bool is_shared, - uint32_t id); +struct ring_buffer *ring_buffer_create(struct k_heap *heap, size_t min_available, + size_t min_free_space, bool is_shared, uint32_t id); #endif /* __SOF_RING_BUFFER_H__ */ diff --git a/src/include/sof/lib/fast-get.h b/src/include/sof/lib/fast-get.h index ffbac19cf1b8..b9b132650af8 100644 --- a/src/include/sof/lib/fast-get.h +++ b/src/include/sof/lib/fast-get.h @@ -9,8 +9,11 @@ #define __SOF_LIB_FAST_GET_H__ #include +#include -const void *fast_get(const void * const dram_ptr, size_t size); -void fast_put(const void *sram_ptr); +struct k_heap; +__syscall const void *fast_get(struct k_heap *heap, const void * const dram_ptr, size_t size); +__syscall void fast_put(struct k_heap *heap, const void *sram_ptr); +#include #endif /* __SOF_LIB_FAST_GET_H__ */ diff --git a/src/include/sof/llext_manager.h b/src/include/sof/llext_manager.h index 44928c5a124f..525fa50b1506 100644 --- a/src/include/sof/llext_manager.h +++ b/src/include/sof/llext_manager.h @@ -15,6 +15,7 @@ struct comp_dev; struct comp_driver; struct comp_ipc_config; +struct k_mem_domain; static inline bool module_is_llext(const struct sof_man_module *mod) { @@ -29,12 +30,16 @@ int llext_manager_free_module(const uint32_t component_id); int llext_manager_add_library(uint32_t module_id); +int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain); +int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain); + bool comp_is_llext(struct comp_dev *comp); #else #define module_is_llext(mod) false #define llext_manager_allocate_module(ipc_config, ipc_specific_config) 0 #define llext_manager_free_module(component_id) 0 #define llext_manager_add_library(module_id) 0 +#define llext_manager_add_domain(component_id, domain) 0 #define comp_is_llext(comp) false #endif diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 0a064204e8d1..195f17fe7116 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include struct processing_module; @@ -84,4 +86,37 @@ int scheduler_dp_task_init(struct task **task, void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size); +struct bind_info; +struct sof_source; +struct sof_sink; +union scheduler_dp_thread_ipc_param { + struct bind_info *bind_data; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + struct sof_source **sources; + int n_sinks; + struct sof_sink **sinks; + } pipeline_state; +}; + +struct dp_heap_user { + struct k_heap heap; + unsigned int client_count; /* devices and buffers */ + // lock; /* protect the counter */ +}; + +#if CONFIG_ZEPHYR_DP_SCHEDULER +int scheduler_dp_thread_ipc(struct processing_module *pmod, enum sof_ipc4_module_type cmd, + union scheduler_dp_thread_ipc_param *param); +#else +static inline int scheduler_dp_thread_ipc(struct processing_module *pmod, + enum sof_ipc4_module_type cmd, + union scheduler_dp_thread_ipc_param *param) +{ + return 0; +} +#endif + #endif /* __SOF_SCHEDULE_DP_SCHEDULE_H__ */ diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 896fedf15463..21ee87606a1d 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -50,7 +50,8 @@ __cold static bool valid_ipc_buffer_desc(const struct sof_ipc_buffer *desc) } /* create a new component in the pipeline */ -__cold struct comp_buffer *buffer_new(const struct sof_ipc_buffer *desc, bool is_shared) +__cold struct comp_buffer *buffer_new(struct k_heap *heap, const struct sof_ipc_buffer *desc, + bool is_shared) { struct comp_buffer *buffer; uint32_t flags = desc->flags; @@ -78,7 +79,7 @@ __cold struct comp_buffer *buffer_new(const struct sof_ipc_buffer *desc, bool is desc->caps, flags); /* allocate buffer */ - buffer = buffer_alloc(desc->size, flags, PLATFORM_DCACHE_ALIGN, + buffer = buffer_alloc(heap, desc->size, flags, PLATFORM_DCACHE_ALIGN, is_shared); if (buffer) { buffer->stream.runtime_stream_params.id = desc->comp.id; diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 4e2f54db3553..68b98da1655a 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -391,7 +392,7 @@ __cold int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool is_shared, uint32_t buf_size, uint32_t src_queue, - uint32_t dst_queue) + uint32_t dst_queue, struct k_heap *heap) { struct sof_ipc_buffer ipc_buf; @@ -402,7 +403,7 @@ __cold static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool ipc_buf.comp.id = IPC4_COMP_ID(src_queue, dst_queue); ipc_buf.comp.pipeline_id = src->ipc_config.pipeline_id; ipc_buf.comp.core = cpu_get_id(); - return buffer_new(&ipc_buf, is_shared); + return buffer_new(heap, &ipc_buf, is_shared); } #if CONFIG_CROSS_CORE_STREAM @@ -578,13 +579,38 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) else buf_size = ibs * 2; + struct k_heap *dp_heap; + +#if CONFIG_ZEPHYR_DP_SCHEDULER + struct comp_dev *dp; + + if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + dp = sink; + else if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + dp = source; + else + dp = NULL; + + dp_heap = dp && dp->mod ? dp->mod->priv.resources.heap : NULL; +#else + dp_heap = NULL; +#endif + buffer = ipc4_create_buffer(source, cross_core_bind, buf_size, bu->extension.r.src_queue, - bu->extension.r.dst_queue); + bu->extension.r.dst_queue, dp_heap); if (!buffer) { tr_err(&ipc_tr, "failed to allocate buffer to bind %d to %d", src_id, sink_id); return IPC4_OUT_OF_MEMORY; } +#if CONFIG_ZEPHYR_DP_SCHEDULER + if (dp_heap) { + struct dp_heap_user *dp_user = CONTAINER_OF(dp_heap, struct dp_heap_user, heap); + + dp_user->client_count++; + } +#endif + /* * set min_free_space and min_available in sink/src api of created buffer. * buffer is connected like: @@ -603,25 +629,27 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP || source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { - struct sof_source *source = audio_buffer_get_source(&buffer->audio_buffer); - struct sof_sink *sink = audio_buffer_get_sink(&buffer->audio_buffer); + struct sof_source *buf_source = audio_buffer_get_source(&buffer->audio_buffer); + struct sof_sink *buf_sink = audio_buffer_get_sink(&buffer->audio_buffer); - ring_buffer = ring_buffer_create(source_get_min_available(source), - sink_get_min_free_space(sink), + ring_buffer = ring_buffer_create(dp_heap, source_get_min_available(buf_source), + sink_get_min_free_space(buf_sink), audio_buffer_is_shared(&buffer->audio_buffer), buf_get_id(buffer)); if (!ring_buffer) goto free; - } - if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) - /* data destination module needs to use ring_buffer */ - audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, false, /* at_input */ - &ring_buffer->audio_buffer); - else if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) - /* data source module needs to use ring_buffer */ - audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, true, /* at_input */ - &ring_buffer->audio_buffer); + if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + /* data destination module needs to use ring_buffer */ + audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, + false /* at_input */, + &ring_buffer->audio_buffer); + else + /* data source module needs to use ring_buffer */ + audio_buffer_attach_secondary_buffer(&buffer->audio_buffer, + true /* at_input */, + &ring_buffer->audio_buffer); + } #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ /* * Connect and bind the buffer to both source and sink components with LL processing been diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 31308d846e2a..9fe9d68cfd0b 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -412,8 +412,8 @@ int lib_manager_free_module(const uint32_t component_id) #define PAGE_SZ 4096 /* equals to MAN_PAGE_SIZE used by rimage */ -uintptr_t lib_manager_allocate_module(const struct comp_ipc_config *ipc_config, - const void *ipc_specific_config, const void **buildinfo) +static uintptr_t lib_manager_allocate_module(const struct comp_ipc_config *ipc_config, + const void *ipc_specific_config, const void **buildinfo) { tr_err(&lib_manager_tr, "Dynamic module allocation is not supported"); return 0; @@ -578,7 +578,7 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, const void *spec) { const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(config->id); - const struct ipc_config_process *args = (struct ipc_config_process *)spec; + const struct ipc_config_process *args = (const struct ipc_config_process *)spec; const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(config->id); const struct module_interface *ops = NULL; const struct sof_man_module *mod; @@ -603,8 +603,18 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, mod = (const struct sof_man_module *) ((const uint8_t *)desc + SOF_MAN_MODULE_OFFSET(entry_index)); + const uintptr_t module_entry_point = lib_manager_allocate_module(mod, config, args->data); + + if (!module_entry_point) { + tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); + return NULL; + } + switch (lib_manager_get_module_type(desc, mod)) { case MOD_TYPE_LLEXT: + agent = NULL; + ops = (const struct module_interface *)module_entry_point; + break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; agent_iface = (const void **)&ops; @@ -617,21 +627,16 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, agent_iface = (const void **)&adapter_priv; break; case MOD_TYPE_INVALID: - return NULL; - } - - const uintptr_t module_entry_point = lib_manager_allocate_module(mod, config, args->data); - if (!module_entry_point) { - tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); - return NULL; + goto err; } /* At this point module resources are allocated and it is moved to L2 memory. */ - - ret = lib_manager_start_agent(drv, config->id, args, module_entry_point, agent, - agent_iface); - if (ret) - goto err; + if (agent) { + ret = lib_manager_start_agent(drv, config->id, args, module_entry_point, agent, + agent_iface); + if (ret) + goto err; + } if (ops && comp_set_adapter_ops(drv, ops) < 0) goto err; diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 9c032319f82d..342621d7d0e5 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -716,6 +717,124 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config return mod_manifest->module.entry_point; } +#ifdef CONFIG_USERSPACE +static int llext_manager_add_partition(struct k_mem_domain *domain, + uintptr_t addr, size_t size, + k_mem_partition_attr_t attr) +{ + size_t pre_pad_size = addr & (PAGE_SZ - 1); + struct k_mem_partition part = { + .start = addr - pre_pad_size, + .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), + .attr = attr, + }; + + tr_dbg(&lib_manager_tr, "add %#zx @ %lx partition", part.size, part.start); + return k_mem_domain_add_partition(domain, &part); +} + +int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain) +{ + const uint32_t module_id = IPC4_MOD_ID(component_id); + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); + const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); + const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + struct lib_manager_module *mctx = ctx->mod + mod_idx; + + /* Executable code (.text) */ + uintptr_t va_base_text = mctx->segment[LIB_MANAGER_TEXT].addr; + size_t text_size = mctx->segment[LIB_MANAGER_TEXT].size; + + /* Read-only data (.rodata and others) */ + uintptr_t va_base_rodata = mctx->segment[LIB_MANAGER_RODATA].addr; + size_t rodata_size = mctx->segment[LIB_MANAGER_RODATA].size; + + /* Writable data (.data, .bss and others) */ + uintptr_t va_base_data = mctx->segment[LIB_MANAGER_DATA].addr; + size_t data_size = mctx->segment[LIB_MANAGER_DATA].size; + + int ret = llext_manager_add_partition(domain, va_base_text, text_size, + K_MEM_PARTITION_P_RX_U_RX); + + if (ret < 0) + return ret; + + if (rodata_size) { + ret = llext_manager_add_partition(domain, va_base_rodata, rodata_size, + K_MEM_PARTITION_P_RO_U_RO); + if (ret < 0) + return ret; + } + + if (data_size) { + ret = llext_manager_add_partition(domain, va_base_data, data_size, + K_MEM_PARTITION_P_RW_U_RW); + if (ret < 0) + return ret; + } + + return 0; +} + +static int llext_manager_rm_partition(struct k_mem_domain *domain, + uintptr_t addr, size_t size, + k_mem_partition_attr_t attr) +{ + size_t pre_pad_size = addr & (PAGE_SZ - 1); + struct k_mem_partition part = { + .start = addr - pre_pad_size, + .size = ALIGN_UP(pre_pad_size + size, PAGE_SZ), + .attr = attr, + }; + + tr_dbg(&lib_manager_tr, "remove %#zx @ %lx partition", part.size, part.start); + return k_mem_domain_remove_partition(domain, &part); +} + +int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain) +{ + const uint32_t module_id = IPC4_MOD_ID(component_id); + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); + const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); + const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + struct lib_manager_module *mctx = ctx->mod + mod_idx; + + /* Executable code (.text) */ + uintptr_t va_base_text = mctx->segment[LIB_MANAGER_TEXT].addr; + size_t text_size = mctx->segment[LIB_MANAGER_TEXT].size; + + /* Read-only data (.rodata and others) */ + uintptr_t va_base_rodata = mctx->segment[LIB_MANAGER_RODATA].addr; + size_t rodata_size = mctx->segment[LIB_MANAGER_RODATA].size; + + /* Writable data (.data, .bss and others) */ + uintptr_t va_base_data = mctx->segment[LIB_MANAGER_DATA].addr; + size_t data_size = mctx->segment[LIB_MANAGER_DATA].size; + + int ret = llext_manager_rm_partition(domain, va_base_text, text_size, + K_MEM_PARTITION_P_RX_U_RX); + + if (ret < 0) + return ret; + + if (rodata_size) { + ret = llext_manager_rm_partition(domain, va_base_rodata, rodata_size, + K_MEM_PARTITION_P_RO_U_RO); + if (ret < 0) + return ret; + } + + if (data_size) { + ret = llext_manager_rm_partition(domain, va_base_data, data_size, + K_MEM_PARTITION_P_RW_U_RW); + if (ret < 0) + return ret; + } + + return 0; +} +#endif + int llext_manager_free_module(const uint32_t component_id) { const uint32_t module_id = IPC4_MOD_ID(component_id); diff --git a/src/probe/probe.c b/src/probe/probe.c index 7a9cb4121573..b1c095deeda4 100644 --- a/src/probe/probe.c +++ b/src/probe/probe.c @@ -80,11 +80,11 @@ struct probe_dma_ext { * Probe main struct */ struct probe_pdata { + struct task dmap_work; /**< probe task */ struct probe_dma_ext ext_dma; /**< extraction DMA */ struct probe_dma_ext inject_dma[CONFIG_PROBE_DMA_MAX]; /**< injection DMA */ struct probe_point probe_points[CONFIG_PROBE_POINTS_MAX]; /**< probe points */ struct probe_data_packet header; /**< data packet header */ - struct task dmap_work; /**< probe task */ }; /** @@ -1660,10 +1660,8 @@ static const struct module_interface probe_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(probe, &probe_interface); - static const struct sof_man_module_manifest mod_manifest __section(".module") __used = - SOF_LLEXT_MODULE_MANIFEST("PROBE", probe_llext_entry, 1, SOF_REG_UUID(probe4), 40); + SOF_LLEXT_MODULE_MANIFEST("PROBE", &probe_interface, 1, SOF_REG_UUID(probe4), 40); SOF_LLEXT_BUILDINFO; diff --git a/src/samples/audio/smart_amp_test_ipc4.c b/src/samples/audio/smart_amp_test_ipc4.c index 53f42b43f03b..52e92699e669 100644 --- a/src/samples/audio/smart_amp_test_ipc4.c +++ b/src/samples/audio/smart_amp_test_ipc4.c @@ -381,10 +381,8 @@ static const struct module_interface smart_amp_test_interface = { #include #include -SOF_LLEXT_MOD_ENTRY(smart_amp_test, &smart_amp_test_interface); - static const struct sof_man_module_manifest mod_manifest[] __section(".module") __used = { - SOF_LLEXT_MODULE_MANIFEST("SMATEST", smart_amp_test_llext_entry, 1, + SOF_LLEXT_MODULE_MANIFEST("SMATEST", &smart_amp_test_interface, 1, SOF_REG_UUID(smart_amp_test), 1), }; diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index cbb36391b310..cae225d93b9f 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -27,20 +29,35 @@ SOF_DEFINE_REG_UUID(dp_sched); DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); +// FIXME: should be per-DP task +K_SEM_DEFINE(dp_thread_ipc_sem, 0, K_SEM_MAX_LIMIT); +K_SEM_DEFINE(dp_thread_sync_sem, 0, 1); + +static struct k_spinlock thread_lock; +static struct k_mem_domain dp_mdom; + struct scheduler_dp_data { struct list_item tasks; /* list of active dp tasks */ struct task ll_tick_src; /* LL task - source of DP tick */ }; +enum sof_dp_part_type { + SOF_DP_PART_HEAP, + SOF_DP_PART_IPC, + SOF_DP_PART_CFG, + SOF_DP_PART_TYPE_COUNT, +}; + struct task_dp_pdata { k_tid_t thread_id; /* zephyr thread ID */ - struct k_thread thread; /* memory space for a thread */ + struct k_thread *thread; /* memory space for a thread */ uint32_t deadline_clock_ticks; /* dp module deadline in Zephyr ticks */ - k_thread_stack_t __sparse_cache *p_stack; /* pointer to thread stack */ - size_t stack_size; /* size of the stack in bytes */ - struct k_sem sem; /* semaphore for task scheduling */ + k_thread_stack_t *p_stack; /* pointer to thread stack */ struct processing_module *mod; /* the module to be scheduled */ uint32_t ll_cycles_to_start; /* current number of LL cycles till delayed start */ + unsigned char pend_ipc; + unsigned char pend_proc; + struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; }; /* Single CPU-wide lock @@ -215,7 +232,7 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 * needed 1.2ms for processing - but the example would be too complicated) */ -void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void *caller_data) +static void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void *caller_data) { (void)receiver_data; (void)event_type; @@ -242,23 +259,23 @@ void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void * mod->dp_startup_delay = false; } - if (curr_task->state == SOF_TASK_STATE_QUEUED) { - bool mod_ready; - - mod_ready = module_is_ready_to_process(mod, mod->sources, - mod->num_of_sources, - mod->sinks, - mod->num_of_sinks); - if (mod_ready) { - /* set a deadline for given num of ticks, starting now */ - k_thread_deadline_set(pdata->thread_id, - pdata->deadline_clock_ticks); - - /* trigger the task */ - curr_task->state = SOF_TASK_STATE_RUNNING; - k_sem_give(&pdata->sem); - } - } + if (curr_task->state != SOF_TASK_STATE_QUEUED || + mod->dev->state < COMP_STATE_ACTIVE) + continue; + + /* set a deadline for given num of ticks, starting now */ + k_thread_deadline_set(pdata->thread_id, + pdata->deadline_clock_ticks); + + /* trigger the task */ + curr_task->state = SOF_TASK_STATE_RUNNING; + + k_spinlock_key_t key = k_spin_lock(&thread_lock); + + pdata->pend_proc++; + k_sem_give(&dp_thread_ipc_sem); + + k_spin_unlock(&thread_lock, key); } scheduler_dp_unlock(lock_key); } @@ -269,6 +286,8 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; + if (!pdata->thread_id) + return 0; /* this is asyn cancel - mark the task as canceled and remove it from scheduling */ lock_key = scheduler_dp_lock(); @@ -276,17 +295,18 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) task->state = SOF_TASK_STATE_CANCEL; list_item_del(&task->list); - /* if there're no more DP task, stop LL tick source */ + /* if there're no more DP task, stop LL tick source */ if (list_is_empty(&dp_sch->tasks)) schedule_task_cancel(&dp_sch->ll_tick_src); /* if the task is waiting on a semaphore - let it run and self-terminate */ - k_sem_give(&pdata->sem); scheduler_dp_unlock(lock_key); /* wait till the task has finished, if there was any task created */ - if (pdata->thread_id) - k_thread_join(pdata->thread_id, K_FOREVER); + k_thread_abort(pdata->thread_id); + k_object_free(pdata->thread); + + pdata->thread_id = NULL; return 0; } @@ -297,75 +317,270 @@ static int scheduler_dp_task_free(void *data, struct task *task) scheduler_dp_task_cancel(data, task); - /* the thread should be terminated at this moment, - * abort is safe and will ensure no use after free - */ - if (pdata->thread_id) { - k_thread_abort(pdata->thread_id); - pdata->thread_id = NULL; - } + pdata->thread_id = NULL; /* free task stack */ - rfree((__sparse_force void *)pdata->p_stack); + k_thread_stack_free(pdata->p_stack); pdata->p_stack = NULL; + k_mem_domain_remove_partition(&dp_mdom, pdata->mpart + SOF_DP_PART_HEAP); + k_mem_domain_remove_partition(&dp_mdom, pdata->mpart + SOF_DP_PART_IPC); + k_mem_domain_remove_partition(&dp_mdom, pdata->mpart + SOF_DP_PART_CFG); + + llext_manager_rm_domain(pdata->mod->dev->ipc_config.id, &dp_mdom); + /* all other memory has been allocated as a single malloc, will be freed later by caller */ return 0; } +/* TODO: make this a shared kernel->module buffer for IPC parameters */ +static uint8_t ipc_buf[4096] __aligned(4096); + +struct ipc4_flat { + unsigned int cmd; + int ret; + union { + struct { + struct ipc4_module_bind_unbind bu; + enum bind_type type; + } bind; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + int n_sinks; + void *source_sink[]; + } pipeline_state; + }; +}; + +/* Pack IPC input data */ +static int ipc_thread_flatten(unsigned int cmd, union scheduler_dp_thread_ipc_param *param, + struct ipc4_flat *flat) +{ + flat->cmd = cmd; + + /* + * FIXME: SOF_IPC4_MOD_* and SOF_IPC4_GLB_* aren't fully orthogonal, but + * so far none of the used ones overlap + */ + switch (cmd) { + case SOF_IPC4_MOD_BIND: + flat->bind.bu = *param->bind_data->ipc4_data; + flat->bind.type = param->bind_data->bind_type; + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: + flat->pipeline_state.trigger_cmd = param->pipeline_state.trigger_cmd; + switch (param->pipeline_state.trigger_cmd) { + case COMP_TRIGGER_STOP: + break; + case COMP_TRIGGER_PREPARE: + if (sizeof(flat->cmd) + sizeof(flat->ret) + sizeof(flat->pipeline_state) + + sizeof(void *) * (param->pipeline_state.n_sources + + param->pipeline_state.n_sinks) > + sizeof(ipc_buf)) + return -ENOMEM; + + flat->pipeline_state.state = param->pipeline_state.state; + flat->pipeline_state.n_sources = param->pipeline_state.n_sources; + flat->pipeline_state.n_sinks = param->pipeline_state.n_sinks; + memcpy(flat->pipeline_state.source_sink, param->pipeline_state.sources, + flat->pipeline_state.n_sources * + sizeof(flat->pipeline_state.source_sink[0])); + memcpy(flat->pipeline_state.source_sink + flat->pipeline_state.n_sources, + param->pipeline_state.sinks, + flat->pipeline_state.n_sinks * + sizeof(flat->pipeline_state.source_sink[0])); + } + } + + return 0; +} + +/* Unpack IPC data and execute a callback */ +static void ipc_thread_unflatten_run(struct processing_module *pmod, struct ipc4_flat *flat) +{ + const struct module_interface *const ops = pmod->dev->drv->adapter_ops; + + switch (flat->cmd) { + case SOF_IPC4_MOD_BIND: + if (ops->bind) { + struct bind_info bind_data = { + .ipc4_data = &flat->bind.bu, + .bind_type = flat->bind.type, + }; + + flat->ret = ops->bind(pmod, &bind_data); + } else { + flat->ret = 0; + } + break; + case SOF_IPC4_MOD_DELETE_INSTANCE: + flat->ret = ops->free(pmod); + break; + case SOF_IPC4_MOD_INIT_INSTANCE: + flat->ret = ops->init(pmod); + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: + switch (flat->pipeline_state.trigger_cmd) { + case COMP_TRIGGER_STOP: + flat->ret = ops->reset(pmod); + break; + case COMP_TRIGGER_PREPARE: + flat->ret = ops->prepare(pmod, + (struct sof_source **)flat->pipeline_state.source_sink, + flat->pipeline_state.n_sources, + (struct sof_sink **)(flat->pipeline_state.source_sink + + flat->pipeline_state.n_sources), + flat->pipeline_state.n_sinks); + } + } +} + +/* Signal an IPC and wait for processing completion */ +int scheduler_dp_thread_ipc(struct processing_module *pmod, enum sof_ipc4_module_type cmd, + union scheduler_dp_thread_ipc_param *param) +{ + struct task_dp_pdata *pdata = pmod->dev->task->priv_data; + int ret; + + if (!pmod) { + tr_err(&dp_tr, "no thread module"); + return -EINVAL; + } + + if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { + /* Wait for the DP thread to start */ + ret = k_sem_take(&dp_thread_sync_sem, K_MSEC(100)); + if (ret < 0) + return ret; + } + + k_spinlock_key_t key = k_spin_lock(&thread_lock); + + struct ipc4_flat *flat = (struct ipc4_flat *)ipc_buf; + + /* IPCs are serialised */ + flat->ret = -ENOSYS; + + ret = ipc_thread_flatten(cmd, param, flat); + if (!ret) { + pdata->pend_ipc++; + k_sem_give(&dp_thread_ipc_sem); + } + + k_spin_unlock(&thread_lock, key); + + if (!ret) { + /* Wait for completion */ + k_sem_take(&dp_thread_sync_sem, K_FOREVER); + ret = flat->ret; + } + + return ret; +} + +#include /* Thread function called in component context, on target core */ static void dp_thread_fn(void *p1, void *p2, void *p3) { struct task *task = p1; - (void)p2; - (void)p3; struct task_dp_pdata *task_pdata = task->priv_data; - unsigned int lock_key; + struct processing_module *pmod = task_pdata->mod; + bool first = true; + //unsigned int lock_key; enum task_state state; bool task_stop; + ARG_UNUSED(p2); + ARG_UNUSED(p3); + do { + if (first) { + /* + * The IPC thread is waiting for the thread to be + * started, it can proceed now. + */ + first = false; + k_sem_give(&dp_thread_sync_sem); + } + /* - * the thread is started immediately after creation, it will stop on semaphore - * Semaphore will be released once the task is ready to process + * The thread is started immediately after creation, it stops on + * semaphore. Semaphore is signaled once the task is ready to process */ - k_sem_take(&task_pdata->sem, K_FOREVER); + k_sem_take(&dp_thread_ipc_sem, K_FOREVER); - if (task->state == SOF_TASK_STATE_RUNNING) - state = task_run(task); - else - state = task->state; /* to avoid undefined variable warning */ + //k_spinlock_key_t key = k_spin_lock(&thread_lock); + unsigned char pend_ipc = task_pdata->pend_ipc, + pend_proc = task_pdata->pend_proc; - lock_key = scheduler_dp_lock(); - /* - * check if task is still running, may have been canceled by external call - * if not, set the state returned by run procedure - */ - if (task->state == SOF_TASK_STATE_RUNNING) { - task->state = state; - switch (state) { - case SOF_TASK_STATE_RESCHEDULE: - /* mark to reschedule, schedule time is already calculated */ - task->state = SOF_TASK_STATE_QUEUED; - break; + task_pdata->pend_proc = 0; + task_pdata->pend_ipc = 0; + + //k_spin_unlock(&thread_lock, key); - case SOF_TASK_STATE_CANCEL: - case SOF_TASK_STATE_COMPLETED: - /* remove from scheduling */ - list_item_del(&task->list); - break; + /* Only 0:1, 1:0 and 1:1 are valid */ + if ((unsigned int)(pend_ipc + pend_proc - 1) > 1) { + tr_err(&dp_tr, "Invalid wake up %u:%u", pend_proc, pend_ipc); + continue; + } - default: - /* illegal state, serious defect, won't happen */ - k_panic(); + if (pend_ipc) { + /* handle IPC */ + tr_dbg(&dp_tr, "got IPC wake up for %p state %d", pmod, task->state); + ipc_thread_unflatten_run(pmod, (struct ipc4_flat *)ipc_buf); + k_sem_give(&dp_thread_sync_sem); + } + + if (pend_proc) { + /* handle audio */ + bool ready = module_is_ready_to_process(pmod, + pmod->sources, pmod->num_of_sources, + pmod->sinks, pmod->num_of_sinks); + + if (ready) { + if (task->state == SOF_TASK_STATE_RUNNING) + state = task_run(task); + else + state = task->state; /* to avoid undefined variable warning */ + } + + //lock_key = scheduler_dp_lock(); + + /* + * check if task is still running, may have been canceled by external call + * if not, set the state returned by run procedure + */ + if (ready && task->state == SOF_TASK_STATE_RUNNING) { + task->state = state; + switch (state) { + case SOF_TASK_STATE_RESCHEDULE: + /* mark to reschedule, schedule time is already calculated */ + task->state = SOF_TASK_STATE_QUEUED; + break; + + case SOF_TASK_STATE_CANCEL: + case SOF_TASK_STATE_COMPLETED: + /* remove from scheduling */ + list_item_del(&task->list); + break; + + default: + /* illegal state, serious defect, won't happen */ + k_panic(); + } + } else { + task->state = SOF_TASK_STATE_QUEUED; } + //} else { + // lock_key = scheduler_dp_lock(); } /* if true exit the while loop, terminate the thread */ task_stop = task->state == SOF_TASK_STATE_COMPLETED || task->state == SOF_TASK_STATE_CANCEL; - - scheduler_dp_unlock(lock_key); + //scheduler_dp_unlock(lock_key); } while (!task_stop); /* call task_complete */ @@ -380,7 +595,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta struct task_dp_pdata *pdata = task->priv_data; unsigned int lock_key; uint64_t deadline_clock_ticks; - int ret; lock_key = scheduler_dp_lock(); @@ -391,22 +605,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return -EINVAL; } - /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(&pdata->thread, (__sparse_force void *)pdata->p_stack, - pdata->stack_size, dp_thread_fn, task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, K_USER, K_FOREVER); - - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(pdata->thread_id, task->core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto err; - } - - /* start the thread, it should immediately stop at a semaphore, so clean it */ - k_sem_reset(&pdata->sem); - k_thread_start(pdata->thread_id); - /* if there's no DP tasks scheduled yet, run ll tick source task */ if (list_is_empty(&dp_sch->tasks)) schedule_task(&dp_sch->ll_tick_src, 0, 0); @@ -429,11 +627,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta tr_dbg(&dp_tr, "DP task scheduled with period %u [us]", (uint32_t)period); return 0; -err: - /* cleanup - unlock and free all allocated resources */ - scheduler_dp_unlock(lock_key); - k_thread_abort(pdata->thread_id); - return ret; } static struct scheduler_ops schedule_dp_ops = { @@ -450,6 +643,8 @@ int scheduler_dp_init(void) if (!dp_sch) return -ENOMEM; + k_spinlock_init(&thread_lock); + list_init(&dp_sch->tasks); scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); @@ -469,6 +664,8 @@ int scheduler_dp_init(void) return 0; } +extern sys_slist_t xtensa_domain_list; + int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, const struct task_ops *ops, @@ -476,12 +673,14 @@ int scheduler_dp_task_init(struct task **task, uint16_t core, size_t stack_size) { - void __sparse_cache *p_stack = NULL; + k_thread_stack_t *p_stack; /* memory allocation helper structure */ struct { struct task task; struct task_dp_pdata pdata; + struct comp_driver drv; + struct module_interface ops; } *task_memory; int ret; @@ -496,53 +695,128 @@ int scheduler_dp_task_init(struct task **task, * As the structure contains zephyr kernel specific data, it must be located in * shared, non cached memory */ - task_memory = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory)); + task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_COHERENT, sizeof(*task_memory), 0); if (!task_memory) { tr_err(&dp_tr, "memory alloc failed"); return -ENOMEM; } + memset(task_memory, 0, sizeof(*task_memory)); + + task_memory->drv = *mod->dev->drv; + task_memory->ops = *mod->dev->drv->adapter_ops; + task_memory->drv.adapter_ops = &task_memory->ops; + mod->dev->drv = &task_memory->drv; + /* allocate stack - must be aligned and cached so a separate alloc */ stack_size = Z_KERNEL_STACK_SIZE_ADJUST(stack_size); - p_stack = (__sparse_force void __sparse_cache *) - rballoc_align(SOF_MEM_FLAG_KERNEL, stack_size, Z_KERNEL_STACK_OBJ_ALIGN); + p_stack = k_thread_stack_alloc(stack_size, K_USER); if (!p_stack) { tr_err(&dp_tr, "stack alloc failed"); ret = -ENOMEM; goto err; } + struct task *ptask = &task_memory->task; + /* internal SOF task init */ - ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, - mod, core, 0); + ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, 0); if (ret < 0) { tr_err(&dp_tr, "schedule_task_init failed"); goto err; } + struct task_dp_pdata *pdata = &task_memory->pdata; + /* initialize other task structures */ - task_memory->task.ops.complete = ops->complete; - task_memory->task.ops.get_deadline = ops->get_deadline; - task_memory->task.state = SOF_TASK_STATE_INIT; - task_memory->task.core = core; + ptask->ops.complete = ops->complete; + ptask->ops.get_deadline = ops->get_deadline; + ptask->priv_data = pdata; + pdata->p_stack = p_stack; + pdata->mod = mod; + + *task = ptask; + + /* create a zephyr thread for the task */ + pdata->thread = k_object_alloc(K_OBJ_THREAD); + if (!pdata->thread) + goto err; + memset(&pdata->thread->arch, 0, sizeof(pdata->thread->arch)); + pdata->thread_id = k_thread_create(pdata->thread, p_stack, stack_size, dp_thread_fn, + ptask, NULL, NULL, CONFIG_DP_THREAD_PRIORITY, + K_USER, K_FOREVER); + + k_thread_access_grant(pdata->thread, &dp_thread_sync_sem, &dp_thread_ipc_sem); + + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(pdata->thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } + + int pidx; + size_t size; + uintptr_t start; + struct k_mem_partition *ppart[SOF_DP_PART_TYPE_COUNT]; + + for (pidx = 0; pidx < ARRAY_SIZE(ppart); pidx++) + ppart[pidx] = pdata->mpart + pidx; + + mod_heap_info(mod, &size, &start); + pdata->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ + .start = start, + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + pdata->mpart[SOF_DP_PART_IPC] = (struct k_mem_partition){ + .start = (uintptr_t)&ipc_buf, + .size = sizeof(ipc_buf), + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + pdata->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ + .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, + .size = 4096, + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + /* Create a memory domain, add the thread and its stack and heap to it */ + if (dp_mdom.arch.ptables) { + bool found = sys_slist_find_and_remove(&xtensa_domain_list, &dp_mdom.arch.node); + + tr_dbg(&dp_tr, "found %u domain %p", found, &dp_mdom); + memset(&dp_mdom, 0, sizeof(dp_mdom)); + } + + ret = k_mem_domain_init(&dp_mdom, SOF_DP_PART_TYPE_COUNT, ppart); + if (ret < 0) { + tr_err(&dp_tr, "failed to init mem domain %d", ret); + goto e_thread; + } - /* initialize semaprhore */ - k_sem_init(&task_memory->pdata.sem, 0, 1); + ret = llext_manager_add_domain(mod->dev->ipc_config.id, &dp_mdom); + if (ret < 0) { + tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); + goto e_thread; + } - /* success, fill the structures */ - task_memory->task.priv_data = &task_memory->pdata; - task_memory->pdata.p_stack = p_stack; - task_memory->pdata.stack_size = stack_size; - task_memory->pdata.mod = mod; - *task = &task_memory->task; + ret = k_mem_domain_add_thread(&dp_mdom, pdata->thread_id); + if (ret < 0) { + tr_err(&dp_tr, "failed to add thread to domain %d", ret); + goto e_thread; + } + k_thread_start(pdata->thread_id); return 0; + +e_thread: + k_thread_abort(pdata->thread_id); + k_object_free(pdata->thread); err: /* cleanup - free all allocated resources */ - rfree((__sparse_force void *)p_stack); - rfree(task_memory); + k_thread_stack_free(p_stack); + mod_free(mod, task_memory); return ret; } diff --git a/test/cmocka/src/audio/mixer/CMakeLists.txt b/test/cmocka/src/audio/mixer/CMakeLists.txt index 723cb091ff61..ea8cad0bd79e 100644 --- a/test/cmocka/src/audio/mixer/CMakeLists.txt +++ b/test/cmocka/src/audio/mixer/CMakeLists.txt @@ -25,6 +25,7 @@ cmocka_test(mixer ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-schedule.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-stream.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-xrun.c + ${PROJECT_SOURCE_DIR}/src/audio/data_blob.c ${PROJECT_SOURCE_DIR}/src/module/audio/source_api.c ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/math/numbers.c diff --git a/west.yml b/west.yml index c428ba32bb74..1b84da43bbcd 100644 --- a/west.yml +++ b/west.yml @@ -43,7 +43,7 @@ manifest: - name: zephyr repo-path: zephyr - revision: 8843fd9fe1800164fbb5221d95d1eba7d0699458 + revision: pull/97201/head remote: zephyrproject # Import some projects listed in zephyr/west.yml@revision diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index f0dd442af0c1..eda0071072ab 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -498,6 +498,9 @@ zephyr_library_sources_ifdef(CONFIG_SHELL sof_shell.c ) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/audio/module_adapter/module/generic.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) + zephyr_library_link_libraries(SOF) target_link_libraries(SOF INTERFACE zephyr_interface) diff --git a/zephyr/edf_schedule.c b/zephyr/edf_schedule.c index 8aca007137e7..cc224be4bb05 100644 --- a/zephyr/edf_schedule.c +++ b/zephyr/edf_schedule.c @@ -5,6 +5,7 @@ // Author: Bartosz Kokoszko #include +#include #include #include #include @@ -111,6 +112,7 @@ int scheduler_init_edf(void) k_thread_suspend(thread); + k_thread_heap_assign(thread, sof_sys_heap_get()); k_thread_cpu_mask_clear(thread); k_thread_cpu_mask_enable(thread, PLATFORM_PRIMARY_CORE_ID); k_thread_name_set(thread, "edf_workq"); diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index 600852418693..b92fba31e5ba 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -106,6 +106,11 @@ void rfree(void *ptr); */ void l3_heap_save(void); +void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, + size_t alignment); +void sof_heap_free(struct k_heap *heap, void *addr); +struct k_heap *sof_sys_heap_get(void); + /* TODO: remove - debug only - only needed for linking */ static inline void heap_trace_all(int force) {} diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index baf93ed159f1..7bc3b2a81de2 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -316,6 +316,11 @@ SYS_INIT(virtual_heap_init, POST_KERNEL, 1); #endif /* CONFIG_VIRTUAL_HEAP */ +struct k_heap *sof_sys_heap_get(void) +{ + return &sof_heap; +} + static void *heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t bytes) { k_spinlock_key_t key; @@ -353,7 +358,7 @@ static void __sparse_cache *heap_alloc_aligned_cached(struct k_heap *h, */ #ifdef CONFIG_SOF_ZEPHYR_HEAP_CACHED min_align = MAX(PLATFORM_DCACHE_ALIGN, min_align); - bytes = ALIGN_UP(bytes, min_align); + bytes = ALIGN_UP(bytes, PLATFORM_DCACHE_ALIGN); #endif ptr = (__sparse_force void __sparse_cache *)heap_alloc_aligned(h, min_align, bytes); @@ -533,6 +538,30 @@ void rfree(void *ptr) } EXPORT_SYMBOL(rfree); +/* + * To match the fall-back SOF main heap all private heaps should also be in the + * uncached address range. + */ +void *sof_heap_alloc(struct k_heap *heap, uint32_t flags, size_t bytes, + size_t alignment) +{ + if (!heap) + heap = &sof_heap; + + if (flags & SOF_MEM_FLAG_COHERENT) + return heap_alloc_aligned(heap, alignment, bytes); + + return (__sparse_force void *)heap_alloc_aligned_cached(heap, alignment, bytes); +} + +void sof_heap_free(struct k_heap *heap, void *addr) +{ + if (!heap) + heap = &sof_heap; + + heap_free(heap, addr); +} + static int heap_init(void) { sys_heap_init(&sof_heap.heap, heapmem, HEAPMEM_SIZE); diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index 3be9475ff90c..3c100d27920d 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -80,7 +80,7 @@ static struct sof_fast_get_entry *fast_get_find_entry(struct sof_fast_get_data * return NULL; } -const void *fast_get(const void *dram_ptr, size_t size) +const void *z_impl_fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) { struct sof_fast_get_data *data = &fast_get_data; struct sof_fast_get_entry *entry; @@ -116,7 +116,7 @@ const void *fast_get(const void *dram_ptr, size_t size) goto out; } - ret = rmalloc(SOF_MEM_FLAG_USER, size); + ret = sof_heap_alloc(heap, SOF_MEM_FLAG_USER, size, 0); if (!ret) goto out; entry->size = size; @@ -131,7 +131,7 @@ const void *fast_get(const void *dram_ptr, size_t size) return ret; } -EXPORT_SYMBOL(fast_get); +EXPORT_SYMBOL(z_impl_fast_get); static struct sof_fast_get_entry *fast_put_find_entry(struct sof_fast_get_data *data, const void *sram_ptr) @@ -146,7 +146,7 @@ static struct sof_fast_get_entry *fast_put_find_entry(struct sof_fast_get_data * return NULL; } -void fast_put(const void *sram_ptr) +void z_impl_fast_put(struct k_heap *heap, const void *sram_ptr) { struct sof_fast_get_data *data = &fast_get_data; struct sof_fast_get_entry *entry; @@ -160,7 +160,7 @@ void fast_put(const void *sram_ptr) } entry->refcount--; if (!entry->refcount) { - rfree(entry->sram_ptr); + sof_heap_free(heap, entry->sram_ptr); memset(entry, 0, sizeof(*entry)); } out: @@ -168,4 +168,16 @@ void fast_put(const void *sram_ptr) entry ? entry->size : 0, entry ? entry->refcount : 0); k_spin_unlock(&data->lock, key); } -EXPORT_SYMBOL(fast_put); +EXPORT_SYMBOL(z_impl_fast_put); + +#ifdef CONFIG_USERSPACE +void z_vrfy_fast_put(struct k_heap *heap, const void *sram_ptr) +{ + z_impl_fast_put(heap, sram_ptr); +} + +const void *z_vrfy_fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) +{ + return z_impl_fast_get(heap, dram_ptr, size); +} +#endif diff --git a/zephyr/sof_shell.c b/zephyr/sof_shell.c index 60785c4eb5c4..f10a2c9275b5 100644 --- a/zephyr/sof_shell.c +++ b/zephyr/sof_shell.c @@ -55,13 +55,15 @@ static int cmd_sof_module_heap_usage(const struct shell *sh, } list_for_item_safe(clist, _clist, &ipc->comp_list) { + size_t usage, hwm; + icd = container_of(clist, struct ipc_comp_dev, list); if (icd->type != COMP_TYPE_COMPONENT) continue; - shell_print(sh, "comp id 0x%08x\t%8zu bytes\t(%zu max)", icd->id, - module_adapter_heap_usage(comp_mod(icd->cd)), - comp_mod(icd->cd)->priv.cfg.heap_bytes); + usage = module_adapter_heap_usage(comp_mod(icd->cd), &hwm); + shell_print(sh, "comp id 0x%08x%9zu usage%9zu hwm %9zu max\tbytes", + icd->id, usage, hwm, comp_mod(icd->cd)->priv.cfg.heap_bytes); } return 0; }