diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index fa809d91f554..85f99bb99e2b 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -33,6 +33,7 @@ CONFIG_DAI_INTEL_DMIC_NHLT=y CONFIG_DAI_DMIC_HAS_OWNERSHIP=y CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DAI_INTEL_SSP=y +CONFIG_PIPELINE_2_0=y CONFIG_ZEPHYR_DP_SCHEDULER=y CONFIG_DMA=y CONFIG_DMA_INTEL_ADSP_GPDMA=y diff --git a/app/boards/intel_adsp_ace20_lnl.conf b/app/boards/intel_adsp_ace20_lnl.conf index 2f02c150b828..be7f1b4f9cff 100644 --- a/app/boards/intel_adsp_ace20_lnl.conf +++ b/app/boards/intel_adsp_ace20_lnl.conf @@ -29,6 +29,7 @@ CONFIG_DAI_INTEL_DMIC_NHLT=y CONFIG_DAI_DMIC_HAS_OWNERSHIP=n CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DAI_INTEL_SSP=y +CONFIG_PIPELINE_2_0=y CONFIG_ZEPHYR_DP_SCHEDULER=y CONFIG_DMA=y CONFIG_DMA_INTEL_ADSP_GPDMA=n diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index cac5d9d2118a..050ff987369b 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -28,6 +28,7 @@ CONFIG_DAI_INTEL_DMIC_NHLT=y CONFIG_DAI_DMIC_HAS_OWNERSHIP=n CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DAI_INTEL_SSP=y +CONFIG_PIPELINE_2_0=y CONFIG_ZEPHYR_DP_SCHEDULER=y CONFIG_DMA=y CONFIG_DMA_INTEL_ADSP_GPDMA=n diff --git a/src/audio/audio_stream.c b/src/audio/audio_stream.c index a4bbecc83bd0..855a67ae3b04 100644 --- a/src/audio/audio_stream.c +++ b/src/audio/audio_stream.c @@ -6,8 +6,8 @@ #include #include +#include #include -#include static size_t audio_stream_get_free_size(struct sof_sink *sink) { @@ -207,22 +207,22 @@ void audio_stream_init(struct audio_stream *audio_stream, void *buff_addr, uint3 } /* get a handler to source API */ -#if CONFIG_ZEPHYR_DP_SCHEDULER +#if CONFIG_PIPELINE_2_0 struct sof_source *audio_stream_get_source(struct audio_stream *audio_stream) { - return audio_stream->dp_queue_source ? - dp_queue_get_source(audio_stream->dp_queue_source) : + return audio_stream->secondary_buffer_source ? + audio_buffer_get_source(audio_stream->secondary_buffer_source) : &audio_stream->_source_api; } struct sof_sink *audio_stream_get_sink(struct audio_stream *audio_stream) { - return audio_stream->dp_queue_sink ? - dp_queue_get_sink(audio_stream->dp_queue_sink) : + return audio_stream->secondary_buffer_sink ? + audio_buffer_get_sink(audio_stream->secondary_buffer_sink) : &audio_stream->_sink_api; } -#else /* CONFIG_ZEPHYR_DP_SCHEDULER */ +#else /* CONFIG_PIPELINE_2_0 */ struct sof_source *audio_stream_get_source(struct audio_stream *audio_stream) { @@ -233,4 +233,4 @@ struct sof_sink *audio_stream_get_sink(struct audio_stream *audio_stream) { return &audio_stream->_sink_api; } -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ +#endif /* CONFIG_PIPELINE_2_0 */ diff --git a/src/audio/buffer.c b/src/audio/buffer.c index 05affe47a32d..1e2d4b5c1eb5 100644 --- a/src/audio/buffer.c +++ b/src/audio/buffer.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include @@ -95,55 +95,48 @@ struct comp_buffer *buffer_alloc(size_t size, uint32_t caps, uint32_t flags, uin return buffer; } -#if CONFIG_ZEPHYR_DP_SCHEDULER -int buffer_create_shadow_dp_queue(struct comp_buffer *buffer, bool at_input) +#if CONFIG_PIPELINE_2_0 +int buffer_attach_secondary_buffer(struct comp_buffer *buffer, bool at_input, + struct sof_audio_buffer *secondary_buffer) { - if (buffer->stream.dp_queue_sink || buffer->stream.dp_queue_source) { - buf_err(buffer, "Only one shadow dp_queue may be attached to a buffer"); + if (buffer->stream.secondary_buffer_sink || buffer->stream.secondary_buffer_source) { + buf_err(buffer, "Only one secondary buffer may be attached to a buffer"); return -EINVAL; } - - struct dp_queue *dp_queue = - dp_queue_create(source_get_min_available(&buffer->stream._source_api), - sink_get_min_free_space(&buffer->stream._sink_api), - buffer->is_shared ? DP_QUEUE_MODE_SHARED : DP_QUEUE_MODE_LOCAL, - buf_get_id(buffer), &buffer->stream.runtime_stream_params); - - if (!dp_queue) - return -ENOMEM; - if (at_input) - buffer->stream.dp_queue_sink = dp_queue; + buffer->stream.secondary_buffer_sink = secondary_buffer; else - buffer->stream.dp_queue_source = dp_queue; + buffer->stream.secondary_buffer_source = secondary_buffer; - buf_info(buffer, "dp_queue attached to buffer as a shadow, at_input: %u", at_input); + buf_info(buffer, "ring_buffer attached to buffer as a secondary, at_input: %u", at_input); return 0; } -int buffer_sync_shadow_dp_queue(struct comp_buffer *buffer, size_t limit) +int buffer_sync_secondary_buffer(struct comp_buffer *buffer, size_t limit) { int err; struct sof_source *data_src; struct sof_sink *data_dst; - if (buffer->stream.dp_queue_sink) { + if (buffer->stream.secondary_buffer_sink) { /* - * comp_buffer sink API is shadowed, that means there's a dp_queue at data input - * get data from dp_queue_sink (use source API) + * comp_buffer sink API is shadowed, that means there's a secondary_buffer + * at data input + * get data from secondary_buffer (use source API) * copy to comp_buffer (use sink API) */ - data_src = dp_queue_get_source(buffer->stream.dp_queue_sink); + data_src = audio_buffer_get_source(buffer->stream.secondary_buffer_sink); data_dst = &buffer->stream._sink_api; - } else if (buffer->stream.dp_queue_source) { + } else if (buffer->stream.secondary_buffer_source) { /* - * comp_buffer source API is shadowed, that means there's a dp_queue at data output + * comp_buffer source API is shadowed, that means there's a secondary_buffer + * at data output * get data from comp_buffer (use source API) - * copy to dp_queue_source (use sink API) + * copy to secondary_buffer (use sink API) */ data_src = &buffer->stream._source_api; - data_dst = dp_queue_get_sink(buffer->stream.dp_queue_source); + data_dst = audio_buffer_get_sink(buffer->stream.secondary_buffer_source); } else { return -EINVAL; @@ -160,7 +153,7 @@ int buffer_sync_shadow_dp_queue(struct comp_buffer *buffer, size_t limit) err = source_to_sink_copy(data_src, data_dst, true, to_copy); return err; } -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ +#endif /* CONFIG_PIPELINE_2_0 */ struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_size, uint32_t caps, uint32_t flags, uint32_t align, bool is_shared) @@ -384,10 +377,10 @@ void buffer_free(struct comp_buffer *buffer) /* In case some listeners didn't unregister from buffer's callbacks */ notifier_unregister_all(NULL, buffer); -#if CONFIG_ZEPHYR_DP_SCHEDULER - dp_queue_free(buffer->stream.dp_queue_sink); - dp_queue_free(buffer->stream.dp_queue_source); -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ +#if CONFIG_PIPELINE_2_0 + audio_buffer_free(buffer->stream.secondary_buffer_sink); + audio_buffer_free(buffer->stream.secondary_buffer_source); +#endif /* CONFIG_PIPELINE_2_0 */ rfree(buffer->stream.addr); rfree(buffer); } diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c new file mode 100644 index 000000000000..9ce7eeee87c2 --- /dev/null +++ b/src/audio/buffers/ring_buffer.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2023 Intel Corporation. All rights reserved. +// + +#include +#include +#include + +#include + +#include +#include + +LOG_MODULE_REGISTER(ring_buffer, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(ring_buffer); +DECLARE_TR_CTX(dp_queue_tr, SOF_UUID(ring_buffer_uuid), LOG_LEVEL_INFO); + +static inline struct ring_buffer *ring_buffer_from_sink(struct sof_sink *sink) +{ + struct sof_audio_buffer *audio_buffer = sof_audo_buffer_from_sink(sink); + + return container_of(audio_buffer, struct ring_buffer, audio_buffer); +} + +static inline struct ring_buffer *ring_buffer_from_source(struct sof_source *source) +{ + struct sof_audio_buffer *audio_buffer = sof_audo_buffer_from_source(source); + + return container_of(audio_buffer, struct ring_buffer, audio_buffer); +} + +/** + * @brief remove the queue from the list, free memory + */ +static void ring_buffer_free(struct sof_audio_buffer *buffer) +{ + struct ring_buffer *ring_buffer = (struct ring_buffer *)buffer; + + rfree((__sparse_force void *)ring_buffer->_data_buffer); +} + +/** + * @brief return true if the queue is shared between 2 cores + */ +static inline +bool ring_buffer_is_shared(struct ring_buffer *ring_buffer) +{ + return !!(ring_buffer->_flags & RING_BUFFER_MODE_SHARED); +} + +static inline uint8_t __sparse_cache *ring_buffer_buffer_end(struct ring_buffer *ring_buffer) +{ + return ring_buffer->_data_buffer + ring_buffer->data_buffer_size; +} + +static inline void ring_buffer_invalidate_shared(struct ring_buffer *ring_buffer, + void __sparse_cache *ptr, size_t size) +{ + /* no cache required in case of not shared queue */ + if (!ring_buffer_is_shared(ring_buffer)) + return; + + /* wrap-around? */ + if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) { + /* writeback till the end of circular buffer */ + dcache_invalidate_region + (ptr, (uintptr_t)ring_buffer_buffer_end(ring_buffer) - (uintptr_t)ptr); + size -= (uintptr_t)ring_buffer_buffer_end(ring_buffer) - (uintptr_t)ptr; + ptr = ring_buffer->_data_buffer; + } + /* invalidate rest of data */ + dcache_invalidate_region(ptr, size); +} + +static inline void ring_buffer_writeback_shared(struct ring_buffer *ring_buffer, + void __sparse_cache *ptr, size_t size) +{ + /* no cache required in case of not shared queue */ + if (!ring_buffer_is_shared(ring_buffer)) + return; + + /* wrap-around? */ + if ((uintptr_t)ptr + size > (uintptr_t)ring_buffer_buffer_end(ring_buffer)) { + /* writeback till the end of circular buffer */ + dcache_writeback_region + (ptr, (uintptr_t)ring_buffer_buffer_end(ring_buffer) - (uintptr_t)ptr); + size -= (uintptr_t)ring_buffer_buffer_end(ring_buffer) - (uintptr_t)ptr; + ptr = ring_buffer->_data_buffer; + } + /* writeback rest of data */ + dcache_writeback_region(ptr, size); +} + +static inline +uint8_t __sparse_cache *ring_buffer_get_pointer(struct ring_buffer *ring_buffer, size_t offset) +{ + /* check if offset is not in "double area" + * lines below do a quicker version of offset %= ring_buffer->data_buffer_size; + */ + if (offset >= ring_buffer->data_buffer_size) + offset -= ring_buffer->data_buffer_size; + return ring_buffer->_data_buffer + offset; +} + +static inline +size_t ring_buffer_inc_offset(struct ring_buffer *ring_buffer, size_t offset, size_t inc) +{ + assert(inc <= ring_buffer->data_buffer_size); + offset += inc; + /* wrap around ? 2*size because of "double area" */ + if (offset >= 2 * ring_buffer->data_buffer_size) + offset -= 2 * ring_buffer->data_buffer_size; + return offset; +} + +static inline +size_t _ring_buffer_get_data_available(struct ring_buffer *ring_buffer) +{ + int32_t avail_data = ring_buffer->_write_offset - ring_buffer->_read_offset; + /* wrap around ? 2*size because of "double area" */ + if (avail_data < 0) + avail_data = 2 * ring_buffer->data_buffer_size + avail_data; + + return avail_data; +} + +static size_t ring_buffer_get_data_available(struct sof_source *source) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_source(source); + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + return _ring_buffer_get_data_available(ring_buffer); +} + +static size_t ring_buffer_get_free_size(struct sof_sink *sink) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_sink(sink); + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + return ring_buffer->data_buffer_size - _ring_buffer_get_data_available(ring_buffer); +} + +static int ring_buffer_get_buffer(struct sof_sink *sink, size_t req_size, + void **data_ptr, void **buffer_start, size_t *buffer_size) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_sink(sink); + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + if (req_size > ring_buffer_get_free_size(sink)) + return -ENODATA; + + /* note, __sparse_force is to be removed once sink/src use __sparse_cache for data ptrs */ + *data_ptr = (__sparse_force void *)ring_buffer_get_pointer(ring_buffer, + ring_buffer->_write_offset); + *buffer_start = (__sparse_force void *)ring_buffer->_data_buffer; + *buffer_size = ring_buffer->data_buffer_size; + + /* no need to invalidate cache - buffer is to be written only */ + return 0; +} + +static int ring_buffer_commit_buffer(struct sof_sink *sink, size_t commit_size) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_sink(sink); + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + if (commit_size) { + ring_buffer_writeback_shared(ring_buffer, + ring_buffer_get_pointer(ring_buffer, + ring_buffer->_write_offset), + commit_size); + + /* move write pointer */ + ring_buffer->_write_offset = ring_buffer_inc_offset(ring_buffer, + ring_buffer->_write_offset, + commit_size); + } + + return 0; +} + +static int ring_buffer_get_data(struct sof_source *source, size_t req_size, + void const **data_ptr, void const **buffer_start, + size_t *buffer_size) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_source(source); + __sparse_cache void *data_ptr_c; + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + if (req_size > ring_buffer_get_data_available(source)) + return -ENODATA; + + data_ptr_c = ring_buffer_get_pointer(ring_buffer, ring_buffer->_read_offset); + + /* clean cache in provided data range */ + ring_buffer_invalidate_shared(ring_buffer, data_ptr_c, req_size); + + *buffer_start = (__sparse_force void *)ring_buffer->_data_buffer; + *buffer_size = ring_buffer->data_buffer_size; + *data_ptr = (__sparse_force void *)data_ptr_c; + + return 0; +} + +static int ring_buffer_release_data(struct sof_source *source, size_t free_size) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_source(source); + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + if (free_size) { + /* data consumed, free buffer space, no need for any special cache operations */ + ring_buffer->_read_offset = ring_buffer_inc_offset(ring_buffer, + ring_buffer->_read_offset, + free_size); + } + + return 0; +} + +static int ring_buffer_set_ipc_params(struct ring_buffer *ring_buffer, + struct sof_ipc_stream_params *params, + bool force_update) +{ + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + if (ring_buffer->_hw_params_configured && !force_update) + return 0; + + ring_buffer->audio_stream_params->frame_fmt = params->frame_fmt; + ring_buffer->audio_stream_params->rate = params->rate; + ring_buffer->audio_stream_params->channels = params->channels; + ring_buffer->audio_stream_params->buffer_fmt = params->buffer_fmt; + + ring_buffer->_hw_params_configured = true; + + return 0; +} + +static int ring_buffer_set_ipc_params_source(struct sof_source *source, + struct sof_ipc_stream_params *params, + bool force_update) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_source(source); + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + return ring_buffer_set_ipc_params(ring_buffer, params, force_update); +} + +static int ring_buffer_set_ipc_params_sink(struct sof_sink *sink, + struct sof_ipc_stream_params *params, + bool force_update) +{ + struct ring_buffer *ring_buffer = ring_buffer_from_sink(sink); + + CORE_CHECK_STRUCT(&ring_buffer->audio_buffer); + return ring_buffer_set_ipc_params(ring_buffer, params, force_update); +} + +static const struct source_ops ring_buffer_source_ops = { + .get_data_available = ring_buffer_get_data_available, + .get_data = ring_buffer_get_data, + .release_data = ring_buffer_release_data, + .audio_set_ipc_params = ring_buffer_set_ipc_params_source, +}; + +static const struct sink_ops ring_buffer_sink_ops = { + .get_free_size = ring_buffer_get_free_size, + .get_buffer = ring_buffer_get_buffer, + .commit_buffer = ring_buffer_commit_buffer, + .audio_set_ipc_params = ring_buffer_set_ipc_params_sink, +}; + +struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, uint32_t flags, + uint32_t id, + struct sof_audio_stream_params *audio_stream_params) +{ + struct ring_buffer *ring_buffer; + + /* allocate ring_buffer structure */ + if (flags & RING_BUFFER_MODE_SHARED) + ring_buffer = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, + sizeof(*ring_buffer)); + else + ring_buffer = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, + sizeof(*ring_buffer)); + if (!ring_buffer) + return NULL; + + ring_buffer->_flags = flags; + ring_buffer->audio_stream_params = audio_stream_params; + + CORE_CHECK_STRUCT_INIT(&ring_buffer->audio_buffer, flags & RING_BUFFER_MODE_SHARED); + + /* initiate structures */ + source_init(audio_buffer_get_source(&ring_buffer->audio_buffer), &ring_buffer_source_ops, + ring_buffer->audio_stream_params); + sink_init(audio_buffer_get_sink(&ring_buffer->audio_buffer), &ring_buffer_sink_ops, + ring_buffer->audio_stream_params); + + /* set obs/ibs in sink/source interfaces */ + sink_set_min_free_space(&ring_buffer->audio_buffer._sink_api, min_free_space); + source_set_min_available(&ring_buffer->audio_buffer._source_api, min_available); + + uint32_t max_ibs_obs = MAX(min_available, min_free_space); + + /* calculate required buffer size */ + ring_buffer->data_buffer_size = 2 * max_ibs_obs; + + /* allocate data buffer - always in cached memory alias */ + ring_buffer->data_buffer_size = + ALIGN_UP(ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); + ring_buffer->_data_buffer = (__sparse_force __sparse_cache void *) + rballoc_align(0, 0, ring_buffer->data_buffer_size, PLATFORM_DCACHE_ALIGN); + if (!ring_buffer->_data_buffer) + goto err; + + ring_buffer->audio_stream_params->id = id; + tr_info(&ring_buffer_tr, "Ring buffer created, id: %u shared: %u min_available: %u min_free_space %u, size %u", + id, ring_buffer_is_shared(ring_buffer), min_available, min_free_space, + ring_buffer->data_buffer_size); + + /* set common buffer api */ + ring_buffer->audio_buffer.free = ring_buffer_free; + ring_buffer->audio_buffer.buffer_type = BUFFER_TYPE_RING_BUFFER; + + /* return a pointer to allocated structure */ + return ring_buffer; +err: + tr_err(&ring_buffer_tr, "Ring buffer creation failure"); + rfree(ring_buffer); + return NULL; +} diff --git a/src/audio/component.c b/src/audio/component.c index f6c0f7fd7921..04020ee98732 100644 --- a/src/audio/component.c +++ b/src/audio/component.c @@ -474,14 +474,14 @@ int comp_copy(struct comp_dev *dev) * * DP components (modules) require two stage processing: * - * LL_mod -> [comp_buffer -> dp_queue] -> dp_mod -> [dp_queue -> comp_buffer] -> LL_mod + * LL_mod -> [comp_buffer->ring_buffer] -> dp_mod -> [ring_buffer ->comp_buffer] -> LL_mod * * - in first step (it means - now) the pipeline must copy source data from comp_buffer - * to dp_queue and result data from dp_queue to comp_buffer + * to ring_buffer and result data from ring_buffer to comp_buffer * * - second step will be performed by a thread specific to the DP module - DP module - * will take data from input dpQueue (using source API) , process it - * and put in output DP queue (using sink API) + * will take data from input ring_buffer (using source API), process it + * and put in output ring_buffer (using sink API) * * this allows the current pipeline structure to see a DP module as a "normal" LL * diff --git a/src/audio/dp_queue.c b/src/audio/dp_queue.c deleted file mode 100644 index dfa77c729b4d..000000000000 --- a/src/audio/dp_queue.c +++ /dev/null @@ -1,298 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause -// -// Copyright(c) 2023 Intel Corporation. All rights reserved. -// - -#include -#include -#include - -#include - -#include -#include - -LOG_MODULE_REGISTER(dp_queue, CONFIG_SOF_LOG_LEVEL); - -SOF_DEFINE_REG_UUID(dp_queue); -DECLARE_TR_CTX(dp_queue_tr, SOF_UUID(dp_queue_uuid), LOG_LEVEL_INFO); - -static inline uint8_t __sparse_cache *dp_queue_buffer_end(struct dp_queue *dp_queue) -{ - return dp_queue->_data_buffer + dp_queue->data_buffer_size; -} - -static inline struct dp_queue *dp_queue_from_sink(struct sof_sink *sink) -{ - return container_of(sink, struct dp_queue, _sink_api); -} - -static inline struct dp_queue *dp_queue_from_source(struct sof_source *source) -{ - return container_of(source, struct dp_queue, _source_api); -} - -static inline void dp_queue_invalidate_shared(struct dp_queue *dp_queue, - void __sparse_cache *ptr, size_t size) -{ - /* no cache required in case of not shared queue */ - if (!dp_queue_is_shared(dp_queue)) - return; - - /* wrap-around? */ - if ((uintptr_t)ptr + size > (uintptr_t)dp_queue_buffer_end(dp_queue)) { - /* writeback till the end of circular buffer */ - dcache_invalidate_region - (ptr, (uintptr_t)dp_queue_buffer_end(dp_queue) - (uintptr_t)ptr); - size -= (uintptr_t)dp_queue_buffer_end(dp_queue) - (uintptr_t)ptr; - ptr = dp_queue->_data_buffer; - } - /* invalidate rest of data */ - dcache_invalidate_region(ptr, size); -} - -static inline void dp_queue_writeback_shared(struct dp_queue *dp_queue, - void __sparse_cache *ptr, size_t size) -{ - /* no cache required in case of not shared queue */ - if (!dp_queue_is_shared(dp_queue)) - return; - - /* wrap-around? */ - if ((uintptr_t)ptr + size > (uintptr_t)dp_queue_buffer_end(dp_queue)) { - /* writeback till the end of circular buffer */ - dcache_writeback_region - (ptr, (uintptr_t)dp_queue_buffer_end(dp_queue) - (uintptr_t)ptr); - size -= (uintptr_t)dp_queue_buffer_end(dp_queue) - (uintptr_t)ptr; - ptr = dp_queue->_data_buffer; - } - /* writeback rest of data */ - dcache_writeback_region(ptr, size); -} - -static inline -uint8_t __sparse_cache *dp_queue_get_pointer(struct dp_queue *dp_queue, size_t offset) -{ - /* check if offset is not in "double area" - * lines below do a quicker version of offset %= dp_queue->data_buffer_size; - */ - if (offset >= dp_queue->data_buffer_size) - offset -= dp_queue->data_buffer_size; - return dp_queue->_data_buffer + offset; -} - -static inline -size_t dp_queue_inc_offset(struct dp_queue *dp_queue, size_t offset, size_t inc) -{ - assert(inc <= dp_queue->data_buffer_size); - offset += inc; - /* wrap around ? 2*size because of "double area" */ - if (offset >= 2 * dp_queue->data_buffer_size) - offset -= 2 * dp_queue->data_buffer_size; - return offset; -} - -static inline -size_t _dp_queue_get_data_available(struct dp_queue *dp_queue) -{ - int32_t avail_data = dp_queue->_write_offset - dp_queue->_read_offset; - /* wrap around ? 2*size because of "double area" */ - if (avail_data < 0) - avail_data = 2 * dp_queue->data_buffer_size + avail_data; - - return avail_data; -} - -static size_t dp_queue_get_data_available(struct sof_source *source) -{ - struct dp_queue *dp_queue = dp_queue_from_source(source); - - CORE_CHECK_STRUCT(dp_queue); - return _dp_queue_get_data_available(dp_queue); -} - -static size_t dp_queue_get_free_size(struct sof_sink *sink) -{ - struct dp_queue *dp_queue = dp_queue_from_sink(sink); - - CORE_CHECK_STRUCT(dp_queue); - return dp_queue->data_buffer_size - _dp_queue_get_data_available(dp_queue); -} - -static int dp_queue_get_buffer(struct sof_sink *sink, size_t req_size, - void **data_ptr, void **buffer_start, size_t *buffer_size) -{ - struct dp_queue *dp_queue = dp_queue_from_sink(sink); - - CORE_CHECK_STRUCT(dp_queue); - if (req_size > dp_queue_get_free_size(sink)) - return -ENODATA; - - /* note, __sparse_force is to be removed once sink/src use __sparse_cache for data ptrs */ - *data_ptr = (__sparse_force void *)dp_queue_get_pointer(dp_queue, dp_queue->_write_offset); - *buffer_start = (__sparse_force void *)dp_queue->_data_buffer; - *buffer_size = dp_queue->data_buffer_size; - - /* no need to invalidate cache - buffer is to be written only */ - return 0; -} - -static int dp_queue_commit_buffer(struct sof_sink *sink, size_t commit_size) -{ - struct dp_queue *dp_queue = dp_queue_from_sink(sink); - - CORE_CHECK_STRUCT(dp_queue); - if (commit_size) { - dp_queue_writeback_shared(dp_queue, - dp_queue_get_pointer(dp_queue, dp_queue->_write_offset), - commit_size); - - /* move write pointer */ - dp_queue->_write_offset = - dp_queue_inc_offset(dp_queue, dp_queue->_write_offset, commit_size); - } - - return 0; -} - -static int dp_queue_get_data(struct sof_source *source, size_t req_size, - void const **data_ptr, void const **buffer_start, size_t *buffer_size) -{ - struct dp_queue *dp_queue = dp_queue_from_source(source); - __sparse_cache void *data_ptr_c; - - CORE_CHECK_STRUCT(dp_queue); - if (req_size > dp_queue_get_data_available(source)) - return -ENODATA; - - data_ptr_c = dp_queue_get_pointer(dp_queue, dp_queue->_read_offset); - - /* clean cache in provided data range */ - dp_queue_invalidate_shared(dp_queue, data_ptr_c, req_size); - - *buffer_start = (__sparse_force void *)dp_queue->_data_buffer; - *buffer_size = dp_queue->data_buffer_size; - *data_ptr = (__sparse_force void *)data_ptr_c; - - return 0; -} - -static int dp_queue_release_data(struct sof_source *source, size_t free_size) -{ - struct dp_queue *dp_queue = dp_queue_from_source(source); - - CORE_CHECK_STRUCT(dp_queue); - if (free_size) { - /* data consumed, free buffer space, no need for any special cache operations */ - dp_queue->_read_offset = - dp_queue_inc_offset(dp_queue, dp_queue->_read_offset, free_size); - } - - return 0; -} - -static int dp_queue_set_ipc_params(struct dp_queue *dp_queue, - struct sof_ipc_stream_params *params, - bool force_update) -{ - CORE_CHECK_STRUCT(dp_queue); - if (dp_queue->_hw_params_configured && !force_update) - return 0; - - dp_queue->audio_stream_params->frame_fmt = params->frame_fmt; - dp_queue->audio_stream_params->rate = params->rate; - dp_queue->audio_stream_params->channels = params->channels; - dp_queue->audio_stream_params->buffer_fmt = params->buffer_fmt; - - dp_queue->_hw_params_configured = true; - - return 0; -} - -static int dp_queue_set_ipc_params_source(struct sof_source *source, - struct sof_ipc_stream_params *params, - bool force_update) -{ - struct dp_queue *dp_queue = dp_queue_from_source(source); - - CORE_CHECK_STRUCT(dp_queue); - return dp_queue_set_ipc_params(dp_queue, params, force_update); -} - -static int dp_queue_set_ipc_params_sink(struct sof_sink *sink, - struct sof_ipc_stream_params *params, - bool force_update) -{ - struct dp_queue *dp_queue = dp_queue_from_sink(sink); - - CORE_CHECK_STRUCT(dp_queue); - return dp_queue_set_ipc_params(dp_queue, params, force_update); -} - -static const struct source_ops dp_queue_source_ops = { - .get_data_available = dp_queue_get_data_available, - .get_data = dp_queue_get_data, - .release_data = dp_queue_release_data, - .audio_set_ipc_params = dp_queue_set_ipc_params_source, -}; - -static const struct sink_ops dp_queue_sink_ops = { - .get_free_size = dp_queue_get_free_size, - .get_buffer = dp_queue_get_buffer, - .commit_buffer = dp_queue_commit_buffer, - .audio_set_ipc_params = dp_queue_set_ipc_params_sink, -}; - -struct dp_queue *dp_queue_create(size_t min_available, size_t min_free_space, uint32_t flags, - uint32_t id, struct sof_audio_stream_params *audio_stream_params) -{ - struct dp_queue *dp_queue; - - /* allocate DP structure */ - if (flags & DP_QUEUE_MODE_SHARED) - dp_queue = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM, - sizeof(*dp_queue)); - else - dp_queue = rzalloc(SOF_MEM_ZONE_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*dp_queue)); - if (!dp_queue) - return NULL; - - dp_queue->_flags = flags; - dp_queue->audio_stream_params = audio_stream_params; - - CORE_CHECK_STRUCT_INIT(dp_queue, flags & DP_QUEUE_MODE_SHARED); - - /* initiate structures */ - source_init(dp_queue_get_source(dp_queue), &dp_queue_source_ops, - dp_queue->audio_stream_params); - sink_init(dp_queue_get_sink(dp_queue), &dp_queue_sink_ops, - dp_queue->audio_stream_params); - - /* set obs/ibs in sink/source interfaces */ - sink_set_min_free_space(&dp_queue->_sink_api, min_free_space); - source_set_min_available(&dp_queue->_source_api, min_available); - - uint32_t max_ibs_obs = MAX(min_available, min_free_space); - - /* calculate required buffer size */ - dp_queue->data_buffer_size = 2 * max_ibs_obs; - - /* allocate data buffer - always in cached memory alias */ - dp_queue->data_buffer_size = ALIGN_UP(dp_queue->data_buffer_size, PLATFORM_DCACHE_ALIGN); - dp_queue->_data_buffer = (__sparse_force __sparse_cache void *) - rballoc_align(0, 0, dp_queue->data_buffer_size, PLATFORM_DCACHE_ALIGN); - if (!dp_queue->_data_buffer) - goto err; - - dp_queue->audio_stream_params->id = id; - tr_info(&dp_queue_tr, "DpQueue created, id: %u shared: %u min_available: %u min_free_space %u, size %u", - id, dp_queue_is_shared(dp_queue), min_available, min_free_space, - dp_queue->data_buffer_size); - - /* return a pointer to allocated structure */ - return dp_queue; -err: - tr_err(&dp_queue_tr, "DpQueue creation failure"); - rfree(dp_queue); - return NULL; -} diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 8c1e825451c6..149bc04400b2 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -902,11 +902,11 @@ static int module_adapter_audio_stream_type_copy(struct comp_dev *dev) return ret; } -#if CONFIG_ZEPHYR_DP_SCHEDULER -static int module_adapter_copy_dp_queues(struct comp_dev *dev) +#if CONFIG_PIPELINE_2_0 +static int module_adapter_copy_ring_buffers(struct comp_dev *dev) { /* - * copy data from component audio streams to dp_queue + * copy data from component audio streams to ring_buffer * DP module processing itself will take place in DP thread * This is an adapter, to be removed when pipeline2.0 is ready */ @@ -916,11 +916,11 @@ static int module_adapter_copy_dp_queues(struct comp_dev *dev) list_for_item(blist, &dev->bsource_list) { /* input - we need to copy data from audio_stream (as source) - * to dp_queue (as sink) + * to ring_buffer (as sink) */ struct comp_buffer *buffer = container_of(blist, struct comp_buffer, sink_list); - err = buffer_sync_shadow_dp_queue(buffer, UINT_MAX); + err = buffer_sync_secondary_buffer(buffer, UINT_MAX); if (err) { comp_err(dev, "LL to DP copy error status: %d", err); @@ -932,7 +932,7 @@ static int module_adapter_copy_dp_queues(struct comp_dev *dev) return 0; list_for_item(blist, &dev->bsink_list) { - /* output - we need to copy data from dp_queue (as source) + /* output - we need to copy data from ring_buffer (as source) * to audio_stream (as sink) * * a trick is needed there: @@ -949,7 +949,7 @@ static int module_adapter_copy_dp_queues(struct comp_dev *dev) struct sof_source *following_mod_data_source = audio_stream_get_source(&buffer->stream); - err = buffer_sync_shadow_dp_queue + err = buffer_sync_secondary_buffer (buffer, source_get_min_available(following_mod_data_source)); @@ -960,12 +960,12 @@ static int module_adapter_copy_dp_queues(struct comp_dev *dev) } return 0; } -#else -static inline int module_adapter_copy_dp_queues(struct comp_dev *dev) +#else /* CONFIG_PIPELINE_2_0 */ +static inline int module_adapter_copy_ring_buffers(struct comp_dev *dev) { return -ENOTSUP; } -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ +#endif /* CONFIG_PIPELINE_2_0 */ static int module_adapter_sink_source_copy(struct comp_dev *dev) { @@ -1110,7 +1110,7 @@ int module_adapter_copy(struct comp_dev *dev) if (IS_PROCESSING_MODE_SINK_SOURCE(mod)) { if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) - return module_adapter_copy_dp_queues(dev); + return module_adapter_copy_ring_buffers(dev); else return module_adapter_sink_source_copy(dev); diff --git a/src/include/sof/audio/audio_buffer.h b/src/include/sof/audio/audio_buffer.h new file mode 100644 index 000000000000..356153924f84 --- /dev/null +++ b/src/include/sof/audio/audio_buffer.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + */ +#ifndef __SOF_AUDIO_BUFFER__ +#define __SOF_AUDIO_BUFFER__ + +#include +#include +#include + +#define BUFFER_TYPE_LEGACY_BUFFER 1 +#define BUFFER_TYPE_LEGACY_RING_HYBRID 2 +#define BUFFER_TYPE_RING_BUFFER 3 + +/* base class for all buffers, all buffers must inherit from it */ +struct sof_audio_buffer { + CORE_CHECK_STRUCT_FIELD; + + /* type of the buffer BUFFER_TYPE_* */ + uint32_t buffer_type; + + /* runtime stream params */ + struct sof_audio_stream_params audio_stream_params; + + /* private: */ + struct sof_source _source_api; /**< src api handler */ + struct sof_sink _sink_api; /**< sink api handler */ + + /* virtual methods */ + /** + * @brief this method must free all structures allocated by buffer implementation + * it must not free the buffer memory itself + */ + void (*free)(struct sof_audio_buffer *buffer); +}; + +/** + * @brief return a handler to sink API of audio_buffer. + * the handler may be used by helper functions defined in sink_api.h + */ +static inline +struct sof_sink *audio_buffer_get_sink(struct sof_audio_buffer *buffer) +{ + CORE_CHECK_STRUCT(buffer); + return &buffer->_sink_api; +} + +/** + * @brief return a handler to source API of audio_buffer + * the handler may be used by helper functions defined in source_api.h + */ +static inline +struct sof_source *audio_buffer_get_source(struct sof_audio_buffer *buffer) +{ + CORE_CHECK_STRUCT(buffer); + return &buffer->_source_api; +} + +/** + * @brief return a pointer to struct sof_audio_buffer from sink pointer + * NOTE! ensure that sink is really provided by sof_audio_buffer + * otherwise a random value will be returned + */ +static inline struct sof_audio_buffer *sof_audo_buffer_from_sink(struct sof_sink *sink) +{ + return container_of(sink, struct sof_audio_buffer, _sink_api); +} + +/** + * @brief return a pointer to struct sof_audio_buffer from source pointer + * NOTE! ensure that source is really provided by sof_audio_buffer + * otherwise a random value will be returned + */ +static inline struct sof_audio_buffer *sof_audo_buffer_from_source(struct sof_source *source) +{ + return container_of(source, struct sof_audio_buffer, _source_api); +} + +/** + * @brief free buffer and all allocated resources + */ +static inline +void audio_buffer_free(struct sof_audio_buffer *buffer) +{ + if (!buffer) + return; + + CORE_CHECK_STRUCT(buffer); + if (buffer->free) + buffer->free(buffer); + rfree(buffer); +} + +#endif /* __SOF_AUDIO_BUFFER__ */ diff --git a/src/include/sof/audio/audio_stream.h b/src/include/sof/audio/audio_stream.h index df9d74092b80..0f73e6f3c84a 100644 --- a/src/include/sof/audio/audio_stream.h +++ b/src/include/sof/audio/audio_stream.h @@ -63,10 +63,19 @@ struct audio_stream { void *end_addr; /**< Buffer end address */ uint8_t byte_align_req; uint8_t frame_align_req; -#if CONFIG_ZEPHYR_DP_SCHEDULER - struct dp_queue *dp_queue_sink; /**< sink API shadow, an additional dp_queue at data in */ - struct dp_queue *dp_queue_source; /**< source API shadow, an additional dp_queue at out */ -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ +#if CONFIG_PIPELINE_2_0 + /** + * sink API of an additional buffer + * of any type at data input + */ + struct sof_audio_buffer *secondary_buffer_sink; + + /** + * source API of an additional buffer + * at data output + */ + struct sof_audio_buffer *secondary_buffer_source; +#endif /* CONFIG_PIPELINE_2_0 */ /* runtime stream params */ struct sof_audio_stream_params runtime_stream_params; diff --git a/src/include/sof/audio/buffer.h b/src/include/sof/audio/buffer.h index 669e701aef58..db758f3a8ca3 100644 --- a/src/include/sof/audio/buffer.h +++ b/src/include/sof/audio/buffer.h @@ -9,6 +9,7 @@ #define __SOF_AUDIO_BUFFER_H__ #include +#include #include #include #include @@ -189,40 +190,45 @@ struct comp_buffer *buffer_alloc(size_t size, uint32_t caps, uint32_t flags, uin struct comp_buffer *buffer_alloc_range(size_t preferred_size, size_t minimum_size, uint32_t caps, uint32_t flags, uint32_t align, bool is_shared); struct comp_buffer *buffer_new(const struct sof_ipc_buffer *desc, bool is_shared); -#if CONFIG_ZEPHYR_DP_SCHEDULER +#if CONFIG_PIPELINE_2_0 /* - * create a shadow dp_queue buffer before buffer (when at_input == true) or behind a buffer + * attach a secondary buffer (any type) before buffer (when at_input == true) or behind a buffer * * before buffer (at_input == true): - * DP mod ==> (sink_API) shadow dp_queue ==> comp_buffer (audio_stream or source API) ==> LL mod + * 2.0 mod ==> (sink_API) secondary buffer ==> + * ==> comp_buffer (audio_stream or source API) ==> 1.0 mod * * after buffer (at_input == false): - * LL mod ==> (audio_stream or sink API) ==> comp_buffer ==> shadow dp_queue (source API) == DP mod + * 1.0 mod ==> (audio_stream or sink API) ==> comp_buffer ==> + * ==> secondary buffer(source API) == 2.0 mod * - * If a shadow buffer is created, it replaces source or sink interface of audio_stream - * allowing the module connected to it using all properties of dp_queue (like - * lockless cross-core connection) keeping legacy interface to other modules + * If a secondary buffer is attached, it replaces source or sink interface of audio_stream + * allowing the module connected to it using all properties of secondary buffer (like + * lockless cross-core connection in case of ring_buffer etc.) keeping legacy interface + * to other modules * - * buffer_sync_shadow_dp_queue must be called every 1 ms to move data to/from - * shadow dp_queue to comp_buffer + * buffer_sync_secondary_buffer must be called every 1 ms to move data to/from + * secondary buffer to comp_buffer * * @param buffer pointer to a buffer - * @param at_input true indicates that a shadow buffer is located at data input, replacing + * @param at_input true indicates that a secondary buffer is located at data input, replacing * sink API of audio_stream - * false indicates that a shadow buffer is located at data output, replacing + * false indicates that a secondary buffer is located at data output, replacing * source API of audio_stream + * @param secondary_buffer pointer to a buffer to be attached */ -int buffer_create_shadow_dp_queue(struct comp_buffer *buffer, bool at_input); +int buffer_attach_secondary_buffer(struct comp_buffer *buffer, bool at_input, + struct sof_audio_buffer *secondary_buffer); /* - * move data from/to shadow buffer, must be called periodically as described above + * move data from/to secondary buffer, must be called periodically as described above * * @param buffer pointer to a buffer - * @param limit data copy limit. Indicates maximum amount of data that will be moved from/to shadow - * buffer in an operation + * @param limit data copy limit. Indicates maximum amount of data that will be moved from/to + * secondary buffer in an operation */ -int buffer_sync_shadow_dp_queue(struct comp_buffer *buffer, size_t limit); -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ +int buffer_sync_secondary_buffer(struct comp_buffer *buffer, size_t limit); +#endif /* CONFIG_PIPELINE_2_0 */ int buffer_set_size(struct comp_buffer *buffer, uint32_t size, uint32_t alignment); int buffer_set_size_range(struct comp_buffer *buffer, size_t preferred_size, size_t minimum_size, uint32_t alignment); diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 37212c08fcb5..2952d86c375d 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -18,7 +18,6 @@ #include #include #include -#include #include "module_interface.h" /* diff --git a/src/include/sof/audio/dp_queue.h b/src/include/sof/audio/ring_buffer.h similarity index 67% rename from src/include/sof/audio/dp_queue.h rename to src/include/sof/audio/ring_buffer.h index 4eaf298ddf17..d900eb6294ec 100644 --- a/src/include/sof/audio/dp_queue.h +++ b/src/include/sof/audio/ring_buffer.h @@ -4,20 +4,22 @@ * */ -#ifndef __SOF_DP_QUEUE_H__ -#define __SOF_DP_QUEUE_H__ +#ifndef __SOF_RING_BUFFER_H__ +#define __SOF_RING_BUFFER_H__ #include #include #include +#include #include #include #include #include /** - * DP queue is a lockless circular buffer - * providing safe consumer/producer cached operations cross cores + * ring_buffer is a lockless async circular buffer + * providing safe consumer/producer cached operations cross cores that may be write/read + * at any time * * prerequisites: * 1) incoming and outgoing data rate MUST be the same @@ -31,7 +33,7 @@ * - cycle0 buffer empty, producer starting processing, consumer must wait * - cycle3 produce 3 bytes (buf occupation = 3) * - cycle6 produce 3 bytes (buf occupation = 6), consumer becomes ready - * in DP thread will start now - asyn to LL cycles + * in consumer thread will start now - asyn to producer cycles * in this example assuming it consumes data in next cycle * - cycle7 consume 5 bytes, (buf occupation = 1) * - cycle9 produce 3 bytes (buf occupation = 4) @@ -59,15 +61,14 @@ * The queue may work in 2 modes * 1) local mode * in case both receiver and sender are located on the same core and cache coherency - * does not matter. dp_queue structure is located in cached memory - * In this case DP Queue is a simple ring buffer + * does not matter. ring_buffer structure is located in cached memory * * 2) shared mode * In this case we need to writeback cache when new data arrive and invalidate cache on - * secondary core. dp_queue structure is located in shared memory + * secondary core. ring_buffer structure is located in shared memory * * - * dpQueue is a lockless consumer/producer safe buffer. It is achieved by having only 2 shared + * ring_buffer is a lockless consumer/producer safe buffer. It is achieved by having only 2 shared * variables: * _write_offset - can be modified by data producer only * _read_offset - can be modified by data consumer only @@ -96,22 +97,22 @@ * always means "buffer full" */ -struct dp_queue; +struct ring_buffer; struct sof_audio_stream_params; -/* DP flags */ -#define DP_QUEUE_MODE_LOCAL 0 -#define DP_QUEUE_MODE_SHARED BIT(1) +/* buffer flags */ +#define RING_BUFFER_MODE_LOCAL 0 +#define RING_BUFFER_MODE_SHARED BIT(1) -/* the dpQueue structure */ -struct dp_queue { - CORE_CHECK_STRUCT_FIELD; +/* the ring_buffer structure */ +struct ring_buffer { + struct sof_audio_buffer audio_buffer; /* public: read only */ /* note! - * as dpQueue is currently used as a shadow for comp_buffer only for DP components, - * the audio_stream_params vector must be shared between comp_buffer and dp_queue + * as ring_buffer is currently used as a shadow for comp_buffer only for DP components, + * the audio_stream_params vector must be shared between comp_buffer and ring_buffer * the audio_stream_params pointer should point to the proper comp_buffer structure * * to be changed to the structure itself when pipeline2.0 is introduced @@ -119,11 +120,7 @@ struct dp_queue { struct sof_audio_stream_params *audio_stream_params; size_t data_buffer_size; - /* private: */ - struct sof_source _source_api; /**< src api handler */ - struct sof_sink _sink_api; /**< sink api handler */ - - uint32_t _flags; /* DP_QUEUE_MODE_* */ + uint32_t _flags; /* RING_BUFFER_MODE_* */ uint8_t __sparse_cache *_data_buffer; size_t _write_offset; /* private: to be modified by data producer using API */ @@ -135,64 +132,19 @@ struct dp_queue { /** * * @param min_available minimum data available in queue required by the module using - * dp_queue's source api + * ring_buffer's source api * @param min_free_space minimum buffer space in queue required by the module using - * dp_queue's sink api + * ring_buffer's sink api * - * @param flags a combinatin of DP_QUEUE_MODE_* flags determining working mode + * @param flags a combinatin of RING_BUFFER_MODE_* flags determining working mode * * @param id a stream ID, accessible later by sink_get_id/source_get_id * - * @param audio_stream_params pointer to audio params vector, shared between dp_queue and + * @param audio_stream_params pointer to audio params vector, shared between ring_buffer and * comp_buffer for dp modules * */ -struct dp_queue *dp_queue_create(size_t min_available, size_t min_free_space, uint32_t flags, +struct ring_buffer *ring_buffer_create(size_t min_available, size_t min_free_space, uint32_t flags, uint32_t id, struct sof_audio_stream_params *audio_stream_params); -/** - * @brief remove the queue from the list, free dp queue memory - */ -static inline -void dp_queue_free(struct dp_queue *dp_queue) -{ - if (!dp_queue) - return; - CORE_CHECK_STRUCT(dp_queue); - rfree((__sparse_force void *)dp_queue->_data_buffer); - rfree(dp_queue); -} - -/** - * @brief return a handler to sink API of dp_queue. - * the handler may be used by helper functions defined in sink_api.h - */ -static inline -struct sof_sink *dp_queue_get_sink(struct dp_queue *dp_queue) -{ - CORE_CHECK_STRUCT(dp_queue); - return &dp_queue->_sink_api; -} - -/** - * @brief return a handler to source API of dp_queue - * the handler may be used by helper functions defined in source_api.h - */ -static inline -struct sof_source *dp_queue_get_source(struct dp_queue *dp_queue) -{ - CORE_CHECK_STRUCT(dp_queue); - return &dp_queue->_source_api; -} - -/** - * @brief return true if the queue is shared between 2 cores - */ -static inline -bool dp_queue_is_shared(struct dp_queue *dp_queue) -{ - CORE_CHECK_STRUCT(dp_queue); - return !!(dp_queue->_flags & DP_QUEUE_MODE_SHARED); -} - -#endif /* __SOF_DP_QUEUE_H__ */ +#endif /* __SOF_RING_BUFFER_H__ */ diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index aa6505218854..893dcf7b4c82 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -375,7 +377,7 @@ static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool is_shar * disable any interrupts. */ -#define ll_block(cross_core_bind) \ +#define ll_block(cross_core_bind, flags) \ do { \ if (cross_core_bind) \ domain_block(sof_get()->platform_timer_domain); \ @@ -383,7 +385,7 @@ static struct comp_buffer *ipc4_create_buffer(struct comp_dev *src, bool is_shar irq_local_disable(flags); \ } while (0) -#define ll_unblock(cross_core_bind) \ +#define ll_unblock(cross_core_bind, flags) \ do { \ if (cross_core_bind) \ domain_unblock(sof_get()->platform_timer_domain); \ @@ -421,8 +423,8 @@ static int ll_wait_finished_on_core(struct comp_dev *dev) #else -#define ll_block(cross_core_bind) irq_local_disable(flags) -#define ll_unblock(cross_core_bind) irq_local_enable(flags) +#define ll_block(cross_core_bind, flags) irq_local_disable(flags) +#define ll_unblock(cross_core_bind, flags) irq_local_enable(flags) #endif @@ -434,7 +436,7 @@ int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) struct comp_dev *sink; struct ipc4_base_module_cfg source_src_cfg; struct ipc4_base_module_cfg sink_src_cfg; - uint32_t flags; + uint32_t flags = 0; uint32_t ibs = 0; uint32_t obs = 0; uint32_t buf_size; @@ -542,19 +544,36 @@ int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) source_set_min_available(audio_stream_get_source(&buffer->stream), ibs); #if CONFIG_ZEPHYR_DP_SCHEDULER + struct ring_buffer *ring_buffer = NULL; + + if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP || + source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) { + ring_buffer = + ring_buffer_create(source_get_min_available(&buffer->stream._source_api), + sink_get_min_free_space(&buffer->stream._sink_api), + buffer->is_shared ? + RING_BUFFER_MODE_SHARED : RING_BUFFER_MODE_LOCAL, + buf_get_id(buffer), + &buffer->stream.runtime_stream_params); + if (!ring_buffer) + goto free; + } + if (sink->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) - /* data destination module needs to use dp_queue */ - buffer_create_shadow_dp_queue(buffer, false /* at_input = false */); + /* data destination module needs to use ring_buffer */ + buffer_attach_secondary_buffer(buffer, false /* at_input = false */, + &ring_buffer->audio_buffer); else if (source->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) - /* data source module needs to use dp_queue */ - buffer_create_shadow_dp_queue(buffer, true /* at_input = true */); + /* data source module needs to use ring_buffer */ + buffer_attach_secondary_buffer(buffer, true /* at_input = true */, + &ring_buffer->audio_buffer); #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ /* * Connect and bind the buffer to both source and sink components with LL processing been * blocked on corresponding core(s) to prevent IPC or IDC task getting preempted which * could result in buffers being only half connected when a pipeline task gets executed. */ - ll_block(cross_core_bind); + ll_block(cross_core_bind, flags); if (cross_core_bind) { #if CONFIG_CROSS_CORE_STREAM @@ -606,7 +625,7 @@ int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) source->direction_set = true; } - ll_unblock(cross_core_bind); + ll_unblock(cross_core_bind, flags); return IPC4_SUCCESS; @@ -617,7 +636,7 @@ int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) e_sink_connect: pipeline_disconnect(source, buffer, PPL_CONN_DIR_COMP_TO_BUFFER); free: - ll_unblock(cross_core_bind); + ll_unblock(cross_core_bind, flags); buffer_free(buffer); return IPC4_INVALID_RESOURCE_STATE; } @@ -634,7 +653,7 @@ int ipc_comp_disconnect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) struct comp_dev *src, *sink; struct list_item *sink_list; uint32_t src_id, sink_id, buffer_id; - uint32_t flags; + uint32_t flags = 0; int ret, ret1; bool cross_core_unbind; @@ -681,24 +700,24 @@ int ipc_comp_disconnect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) * IPC or IDC task getting preempted which could result in buffers being only half connected * when a pipeline task gets executed. */ - ll_block(cross_core_unbind); + ll_block(cross_core_unbind, flags); if (cross_core_unbind) { #if CONFIG_CROSS_CORE_STREAM /* Make sure LL has finished on both cores */ if (!cpu_is_me(src->ipc_config.core)) if (ll_wait_finished_on_core(src) < 0) { - ll_unblock(cross_core_unbind); + ll_unblock(cross_core_unbind, flags); return IPC4_FAILURE; } if (!cpu_is_me(sink->ipc_config.core)) if (ll_wait_finished_on_core(sink) < 0) { - ll_unblock(cross_core_unbind); + ll_unblock(cross_core_unbind, flags); return IPC4_FAILURE; } #else tr_err(&ipc_tr, "Cross-core binding is disabled"); - ll_unblock(cross_core_unbind); + ll_unblock(cross_core_unbind, flags); return IPC4_FAILURE; #endif } @@ -709,7 +728,7 @@ int ipc_comp_disconnect(struct ipc *ipc, ipc_pipe_comp_connect *_connect) ret = comp_unbind(src, bu); ret1 = comp_unbind(sink, bu); - ll_unblock(cross_core_unbind); + ll_unblock(cross_core_unbind, flags); buffer_free(buffer); diff --git a/uuid-registry.txt b/uuid-registry.txt index 2dad1e151ffb..a479313690b2 100644 --- a/uuid-registry.txt +++ b/uuid-registry.txt @@ -58,7 +58,6 @@ bc3526a7-9b86-4ab4-84a52e02ae70cc10 dma 729bf8b5-e873-4bf5-96908e2a3fd33911 dma_copy 58782c63-1326-4185-845922272e12d1f1 dma_trace 2b972272-c5b1-4b7e-926f0fc5cb4c4690 dma_trace_task -393608d8-4188-11ee-be560242ac122002 dp_queue 87858bc2-baa9-40b6-8e4c2c95ba8b1545 dp_sched ee755917-96b9-4130-b49e37b9d0501993 dp_task b36ee4da-006f-47f9-a06dfecbe2d8b6ce drc @@ -124,6 +123,7 @@ d7f6712d-131c-45a7-82ed6aa9dc2291ea pm_runtime 9d1fb66e-4ffb-497f-994b17719686596e probe 7cad0808-ab10-cd23-ef4512ab34cd56ef probe4 2f0b1901-cac0-4b87-812ff2d5e4f19e4a probe_task +393608d8-4188-11ee-be560242ac122002 ring_buffer 5c7ca334-e15d-11eb-ba800242ac130004 rtnr 5276b491-5b64-464e-8984dc228ef9e6a1 sa 9302adf5-88be-4234-a0a7dca538ef81f4 sai diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 46264a6889ed..c9de5f9b5cbb 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -453,8 +453,8 @@ zephyr_library_sources_ifdef(CONFIG_MATH_LUT_SINE_FIXED # SOF module interface functions add_subdirectory(../src/module module_unused_install/) -if(CONFIG_ZEPHYR_DP_SCHEDULER) - zephyr_library_sources(${SOF_AUDIO_PATH}/dp_queue.c) +if(CONFIG_PIPELINE_2_0) + zephyr_library_sources(${SOF_AUDIO_PATH}/buffers/ring_buffer.c) endif() if(CONFIG_SCHEDULE_DMA_SINGLE_CHANNEL AND NOT(CONFIG_DMA_DOMAIN)) zephyr_library_sources(${SOF_SRC_PATH}/schedule/dma_single_chan_domain.c) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 4112a2b19f50..b8915b69d8ee 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -43,6 +43,14 @@ config DMA_DOMAIN_SEM_LIMIT that SEM_LIMIT covers the maximum number of tasks your system will be executing at some point (worst case). +config PIPELINE_2_0 + bool "Enable pipeline 2.0 changes" + depends on IPC_MAJOR_4 + default y if ACE + help + This flag enables changes to new pipeline structure, known as pipeline2_0 + It is required for certain new features, like DP_SCHEDULER. + config ZEPHYR_DP_SCHEDULER bool "use Zephyr thread based DP scheduler" default y if ACE @@ -50,6 +58,7 @@ config ZEPHYR_DP_SCHEDULER depends on IPC_MAJOR_4 depends on ZEPHYR_SOF_MODULE depends on ACE + depends on PIPELINE_2_0 help Enable Data Processing preemptive scheduler based on Zephyr preemptive threads.