From 63e5ae535e42a4eb2ad9a46729a9b69f242ad768 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 30 Mar 2020 09:54:36 +0200 Subject: [PATCH 1/7] notifier: add flags Adds possibility of defining flags, when performing notifier registration. Signed-off-by: Tomasz Lauda --- src/audio/dai.c | 2 +- src/audio/host.c | 2 +- src/audio/kpb.c | 2 +- src/drivers/dw/dma.c | 2 +- src/include/sof/lib/notifier.h | 3 ++- src/lib/notifier.c | 4 +++- src/probe/probe.c | 4 ++-- src/schedule/dma_single_chan_domain.c | 2 +- src/schedule/ll_schedule.c | 2 +- test/cmocka/src/notifier_mocks.c | 2 +- 10 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/audio/dai.c b/src/audio/dai.c index 4bc72808af3b..ea8080c854d4 100644 --- a/src/audio/dai.c +++ b/src/audio/dai.c @@ -819,7 +819,7 @@ static int dai_config(struct comp_dev *dev, struct sof_ipc_dai_config *config) /* setup callback */ notifier_register(dev, dd->chan, NOTIFIER_ID_DMA_COPY, - dai_dma_cb); + dai_dma_cb, 0); } return dai_set_config(dd->dai, config); diff --git a/src/audio/host.c b/src/audio/host.c index 985dfede0e9c..34307f7cec8b 100644 --- a/src/audio/host.c +++ b/src/audio/host.c @@ -761,7 +761,7 @@ static int host_params(struct comp_dev *dev, } /* set up callback */ - notifier_register(dev, hd->chan, NOTIFIER_ID_DMA_COPY, host_dma_cb); + notifier_register(dev, hd->chan, NOTIFIER_ID_DMA_COPY, host_dma_cb, 0); /* set copy function */ hd->copy = hd->copy_type == COMP_COPY_ONE_SHOT ? host_copy_one_shot : diff --git a/src/audio/kpb.c b/src/audio/kpb.c index e124b9869da4..2066232b96f3 100644 --- a/src/audio/kpb.c +++ b/src/audio/kpb.c @@ -491,7 +491,7 @@ static int kpb_prepare(struct comp_dev *dev) /* Register KPB for notification */ ret = notifier_register(dev, NULL, NOTIFIER_ID_KPB_CLIENT_EVT, - kpb_event_handler); + kpb_event_handler, 0); if (ret < 0) { kpb_free_history_buffer(kpb->hd.c_hb); kpb->hd.c_hb = NULL; diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index e187f1ca6524..b54e3c82b198 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -197,7 +197,7 @@ static struct dma_chan_data *dw_dma_channel_get(struct dma *dma, atomic_add(&dma->num_channels_busy, 1); #if !CONFIG_HW_LLI notifier_register(&dma->chan[i], &dma->chan[i], - NOTIFIER_ID_DMA_IRQ, dw_dma_chan_reload_lli_cb); + NOTIFIER_ID_DMA_IRQ, dw_dma_chan_reload_lli_cb, 0); #endif /* return channel */ diff --git a/src/include/sof/lib/notifier.h b/src/include/sof/lib/notifier.h index 1fe44802ee76..25f40efd4c25 100644 --- a/src/include/sof/lib/notifier.h +++ b/src/include/sof/lib/notifier.h @@ -51,7 +51,8 @@ struct notify_data { struct notify **arch_notify_get(void); int notifier_register(void *receiver, void *caller, enum notify_id type, - void (*cb)(void *arg, enum notify_id type, void *data)); + void (*cb)(void *arg, enum notify_id type, void *data), + uint32_t flags); void notifier_unregister(void *receiver, void *caller, enum notify_id type); void notifier_unregister_all(void *receiver, void *caller); diff --git a/src/lib/notifier.c b/src/lib/notifier.c index 9116212b3f5a..317d35e45fdf 100644 --- a/src/lib/notifier.c +++ b/src/lib/notifier.c @@ -15,6 +15,7 @@ #include #include #include +#include #define trace_notifier(__e, ...) \ trace_event(TRACE_CLASS_NOTIFIER, __e, ##__VA_ARGS__) @@ -33,7 +34,8 @@ struct callback_handle { }; int notifier_register(void *receiver, void *caller, enum notify_id type, - void (*cb)(void *arg, enum notify_id type, void *data)) + void (*cb)(void *arg, enum notify_id type, void *data), + uint32_t flags) { struct notify *notify = *arch_notify_get(); struct callback_handle *handle; diff --git a/src/probe/probe.c b/src/probe/probe.c index 03354123ef41..b92fba88042e 100644 --- a/src/probe/probe.c +++ b/src/probe/probe.c @@ -1038,9 +1038,9 @@ int probe_point_add(uint32_t count, struct probe_point *probe) probe[i].stream_tag; notifier_register(_probe, dev->cb, NOTIFIER_ID_BUFFER_PRODUCE, - &probe_cb_produce); + &probe_cb_produce, 0); notifier_register(_probe, dev->cb, NOTIFIER_ID_BUFFER_FREE, - &probe_cb_free); + &probe_cb_free, 0); } return 0; diff --git a/src/schedule/dma_single_chan_domain.c b/src/schedule/dma_single_chan_domain.c index 9c717d5f06c0..04acea5f9b0c 100644 --- a/src/schedule/dma_single_chan_domain.c +++ b/src/schedule/dma_single_chan_domain.c @@ -247,7 +247,7 @@ static int dma_single_chan_domain_register(struct ll_schedule_domain *domain, /* register for source change notifications */ if (register_needed) notifier_register(domain, NULL, NOTIFIER_ID_DMA_DOMAIN_CHANGE, - dma_domain_changed); + dma_domain_changed, 0); dma_domain->owner = channel->core; diff --git a/src/schedule/ll_schedule.c b/src/schedule/ll_schedule.c index 1a86de950b14..314fb363e321 100644 --- a/src/schedule/ll_schedule.c +++ b/src/schedule/ll_schedule.c @@ -530,7 +530,7 @@ int scheduler_init_ll(struct ll_schedule_domain *domain) /* notification of clock changes */ notifier_register(sch, NULL, NOTIFIER_CLK_CHANGE_ID(domain->clk), - ll_scheduler_notify); + ll_scheduler_notify, 0); scheduler_init(domain->type, &schedule_ll_ops, sch); diff --git a/test/cmocka/src/notifier_mocks.c b/test/cmocka/src/notifier_mocks.c index 07557bbca1cc..91b607ed5dcb 100644 --- a/test/cmocka/src/notifier_mocks.c +++ b/test/cmocka/src/notifier_mocks.c @@ -50,7 +50,7 @@ void notifier_event(const void *caller, enum notify_id type, uint32_t core_mask, } int notifier_register(void *receiver, void *caller, enum notify_id type, - void (*cb)(void *arg, enum notify_id type, void *data)) + void (*cb)(void *arg, enum notify_id type, void *data), uint32_t flags) { struct notify *notify = *arch_notify_get(); struct callback_handle *handle; From 51e2a0761b441128eb76bb1be3ab4d7351355558 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 30 Mar 2020 10:00:43 +0200 Subject: [PATCH 2/7] notifier: handle new aggregate flag Implements handling of new notifier aggregate flag. With this flag set the notifier will automatically use existing handle instead of creating a new one. It's helpful when we want to register only once for an event, but it's very hard to explicitly keep track of number of registrations. Signed-off-by: Tomasz Lauda --- src/include/sof/lib/notifier.h | 4 ++++ src/lib/notifier.c | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/include/sof/lib/notifier.h b/src/include/sof/lib/notifier.h index 25f40efd4c25..5d43f3e8e778 100644 --- a/src/include/sof/lib/notifier.h +++ b/src/include/sof/lib/notifier.h @@ -8,6 +8,7 @@ #ifndef __SOF_LIB_NOTIFIER_H__ #define __SOF_LIB_NOTIFIER_H__ +#include #include #include #include @@ -17,6 +18,9 @@ #define NOTIFIER_TARGET_CORE_LOCAL NOTIFIER_TARGET_CORE_MASK(cpu_get_id()) #define NOTIFIER_TARGET_CORE_ALL_MASK 0xFFFFFFFF +/** \brief Notifier flags. */ +#define NOTIFIER_FLAG_AGGREGATE BIT(0) + enum notify_id { NOTIFIER_ID_CPU_FREQ = 0, /* struct clock_notify_data * */ NOTIFIER_ID_SSP_FREQ, /* struct clock_notify_data * */ diff --git a/src/lib/notifier.c b/src/lib/notifier.c index 317d35e45fdf..f7319bc49eef 100644 --- a/src/lib/notifier.c +++ b/src/lib/notifier.c @@ -31,6 +31,7 @@ struct callback_handle { void *caller; void (*cb)(void *arg, enum notify_id, void *data); struct list_item list; + uint32_t num_registrations; }; int notifier_register(void *receiver, void *caller, enum notify_id type, @@ -42,6 +43,16 @@ int notifier_register(void *receiver, void *caller, enum notify_id type, assert(type >= NOTIFIER_ID_CPU_FREQ && type < NOTIFIER_ID_COUNT); + /* Find already registered event of this type */ + if (flags & NOTIFIER_FLAG_AGGREGATE && + !list_is_empty(¬ify->list[type])) { + handle = container_of((¬ify->list[type])->next, + struct callback_handle, list); + handle->num_registrations++; + + return 0; + } + handle = rzalloc(SOF_MEM_ZONE_SYS_RUNTIME, 0, SOF_MEM_CAPS_RAM, sizeof(*handle)); @@ -53,6 +64,7 @@ int notifier_register(void *receiver, void *caller, enum notify_id type, handle->receiver = receiver; handle->caller = caller; handle->cb = cb; + handle->num_registrations = 1; list_item_prepend(&handle->list, ¬ify->list[type]); @@ -82,8 +94,10 @@ void notifier_unregister(void *receiver, void *caller, enum notify_id type) handle = container_of(wlist, struct callback_handle, list); if ((!receiver || handle->receiver == receiver) && (!caller || handle->caller == caller)) { - list_item_del(&handle->list); - rfree(handle); + if (!--handle->num_registrations) { + list_item_del(&handle->list); + rfree(handle); + } } } } From c2c09c2808f0d3297551eba5a2418da987c118cd Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 6 Apr 2020 10:20:17 +0200 Subject: [PATCH 3/7] notifier: protect from preemption Adds irq_local_disable/enable during notifier_register and notifier_unregister calls to protect notify list from preemption. Signed-off-by: Tomasz Lauda --- src/lib/notifier.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/lib/notifier.c b/src/lib/notifier.c index f7319bc49eef..cdb065c938d5 100644 --- a/src/lib/notifier.c +++ b/src/lib/notifier.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -40,9 +41,13 @@ int notifier_register(void *receiver, void *caller, enum notify_id type, { struct notify *notify = *arch_notify_get(); struct callback_handle *handle; + uint32_t irq_flags; + int ret = 0; assert(type >= NOTIFIER_ID_CPU_FREQ && type < NOTIFIER_ID_COUNT); + irq_local_disable(irq_flags); + /* Find already registered event of this type */ if (flags & NOTIFIER_FLAG_AGGREGATE && !list_is_empty(¬ify->list[type])) { @@ -50,7 +55,7 @@ int notifier_register(void *receiver, void *caller, enum notify_id type, struct callback_handle, list); handle->num_registrations++; - return 0; + goto out; } handle = rzalloc(SOF_MEM_ZONE_SYS_RUNTIME, 0, SOF_MEM_CAPS_RAM, @@ -58,7 +63,8 @@ int notifier_register(void *receiver, void *caller, enum notify_id type, if (!handle) { trace_notifier_error("notifier_register(): callback handle allocation failed."); - return -ENOMEM; + ret = -ENOMEM; + goto out; } handle->receiver = receiver; @@ -68,7 +74,10 @@ int notifier_register(void *receiver, void *caller, enum notify_id type, list_item_prepend(&handle->list, ¬ify->list[type]); - return 0; +out: + irq_local_enable(irq_flags); + + return ret; } void notifier_unregister(void *receiver, void *caller, enum notify_id type) @@ -77,9 +86,12 @@ void notifier_unregister(void *receiver, void *caller, enum notify_id type) struct list_item *wlist; struct list_item *tlist; struct callback_handle *handle; + uint32_t flags; assert(type >= NOTIFIER_ID_CPU_FREQ && type < NOTIFIER_ID_COUNT); + irq_local_disable(flags); + /* * Unregister all matching callbacks * If receiver is NULL, unregister all callbacks with matching callers @@ -100,6 +112,8 @@ void notifier_unregister(void *receiver, void *caller, enum notify_id type) } } } + + irq_local_enable(flags); } void notifier_unregister_all(void *receiver, void *caller) From 5620faa8fae088cef25de44bd6caf67016de9b6f Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 30 Mar 2020 10:10:29 +0200 Subject: [PATCH 4/7] hda-dma: save irq_disabled in channel data Saves irq_disabled config flag in private channel data. Signed-off-by: Tomasz Lauda --- src/drivers/intel/cavs/hda-dma.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index ec1a0c80454b..beae473f412b 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -135,6 +136,10 @@ struct hda_chan_data { uint32_t period_bytes; uint32_t buffer_bytes; + bool irq_disabled; /**< indicates whether channel is used by the + * pipeline scheduled on DMA + */ + #if HDA_DMA_PTR_DBG struct hda_dbg_data dbg_data; #endif @@ -657,6 +662,7 @@ static int hda_dma_set_config(struct dma_chan_data *channel, hda_chan = dma_chan_get_data(channel); hda_chan->period_bytes = period_bytes; hda_chan->buffer_bytes = buffer_bytes; + hda_chan->irq_disabled = config->irq_disabled; /* init channel in HW */ dma_chan_reg_write(channel, DGBBA, buffer_addr); From 5d94349bf6ea1b1328fadcafa3b438e2ca813d96 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 30 Mar 2020 10:24:38 +0200 Subject: [PATCH 5/7] hda-dma: exit L1 only once for all DMA channels Improves host component performance by extracting DMA L1 exit to be executed commonly for all active DMA channels as registered callback in low latency scheduler for timer domain. There is no need to wait so many cycles for every channel separately. Signed-off-by: Tomasz Lauda --- src/drivers/intel/cavs/hda-dma.c | 77 ++++++++++++++++++++++++++++---- src/include/sof/lib/notifier.h | 2 + src/schedule/ll_schedule.c | 6 +++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index beae473f412b..3447aa5d7dec 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -220,6 +221,12 @@ static void hda_dma_get_dbg_vals(struct dma_chan_data *chan, #define hda_dma_ptr_trace(...) #endif +static void hda_dma_l1_exit_notify(void *arg, enum notify_id type, void *data) +{ + /* Force Host DMA to exit L1 */ + pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0); +} + static inline int hda_dma_is_buffer_full(struct dma_chan_data *chan) { return dma_chan_reg_read(chan, DGCS) & DGCS_BF; @@ -282,6 +289,7 @@ static int hda_dma_wait_for_buffer_empty(struct dma_chan_data *chan) static void hda_dma_post_copy(struct dma_chan_data *chan, int bytes) { + struct hda_chan_data *hda_chan = dma_chan_get_data(chan); struct dma_cb_data next = { .channel = chan, .elem = { .size = bytes }, @@ -297,8 +305,9 @@ static void hda_dma_post_copy(struct dma_chan_data *chan, int bytes) */ hda_dma_inc_fp(chan, bytes); - /* Force Host DMA to exit L1 */ - pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0); + /* Force Host DMA to exit L1 if scheduled on DMA */ + if (!hda_chan->irq_disabled) + pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0); } else { /* * set BFPI to let link gateway know we have read size, @@ -323,10 +332,47 @@ static int hda_dma_link_copy_ch(struct dma_chan_data *chan, int bytes) return 0; } +static int hda_dma_host_start(struct dma_chan_data *channel) +{ + struct hda_chan_data *hda_chan = dma_chan_get_data(channel); + int ret = 0; + + /* Force Host DMA to exit L1 only on start*/ + if (!(hda_chan->state & HDA_STATE_RELEASE)) + pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0); + + if (!hda_chan->irq_disabled) + return ret; + + /* Register common L1 exit for all channels */ + ret = notifier_register(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER), + NOTIFIER_ID_LL_POST_RUN, hda_dma_l1_exit_notify, + NOTIFIER_FLAG_AGGREGATE); + if (ret < 0) + trace_hddma_error("hda-dmac: %d channel %d, cannot register notification %d", + channel->dma->plat_data.id, channel->index, + ret); + + return ret; +} + +static void hda_dma_host_stop(struct dma_chan_data *channel) +{ + struct hda_chan_data *hda_chan = dma_chan_get_data(channel); + + if (!hda_chan->irq_disabled) + return; + + /* Unregister L1 exit */ + notifier_unregister(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER), + NOTIFIER_ID_LL_POST_RUN); +} + /* lock should be held by caller */ -static void hda_dma_enable_unlock(struct dma_chan_data *channel) +static int hda_dma_enable_unlock(struct dma_chan_data *channel) { struct hda_chan_data *hda_chan; + int ret; trace_hddma("hda-dmac: %d channel %d -> enable", channel->dma->plat_data.id, channel->index); @@ -341,10 +387,12 @@ static void hda_dma_enable_unlock(struct dma_chan_data *channel) hda_chan = dma_chan_get_data(channel); hda_chan->desc_avail = channel->desc_count; - /* Force Host DMA to exit L1 */ if (channel->direction == DMA_DIR_HMEM_TO_LMEM || - channel->direction == DMA_DIR_LMEM_TO_HMEM) - pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0); + channel->direction == DMA_DIR_LMEM_TO_HMEM) { + ret = hda_dma_host_start(channel); + if (ret < 0) + return ret; + } /* start link output transfer now */ if (channel->direction == DMA_DIR_MEM_TO_DEV && @@ -355,6 +403,8 @@ static void hda_dma_enable_unlock(struct dma_chan_data *channel) hda_dma_get_dbg_vals(channel, HDA_DBG_POST, HDA_DBG_BOTH); hda_dma_ptr_trace(channel, "enable", HDA_DBG_BOTH); + + return 0; } /* notify DMA to copy bytes */ @@ -478,7 +528,9 @@ static int hda_dma_start(struct dma_chan_data *channel) goto out; } - hda_dma_enable_unlock(channel); + ret = hda_dma_enable_unlock(channel); + if (ret < 0) + goto out; channel->status = COMP_STATE_ACTIVE; channel->core = cpu_get_id(); @@ -492,6 +544,7 @@ static int hda_dma_release(struct dma_chan_data *channel) { struct hda_chan_data *hda_chan = dma_chan_get_data(channel); uint32_t flags; + int ret = 0; irq_local_disable(flags); @@ -504,8 +557,12 @@ static int hda_dma_release(struct dma_chan_data *channel) */ hda_chan->state |= HDA_STATE_RELEASE; + if (channel->direction == DMA_DIR_HMEM_TO_LMEM || + channel->direction == DMA_DIR_LMEM_TO_HMEM) + ret = hda_dma_host_start(channel); + irq_local_enable(flags); - return 0; + return ret; } static int hda_dma_pause(struct dma_chan_data *channel) @@ -542,6 +599,10 @@ static int hda_dma_stop(struct dma_chan_data *channel) trace_hddma("hda-dmac: %d channel %d -> stop", channel->dma->plat_data.id, channel->index); + if (channel->direction == DMA_DIR_HMEM_TO_LMEM || + channel->direction == DMA_DIR_LMEM_TO_HMEM) + hda_dma_host_stop(channel); + /* disable the channel */ dma_chan_reg_update_bits(channel, DGCS, DGCS_GEN | DGCS_FIFORDY, 0); channel->status = COMP_STATE_PREPARE; diff --git a/src/include/sof/lib/notifier.h b/src/include/sof/lib/notifier.h index 5d43f3e8e778..e537ae38eb83 100644 --- a/src/include/sof/lib/notifier.h +++ b/src/include/sof/lib/notifier.h @@ -30,6 +30,8 @@ enum notify_id { NOTIFIER_ID_BUFFER_CONSUME, /* struct buffer_cb_transact* */ NOTIFIER_ID_BUFFER_FREE, /* struct buffer_cb_free* */ NOTIFIER_ID_DMA_COPY, /* struct dma_cb_data* */ + NOTIFIER_ID_LL_PRE_RUN, /* NULL */ + NOTIFIER_ID_LL_POST_RUN, /* NULL */ NOTIFIER_ID_DMA_IRQ, /* struct dma_chan_data * */ NOTIFIER_ID_COUNT }; diff --git a/src/schedule/ll_schedule.c b/src/schedule/ll_schedule.c index 314fb363e321..b25939f36dc7 100644 --- a/src/schedule/ll_schedule.c +++ b/src/schedule/ll_schedule.c @@ -175,10 +175,16 @@ static void schedule_ll_tasks_run(void *data) perf_cnt_init(&sch->pcd); + notifier_event(sch, NOTIFIER_ID_LL_PRE_RUN, + NOTIFIER_TARGET_CORE_LOCAL, NULL, 0); + /* run tasks if there are any pending */ if (schedule_ll_is_pending(sch)) schedule_ll_tasks_execute(sch, last_tick); + notifier_event(sch, NOTIFIER_ID_LL_POST_RUN, + NOTIFIER_TARGET_CORE_LOCAL, NULL, 0); + perf_cnt_stamp(&sch->pcd, perf_ll_sched_trace, sch); spin_lock(&sch->domain->lock); From fa7e7aa5734054c389b75fc312c0b8796e832b52 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 6 Apr 2020 13:52:00 +0200 Subject: [PATCH 6/7] pm_runtime: cavs: make private data shared Allocates cavs_pm_runtime_data as shared to make it usable by multicore code. Signed-off-by: Tomasz Lauda --- src/platform/intel/cavs/lib/pm_runtime.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/platform/intel/cavs/lib/pm_runtime.c b/src/platform/intel/cavs/lib/pm_runtime.c index c4b8b843e376..b107eb866281 100644 --- a/src/platform/intel/cavs/lib/pm_runtime.c +++ b/src/platform/intel/cavs/lib/pm_runtime.c @@ -81,16 +81,20 @@ static inline void cavs_pm_runtime_enable_dsp(bool enable) irq_local_enable(flags); trace_power("pm_runtime_enable_dsp dsp_d0_sref %d", pprd->dsp_d0_sref); + + platform_shared_commit(pprd, sizeof(*pprd)); } static inline bool cavs_pm_runtime_is_active_dsp(void) { struct pm_runtime_data *prd = pm_runtime_data_get(); struct cavs_pm_runtime_data *pprd = prd->platform_data; + int is_active = pprd->dsp_d0_sref > 0; platform_shared_commit(prd, sizeof(*prd)); + platform_shared_commit(pprd, sizeof(*pprd)); - return pprd->dsp_d0_sref > 0; + return is_active; } #if CONFIG_CAVS_SSP @@ -368,7 +372,8 @@ void platform_pm_runtime_init(struct pm_runtime_data *prd) { struct cavs_pm_runtime_data *pprd; - pprd = rzalloc(SOF_MEM_ZONE_SYS, 0, SOF_MEM_CAPS_RAM, sizeof(*pprd)); + pprd = rzalloc(SOF_MEM_ZONE_SYS, SOF_MEM_FLAG_SHARED, SOF_MEM_CAPS_RAM, + sizeof(*pprd)); prd->platform_data = pprd; } From de843c992752a0ce0b697cf548168bbc0f797ed3 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 6 Apr 2020 13:45:13 +0200 Subject: [PATCH 7/7] pm_runtime: add core synchronization for Host DMA L1 Exit Implements multicore synchronization mechanism for Host DMA L1 Exit. Simple reference counter is added to avoid a situation, where shorter processing on one core forces Host DMA bus to exit L1, when there is still transfer happening on other core. PM_RUNTIME_HOST_DMA_L1 with get is called in new NOTIFIER_ID_LL_PRE_RUN notification event. Signed-off-by: Tomasz Lauda --- src/drivers/intel/cavs/hda-dma.c | 25 ++++++++++++++++++ .../intel/cavs/include/cavs/lib/pm_runtime.h | 1 + src/platform/intel/cavs/lib/pm_runtime.c | 26 ++++++++++++++++++- 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index 3447aa5d7dec..66ad146cd862 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -221,6 +221,12 @@ static void hda_dma_get_dbg_vals(struct dma_chan_data *chan, #define hda_dma_ptr_trace(...) #endif +static void hda_dma_l1_entry_notify(void *arg, enum notify_id type, void *data) +{ + /* Notify about Host DMA usage */ + pm_runtime_get(PM_RUNTIME_HOST_DMA_L1, 0); +} + static void hda_dma_l1_exit_notify(void *arg, enum notify_id type, void *data) { /* Force Host DMA to exit L1 */ @@ -344,6 +350,15 @@ static int hda_dma_host_start(struct dma_chan_data *channel) if (!hda_chan->irq_disabled) return ret; + /* Inform about Host DMA usage */ + ret = notifier_register(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER), + NOTIFIER_ID_LL_PRE_RUN, hda_dma_l1_entry_notify, + NOTIFIER_FLAG_AGGREGATE); + if (ret < 0) + trace_hddma_error("hda-dmac: %d channel %d, cannot register notification %d", + channel->dma->plat_data.id, channel->index, + ret); + /* Register common L1 exit for all channels */ ret = notifier_register(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER), NOTIFIER_ID_LL_POST_RUN, hda_dma_l1_exit_notify, @@ -363,6 +378,10 @@ static void hda_dma_host_stop(struct dma_chan_data *channel) if (!hda_chan->irq_disabled) return; + /* Unregister L1 entry */ + notifier_unregister(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER), + NOTIFIER_ID_LL_PRE_RUN); + /* Unregister L1 exit */ notifier_unregister(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER), NOTIFIER_ID_LL_POST_RUN); @@ -389,6 +408,7 @@ static int hda_dma_enable_unlock(struct dma_chan_data *channel) if (channel->direction == DMA_DIR_HMEM_TO_LMEM || channel->direction == DMA_DIR_LMEM_TO_HMEM) { + pm_runtime_get(PM_RUNTIME_HOST_DMA_L1, 0); ret = hda_dma_host_start(channel); if (ret < 0) return ret; @@ -418,6 +438,7 @@ static int hda_dma_link_copy(struct dma_chan_data *channel, int bytes, static int hda_dma_host_copy(struct dma_chan_data *channel, int bytes, uint32_t flags) { + struct hda_chan_data *hda_chan = dma_chan_get_data(channel); int ret; tracev_hddma("hda-dmac: %d channel %d -> copy 0x%x bytes", @@ -425,6 +446,10 @@ static int hda_dma_host_copy(struct dma_chan_data *channel, int bytes, hda_dma_get_dbg_vals(channel, HDA_DBG_PRE, HDA_DBG_HOST); + /* Register Host DMA usage */ + if (!hda_chan->irq_disabled) + pm_runtime_get(PM_RUNTIME_HOST_DMA_L1, 0); + /* blocking mode copy */ if (flags & DMA_COPY_BLOCKING) { ret = channel->direction == DMA_DIR_HMEM_TO_LMEM ? diff --git a/src/platform/intel/cavs/include/cavs/lib/pm_runtime.h b/src/platform/intel/cavs/include/cavs/lib/pm_runtime.h index 0234b9eb1ffd..94e590c83271 100644 --- a/src/platform/intel/cavs/include/cavs/lib/pm_runtime.h +++ b/src/platform/intel/cavs/include/cavs/lib/pm_runtime.h @@ -21,6 +21,7 @@ struct pm_runtime_data; /** \brief cAVS specific runtime power management data. */ struct cavs_pm_runtime_data { int dsp_d0_sref; /**< simple ref counter, accessed by core 0 only */ + int host_dma_l1_sref; /**< ref counter for Host DMA accesses */ }; #endif diff --git a/src/platform/intel/cavs/lib/pm_runtime.c b/src/platform/intel/cavs/lib/pm_runtime.c index b107eb866281..bf0e497e23ed 100644 --- a/src/platform/intel/cavs/lib/pm_runtime.c +++ b/src/platform/intel/cavs/lib/pm_runtime.c @@ -40,17 +40,35 @@ #include #endif +/** \brief Registers Host DMA access by incrementing ref counter. */ +static void cavs_pm_runtime_host_dma_l1_entry(void) +{ + struct pm_runtime_data *prd = pm_runtime_data_get(); + struct cavs_pm_runtime_data *pprd = prd->platform_data; + uint32_t flags; + + spin_lock_irq(&prd->lock, flags); + + pprd->host_dma_l1_sref++; + + platform_shared_commit(prd, sizeof(*prd)); + platform_shared_commit(pprd, sizeof(*pprd)); + + spin_unlock_irq(&prd->lock, flags); +} + /** * \brief Forces Host DMAs to exit L1. */ static inline void cavs_pm_runtime_force_host_dma_l1_exit(void) { struct pm_runtime_data *prd = pm_runtime_data_get(); + struct cavs_pm_runtime_data *pprd = prd->platform_data; uint32_t flags; spin_lock_irq(&prd->lock, flags); - if (!(shim_read(SHIM_SVCFG) & SHIM_SVCFG_FORCE_L1_EXIT)) { + if (!--pprd->host_dma_l1_sref) { shim_write(SHIM_SVCFG, shim_read(SHIM_SVCFG) | SHIM_SVCFG_FORCE_L1_EXIT); @@ -60,6 +78,9 @@ static inline void cavs_pm_runtime_force_host_dma_l1_exit(void) shim_read(SHIM_SVCFG) & ~(SHIM_SVCFG_FORCE_L1_EXIT)); } + platform_shared_commit(prd, sizeof(*prd)); + platform_shared_commit(pprd, sizeof(*pprd)); + spin_unlock_irq(&prd->lock, flags); } @@ -382,6 +403,9 @@ void platform_pm_runtime_get(enum pm_runtime_context context, uint32_t index, { /* Action based on context */ switch (context) { + case PM_RUNTIME_HOST_DMA_L1: + cavs_pm_runtime_host_dma_l1_entry(); + break; #if CONFIG_CAVS_SSP case SSP_CLK: cavs_pm_runtime_dis_ssp_clk_gating(index);