hda-dma: exit L1 only once for all DMA channels

src/audio/dai.c

@@ -819,7 +819,7 @@ static int dai_config(struct comp_dev *dev, struct sof_ipc_dai_config *config)
 
 		/* setup callback */
 		notifier_register(dev, dd->chan, NOTIFIER_ID_DMA_COPY,
-				  dai_dma_cb);
+				  dai_dma_cb, 0);
 	}
 
 	return dai_set_config(dd->dai, config);

src/audio/host.c

@@ -761,7 +761,7 @@ static int host_params(struct comp_dev *dev,
 	}
 
 	/* set up callback */
-	notifier_register(dev, hd->chan, NOTIFIER_ID_DMA_COPY, host_dma_cb);
+	notifier_register(dev, hd->chan, NOTIFIER_ID_DMA_COPY, host_dma_cb, 0);
 
 	/* set copy function */
 	hd->copy = hd->copy_type == COMP_COPY_ONE_SHOT ? host_copy_one_shot :

src/audio/kpb.c

@@ -491,7 +491,7 @@ static int kpb_prepare(struct comp_dev *dev)
 
 	/* Register KPB for notification */
 	ret = notifier_register(dev, NULL, NOTIFIER_ID_KPB_CLIENT_EVT,
-				kpb_event_handler);
+				kpb_event_handler, 0);
 	if (ret < 0) {
 		kpb_free_history_buffer(kpb->hd.c_hb);
 		kpb->hd.c_hb = NULL;

src/drivers/dw/dma.c

@@ -197,7 +197,7 @@ static struct dma_chan_data *dw_dma_channel_get(struct dma *dma,
 		atomic_add(&dma->num_channels_busy, 1);
 #if !CONFIG_HW_LLI
 		notifier_register(&dma->chan[i], &dma->chan[i],
-			NOTIFIER_ID_DMA_IRQ, dw_dma_chan_reload_lli_cb);
+			NOTIFIER_ID_DMA_IRQ, dw_dma_chan_reload_lli_cb, 0);
 #endif
 
 		/* return channel */

src/drivers/intel/cavs/hda-dma.c

@@ -18,11 +18,13 @@
 #include <sof/lib/pm_runtime.h>
 #include <sof/lib/notifier.h>
 #include <sof/platform.h>
+#include <sof/schedule/schedule.h>
 #include <sof/spinlock.h>
 #include <sof/trace/trace.h>
 #include <ipc/topology.h>
 #include <user/trace.h>
 #include <errno.h>
+#include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -135,6 +137,10 @@ struct hda_chan_data {
 	uint32_t period_bytes;
 	uint32_t buffer_bytes;
 
+	bool irq_disabled;	/**< indicates whether channel is used by the
+				  * pipeline scheduled on DMA
+				  */
+
 #if HDA_DMA_PTR_DBG
 	struct hda_dbg_data dbg_data;
 #endif
@@ -215,6 +221,18 @@ static void hda_dma_get_dbg_vals(struct dma_chan_data *chan,
 #define hda_dma_ptr_trace(...)
 #endif
 
+static void hda_dma_l1_entry_notify(void *arg, enum notify_id type, void *data)
+{
+	/* Notify about Host DMA usage */
+	pm_runtime_get(PM_RUNTIME_HOST_DMA_L1, 0);
+}
+
+static void hda_dma_l1_exit_notify(void *arg, enum notify_id type, void *data)
+{
+	/* Force Host DMA to exit L1 */
+	pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0);
+}
+
 static inline int hda_dma_is_buffer_full(struct dma_chan_data *chan)
 {
 	return dma_chan_reg_read(chan, DGCS) & DGCS_BF;
@@ -277,6 +295,7 @@ static int hda_dma_wait_for_buffer_empty(struct dma_chan_data *chan)
 
 static void hda_dma_post_copy(struct dma_chan_data *chan, int bytes)
 {
+	struct hda_chan_data *hda_chan = dma_chan_get_data(chan);
 	struct dma_cb_data next = {
 		.channel = chan,
 		.elem = { .size = bytes },
@@ -292,8 +311,9 @@ static void hda_dma_post_copy(struct dma_chan_data *chan, int bytes)
 		 */
 		hda_dma_inc_fp(chan, bytes);
 
-		/* Force Host DMA to exit L1 */
-		pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0);
+		/* Force Host DMA to exit L1 if scheduled on DMA */
+		if (!hda_chan->irq_disabled)
+			pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0);
 	} else {
 		/*
 		 * set BFPI to let link gateway know we have read size,
@@ -318,10 +338,60 @@ static int hda_dma_link_copy_ch(struct dma_chan_data *chan, int bytes)
 	return 0;
 }
 
+static int hda_dma_host_start(struct dma_chan_data *channel)
+{
+	struct hda_chan_data *hda_chan = dma_chan_get_data(channel);
+	int ret = 0;
+
+	/* Force Host DMA to exit L1 only on start*/
+	if (!(hda_chan->state & HDA_STATE_RELEASE))
+		pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0);
+
+	if (!hda_chan->irq_disabled)
+		return ret;
+
+	/* Inform about Host DMA usage */
+	ret = notifier_register(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER),
+				NOTIFIER_ID_LL_PRE_RUN, hda_dma_l1_entry_notify,
+				NOTIFIER_FLAG_AGGREGATE);
+	if (ret < 0)
+		trace_hddma_error("hda-dmac: %d channel %d, cannot register notification %d",
+				  channel->dma->plat_data.id, channel->index,
+				  ret);
+
+	/* Register common L1 exit for all channels */
+	ret = notifier_register(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER),
+				NOTIFIER_ID_LL_POST_RUN, hda_dma_l1_exit_notify,
+				NOTIFIER_FLAG_AGGREGATE);
+	if (ret < 0)
+		trace_hddma_error("hda-dmac: %d channel %d, cannot register notification %d",
+				  channel->dma->plat_data.id, channel->index,
+				  ret);
+
+	return ret;
+}
+
+static void hda_dma_host_stop(struct dma_chan_data *channel)
+{
+	struct hda_chan_data *hda_chan = dma_chan_get_data(channel);
+
+	if (!hda_chan->irq_disabled)
+		return;
+
+	/* Unregister L1 entry */
+	notifier_unregister(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER),
+			    NOTIFIER_ID_LL_PRE_RUN);
+
+	/* Unregister L1 exit */
+	notifier_unregister(NULL, scheduler_get_data(SOF_SCHEDULE_LL_TIMER),
+			    NOTIFIER_ID_LL_POST_RUN);
+}
+
 /* lock should be held by caller */
-static void hda_dma_enable_unlock(struct dma_chan_data *channel)
+static int hda_dma_enable_unlock(struct dma_chan_data *channel)
 {
 	struct hda_chan_data *hda_chan;
+	int ret;
 
 	trace_hddma("hda-dmac: %d channel %d -> enable",
 		    channel->dma->plat_data.id, channel->index);
@@ -336,10 +406,13 @@ static void hda_dma_enable_unlock(struct dma_chan_data *channel)
 	hda_chan = dma_chan_get_data(channel);
 	hda_chan->desc_avail = channel->desc_count;
 
-	/* Force Host DMA to exit L1 */
 	if (channel->direction == DMA_DIR_HMEM_TO_LMEM ||
-	    channel->direction == DMA_DIR_LMEM_TO_HMEM)
-		pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0);
+	    channel->direction == DMA_DIR_LMEM_TO_HMEM) {
+		pm_runtime_get(PM_RUNTIME_HOST_DMA_L1, 0);
+		ret = hda_dma_host_start(channel);
+		if (ret < 0)
+			return ret;
+	}
 
 	/* start link output transfer now */
 	if (channel->direction == DMA_DIR_MEM_TO_DEV &&
@@ -350,6 +423,8 @@ static void hda_dma_enable_unlock(struct dma_chan_data *channel)
 
 	hda_dma_get_dbg_vals(channel, HDA_DBG_POST, HDA_DBG_BOTH);
 	hda_dma_ptr_trace(channel, "enable", HDA_DBG_BOTH);
+
+	return 0;
 }
 
 /* notify DMA to copy bytes */
@@ -363,13 +438,18 @@ static int hda_dma_link_copy(struct dma_chan_data *channel, int bytes,
 static int hda_dma_host_copy(struct dma_chan_data *channel, int bytes,
 			     uint32_t flags)
 {
+	struct hda_chan_data *hda_chan = dma_chan_get_data(channel);
 	int ret;
 
 	tracev_hddma("hda-dmac: %d channel %d -> copy 0x%x bytes",
 		     channel->dma->plat_data.id, channel->index, bytes);
 
 	hda_dma_get_dbg_vals(channel, HDA_DBG_PRE, HDA_DBG_HOST);
 
+	/* Register Host DMA usage */
+	if (!hda_chan->irq_disabled)
+		pm_runtime_get(PM_RUNTIME_HOST_DMA_L1, 0);
+
 	/* blocking mode copy */
 	if (flags & DMA_COPY_BLOCKING) {
 		ret = channel->direction == DMA_DIR_HMEM_TO_LMEM ?
@@ -473,7 +553,9 @@ static int hda_dma_start(struct dma_chan_data *channel)
 		goto out;
 	}
 
-	hda_dma_enable_unlock(channel);
+	ret = hda_dma_enable_unlock(channel);
+	if (ret < 0)
+		goto out;
 
 	channel->status = COMP_STATE_ACTIVE;
 	channel->core = cpu_get_id();
@@ -487,6 +569,7 @@ static int hda_dma_release(struct dma_chan_data *channel)
 {
 	struct hda_chan_data *hda_chan = dma_chan_get_data(channel);
 	uint32_t flags;
+	int ret = 0;
 
 	irq_local_disable(flags);
 
@@ -499,8 +582,12 @@ static int hda_dma_release(struct dma_chan_data *channel)
 	 */
 	hda_chan->state |= HDA_STATE_RELEASE;
 
+	if (channel->direction == DMA_DIR_HMEM_TO_LMEM ||
+	    channel->direction == DMA_DIR_LMEM_TO_HMEM)
+		ret = hda_dma_host_start(channel);
+
 	irq_local_enable(flags);
-	return 0;
+	return ret;
 }
 
 static int hda_dma_pause(struct dma_chan_data *channel)
@@ -537,6 +624,10 @@ static int hda_dma_stop(struct dma_chan_data *channel)
 	trace_hddma("hda-dmac: %d channel %d -> stop",
 		    channel->dma->plat_data.id, channel->index);
 
+	if (channel->direction == DMA_DIR_HMEM_TO_LMEM ||
+	    channel->direction == DMA_DIR_LMEM_TO_HMEM)
+		hda_dma_host_stop(channel);
+
 	/* disable the channel */
 	dma_chan_reg_update_bits(channel, DGCS, DGCS_GEN | DGCS_FIFORDY, 0);
 	channel->status = COMP_STATE_PREPARE;
@@ -657,6 +748,7 @@ static int hda_dma_set_config(struct dma_chan_data *channel,
 	hda_chan = dma_chan_get_data(channel);
 	hda_chan->period_bytes = period_bytes;
 	hda_chan->buffer_bytes = buffer_bytes;
+	hda_chan->irq_disabled = config->irq_disabled;
 
 	/* init channel in HW */
 	dma_chan_reg_write(channel, DGBBA, buffer_addr);

src/include/sof/lib/notifier.h

@@ -8,6 +8,7 @@
 #ifndef __SOF_LIB_NOTIFIER_H__
 #define __SOF_LIB_NOTIFIER_H__
 
+#include <sof/bit.h>
 #include <sof/list.h>
 #include <sof/sof.h>
 #include <stdint.h>
@@ -17,6 +18,9 @@
 #define NOTIFIER_TARGET_CORE_LOCAL	NOTIFIER_TARGET_CORE_MASK(cpu_get_id())
 #define NOTIFIER_TARGET_CORE_ALL_MASK	0xFFFFFFFF
 
+/** \brief Notifier flags. */
+#define NOTIFIER_FLAG_AGGREGATE		BIT(0)
+
 enum notify_id {
 	NOTIFIER_ID_CPU_FREQ = 0,		/* struct clock_notify_data * */
 	NOTIFIER_ID_SSP_FREQ,			/* struct clock_notify_data * */
@@ -26,6 +30,8 @@ enum notify_id {
 	NOTIFIER_ID_BUFFER_CONSUME,		/* struct buffer_cb_transact* */
 	NOTIFIER_ID_BUFFER_FREE,		/* struct buffer_cb_free* */
 	NOTIFIER_ID_DMA_COPY,			/* struct dma_cb_data* */
+	NOTIFIER_ID_LL_PRE_RUN,			/* NULL */
+	NOTIFIER_ID_LL_POST_RUN,		/* NULL */
 	NOTIFIER_ID_DMA_IRQ,			/* struct dma_chan_data * */
 	NOTIFIER_ID_COUNT
 };
@@ -51,7 +57,8 @@ struct notify_data {
 struct notify **arch_notify_get(void);
 
 int notifier_register(void *receiver, void *caller, enum notify_id type,
-		void (*cb)(void *arg, enum notify_id type, void *data));
+		      void (*cb)(void *arg, enum notify_id type, void *data),
+		      uint32_t flags);
 void notifier_unregister(void *receiver, void *caller, enum notify_id type);
 void notifier_unregister_all(void *receiver, void *caller);
 

src/lib/notifier.c

@@ -7,6 +7,7 @@
 #include <sof/common.h>
 #include <sof/debug/panic.h>
 #include <sof/drivers/idc.h>
+#include <sof/drivers/interrupt.h>
 #include <sof/lib/alloc.h>
 #include <sof/lib/cache.h>
 #include <sof/lib/cpu.h>
@@ -15,6 +16,7 @@
 #include <sof/list.h>
 #include <sof/sof.h>
 #include <ipc/topology.h>
+#include <stdint.h>
 
 #define trace_notifier(__e, ...) \
 	trace_event(TRACE_CLASS_NOTIFIER, __e, ##__VA_ARGS__)
@@ -30,31 +32,52 @@ struct callback_handle {
 	void *caller;
 	void (*cb)(void *arg, enum notify_id, void *data);
 	struct list_item list;
+	uint32_t num_registrations;
 };
 
 int notifier_register(void *receiver, void *caller, enum notify_id type,
-		      void (*cb)(void *arg, enum notify_id type, void *data))
+		      void (*cb)(void *arg, enum notify_id type, void *data),
+		      uint32_t flags)
 {
 	struct notify *notify = *arch_notify_get();
 	struct callback_handle *handle;
+	uint32_t irq_flags;
+	int ret = 0;
 
 	assert(type >= NOTIFIER_ID_CPU_FREQ && type < NOTIFIER_ID_COUNT);
 
+	irq_local_disable(irq_flags);
+
+	/* Find already registered event of this type */
+	if (flags & NOTIFIER_FLAG_AGGREGATE &&
+	    !list_is_empty(&notify->list[type])) {
+		handle = container_of((&notify->list[type])->next,
+				      struct callback_handle, list);
+		handle->num_registrations++;
+
+		goto out;
+	}
+
 	handle = rzalloc(SOF_MEM_ZONE_SYS_RUNTIME, 0, SOF_MEM_CAPS_RAM,
 			 sizeof(*handle));
 
 	if (!handle) {
 		trace_notifier_error("notifier_register(): callback handle allocation failed.");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto out;
 	}
 
 	handle->receiver = receiver;
 	handle->caller = caller;
 	handle->cb = cb;
+	handle->num_registrations = 1;
 
 	list_item_prepend(&handle->list, &notify->list[type]);
 
-	return 0;
+out:
+	irq_local_enable(irq_flags);
+
+	return ret;
 }
 
 void notifier_unregister(void *receiver, void *caller, enum notify_id type)
@@ -63,9 +86,12 @@ void notifier_unregister(void *receiver, void *caller, enum notify_id type)
 	struct list_item *wlist;
 	struct list_item *tlist;
 	struct callback_handle *handle;
+	uint32_t flags;
 
 	assert(type >= NOTIFIER_ID_CPU_FREQ && type < NOTIFIER_ID_COUNT);
 
+	irq_local_disable(flags);
+
 	/*
 	 * Unregister all matching callbacks
 	 * If receiver is NULL, unregister all callbacks with matching callers
@@ -80,10 +106,14 @@ void notifier_unregister(void *receiver, void *caller, enum notify_id type)
 		handle = container_of(wlist, struct callback_handle, list);
 		if ((!receiver || handle->receiver == receiver) &&
 		    (!caller || handle->caller == caller)) {
-			list_item_del(&handle->list);
-			rfree(handle);
+			if (!--handle->num_registrations) {
+				list_item_del(&handle->list);
+				rfree(handle);
+			}
 		}
 	}
+
+	irq_local_enable(flags);
 }
 
 void notifier_unregister_all(void *receiver, void *caller)