diff --git a/sound/soc/sof/sof-audio.c b/sound/soc/sof/sof-audio.c
index 1338c1d9984904..7d0dd62bf6b463 100644
--- a/sound/soc/sof/sof-audio.c
+++ b/sound/soc/sof/sof-audio.c
@@ -736,8 +736,8 @@ int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm,
 	struct snd_soc_dapm_widget *widget;
 	int i, ret;
 
-	/* nothing to set up */
-	if (!list)
+	/* nothing to set up or setup has been already done */
+	if (!list || spcm->setup_done[dir])
 		return 0;
 
 	/* Set up is used to send the IPC to the DSP to create the widget */
@@ -794,6 +794,8 @@ int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm,
 		}
 	}
 
+	spcm->setup_done[dir] = true;
+
 	return 0;
 
 widget_free:
@@ -811,12 +813,13 @@ int sof_widget_list_free(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, int
 	int ret;
 
 	/* nothing to free */
-	if (!list)
+	if (!list || !spcm->setup_done[dir])
 		return 0;
 
 	/* send IPC to free widget in the DSP */
 	ret = sof_walk_widgets_in_order(sdev, spcm, NULL, NULL, dir, SOF_WIDGET_FREE);
 
+	spcm->setup_done[dir] = false;
 	pipeline_list->count = 0;
 
 	return ret;
diff --git a/sound/soc/sof/sof-audio.h b/sound/soc/sof/sof-audio.h
index a72de717f2d47e..00089df68b668f 100644
--- a/sound/soc/sof/sof-audio.h
+++ b/sound/soc/sof/sof-audio.h
@@ -356,6 +356,7 @@ struct snd_sof_pcm {
 	struct snd_pcm_hw_params params[2];
 	struct snd_sof_platform_stream_params platform_params[2];
 	bool prepared[2]; /* PCM_PARAMS set successfully */
+	bool setup_done[2]; /* the setup of the SOF PCM device is done */
 	bool pending_stop[2]; /* only used if (!pcm_ops->platform_stop_during_hw_free) */
 
 	/* Must be last - ends in a flex-array member. */