diff --git a/src/arch/xtensa/configs/baytrail_xcc_defconfig b/src/arch/xtensa/configs/baytrail_xcc_defconfig
index 6710e05ca0be..4771aad18b30 100644
--- a/src/arch/xtensa/configs/baytrail_xcc_defconfig
+++ b/src/arch/xtensa/configs/baytrail_xcc_defconfig
@@ -5,4 +5,5 @@ CONFIG_COMP_SEL=n
 CONFIG_COMP_MUX=n
 CONFIG_COMP_FIR=n
 CONFIG_COMP_IIR=n
+CONFIG_COMP_ASRC=n
 CONFIG_OPTIMIZE_FOR_SIZE=y
diff --git a/src/arch/xtensa/configs/cherrytrail_xcc_defconfig b/src/arch/xtensa/configs/cherrytrail_xcc_defconfig
index 8d53bf6586cd..b3cce0603899 100644
--- a/src/arch/xtensa/configs/cherrytrail_xcc_defconfig
+++ b/src/arch/xtensa/configs/cherrytrail_xcc_defconfig
@@ -5,4 +5,5 @@ CONFIG_COMP_SEL=n
 CONFIG_COMP_MUX=n
 CONFIG_COMP_FIR=n
 CONFIG_COMP_IIR=n
+CONFIG_COMP_ASRC=n
 CONFIG_OPTIMIZE_FOR_SIZE=y