diff --git a/cmake/modules/contrib/EthosU.cmake b/cmake/modules/contrib/EthosU.cmake
index f66e4af55626..f4379753f9e2 100644
--- a/cmake/modules/contrib/EthosU.cmake
+++ b/cmake/modules/contrib/EthosU.cmake
@@ -20,4 +20,10 @@ if(USE_ETHOSU)
        CONFIGURE_DEPENDS src/relay/backend/contrib/ethosu/*
        CONFIGURE_DEPENDS src/contrib/ethosu/cascader/*)
   list(APPEND COMPILER_SRCS ${COMPILER_ETHOSU_SRCS})
+else()
+  # Keeping just utils.cc because it has Object definitions
+  # used by python side
+  file(GLOB COMPILER_ETHOSU_SRCS
+          CONFIGURE_DEPENDS src/relay/backend/contrib/ethosu/utils.cc)
+  list(APPEND COMPILER_SRCS ${COMPILER_ETHOSU_SRCS})
 endif(USE_ETHOSU)
\ No newline at end of file