diff --git a/cmake/modules/Vulkan.cmake b/cmake/modules/Vulkan.cmake
index 4df8986c800c..095790f08547 100644
--- a/cmake/modules/Vulkan.cmake
+++ b/cmake/modules/Vulkan.cmake
@@ -26,16 +26,11 @@ IF USE_VULKAN)
 tvm_option(USE_VULKAN_VALIDATION "Enable Vulkan API validation layers" OFF
   IF USE_VULKAN)
 
-if(Vulkan_FOUND)
-  # always set the includedir
-  # avoid global retrigger of cmake
-  include_directories(SYSTEM ${Vulkan_INCLUDE_DIRS})
-endif(Vulkan_FOUND)
-
 if(USE_VULKAN)
   if(NOT Vulkan_FOUND)
     message(FATAL_ERROR "Cannot find Vulkan, USE_VULKAN=" ${USE_VULKAN})
   endif()
+  include_directories(SYSTEM ${Vulkan_INCLUDE_DIRS})
   message(STATUS "Build with Vulkan support")
   file(GLOB RUNTIME_VULKAN_SRCS src/runtime/vulkan/vulkan.cc)
   file(GLOB COMPILER_VULKAN_SRCS src/target/spirv/*.cc)