diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 611402315eb..ac8ac2fb353 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -1406,7 +1406,6 @@ if(ARROW_JEMALLOC)
                                    INTERFACE_INCLUDE_DIRECTORIES
                                    "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src")
   add_dependencies(jemalloc::jemalloc jemalloc_ep)
-  add_dependencies(toolchain jemalloc_ep)
 endif()
 
 # ----------------------------------------------------------------------
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index f3b9a37c3a6..1c613a09db3 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -145,6 +145,14 @@ set(ARROW_SRCS
     util/utf8.cc
     vendored/datetime/tz.cpp)
 
+if(ARROW_JEMALLOC)
+  if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
+    set_source_files_properties(memory_pool.cc PROPERTIES OBJECT_DEPENDS jemalloc_ep)
+  else()
+    add_dependencies(arrow_dependencies jemalloc_ep)
+  endif()
+endif()
+
 if(ARROW_JSON)
   add_subdirectory(json)
   set(ARROW_SRCS
@@ -205,10 +213,6 @@ if(ARROW_CUDA)
   add_subdirectory(gpu)
 endif()
 
-if(ARROW_JEMALLOC AND JEMALLOC_VENDORED)
-  add_dependencies(arrow_dependencies jemalloc::jemalloc)
-endif()
-
 if(ARROW_WITH_BROTLI)
   add_definitions(-DARROW_WITH_BROTLI)
   set(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS})