diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 70e9c663bd3..787c01b153e 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -68,10 +68,15 @@
 
 const char* je_arrow_malloc_conf =
     ("oversize_threshold:0"
+#ifdef USE_JEMALLOC_BACKGROUND_THREAD
      ",dirty_decay_ms:1000"
      ",muzzy_decay_ms:1000"
-#ifdef USE_JEMALLOC_BACKGROUND_THREAD
      ",background_thread:true"
+#else
+     // ARROW-6994: return memory immediately to the OS if the
+     // background_thread option isn't available
+     ",dirty_decay_ms:0"
+     ",muzzy_decay_ms:0"
 #endif
      JEMALLOC_DEBUG_OPTIONS);  // NOLINT: whitespace/parens