diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 5cdb8040b8f..8436e65ba80 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -190,8 +190,9 @@ Pass multiple labels by dividing with semicolons")
     "Build Arrow Fuzzing executables"
     OFF)
 
-  option(ARROW_SSE3
-    "Build Arrow with SSE3"
+  # Disable this option to exercise non-SIMD fallbacks
+  option(ARROW_USE_SIMD
+    "Build with SIMD optimizations"
     ON)
 
   option(ARROW_ALTIVEC
@@ -222,10 +223,6 @@ Pass multiple labels by dividing with semicolons")
     "Build the plasma object store java client"
     OFF)
 
-  option(ARROW_USE_SSE
-    "Build with SSE4 optimizations"
-    OFF)
-
   option(ARROW_WITH_BROTLI
     "Build with Brotli compression"
     ON)
diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
index 6f379f9e01d..d239d69a93d 100644
--- a/cpp/cmake_modules/SetupCxxFlags.cmake
+++ b/cpp/cmake_modules/SetupCxxFlags.cmake
@@ -220,8 +220,8 @@ if (CXX_SUPPORTS_ALTIVEC AND ARROW_ALTIVEC)
   set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -maltivec")
 endif()
 
-if (ARROW_USE_SSE)
-  add_definitions(-DARROW_USE_SSE)
+if (ARROW_USE_SIMD)
+  add_definitions(-DARROW_USE_SIMD)
 endif()
 
 if (APPLE)
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index 0119c1f046a..cd3d5b0c58f 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -62,11 +62,6 @@
 #include "arrow/util/type_traits.h"
 #include "arrow/util/visibility.h"
 
-#ifdef ARROW_USE_SSE
-#include "arrow/util/cpu-info.h"
-#include "arrow/util/sse-util.h"
-#endif
-
 namespace arrow {
 
 class Buffer;
diff --git a/cpp/src/arrow/util/cpu-info.cc b/cpp/src/arrow/util/cpu-info.cc
index 3f3488217c6..514b862c6d5 100644
--- a/cpp/src/arrow/util/cpu-info.cc
+++ b/cpp/src/arrow/util/cpu-info.cc
@@ -279,7 +279,7 @@ void CpuInfo::VerifyCpuRequirements() {
 }
 
 bool CpuInfo::CanUseSSE4_2() const {
-#ifdef ARROW_USE_SSE
+#ifdef ARROW_USE_SIMD
   return IsSupported(CpuInfo::SSE4_2);
 #else
   return false;
diff --git a/cpp/src/arrow/util/sse-util.h b/cpp/src/arrow/util/sse-util.h
index 0ff1ff3ae35..edaf6686be5 100644
--- a/cpp/src/arrow/util/sse-util.h
+++ b/cpp/src/arrow/util/sse-util.h
@@ -24,6 +24,8 @@
 #undef ARROW_HAVE_SSE2
 #undef ARROW_HAVE_SSE4_2
 
+#ifdef ARROW_USE_SIMD
+
 // MSVC x86-64
 
 #if (defined(_M_AMD64) || defined(_M_X64))
@@ -44,8 +46,6 @@
 #include <nmmintrin.h>
 #endif
 
-#if defined(ARROW_USE_SSE) && !defined(ARROW_HAVE_SSE2)
-#error "ARROW_USE_SSE enabled but no intrinsics available"
 #endif
 
 namespace arrow {