diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f1b41cfacfb..fc8f74db536 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -980,6 +980,9 @@ endif()
 
 if(ARROW_JNI)
   add_subdirectory(src/jni)
+  if(ARROW_DATASET)
+    add_subdirectory(../java/dataset/src/main/cpp ./java/jni)
+  endif()
 endif()
 
 if(ARROW_GANDIVA)
diff --git a/cpp/src/jni/CMakeLists.txt b/cpp/src/jni/CMakeLists.txt
index 3a5cc7fca80..835f58cd8f0 100644
--- a/cpp/src/jni/CMakeLists.txt
+++ b/cpp/src/jni/CMakeLists.txt
@@ -21,7 +21,3 @@
 if(ARROW_ORC)
   add_subdirectory(orc)
 endif()
-
-if(ARROW_DATASET)
-  add_subdirectory(dataset)
-endif()
diff --git a/cpp/src/jni/dataset/CMakeLists.txt b/java/dataset/src/main/cpp/CMakeLists.txt
similarity index 97%
rename from cpp/src/jni/dataset/CMakeLists.txt
rename to java/dataset/src/main/cpp/CMakeLists.txt
index f3e309b614a..6a0be9b7f58 100644
--- a/cpp/src/jni/dataset/CMakeLists.txt
+++ b/java/dataset/src/main/cpp/CMakeLists.txt
@@ -29,7 +29,7 @@ add_custom_target(arrow_dataset_jni)
 
 set(JNI_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
 
-add_subdirectory(../../../../java/dataset ./java)
+add_subdirectory(../../../../dataset ./java)
 
 set(ARROW_BUILD_STATIC OFF)
 
diff --git a/cpp/src/jni/dataset/jni_util.cc b/java/dataset/src/main/cpp/jni_util.cc
similarity index 99%
rename from cpp/src/jni/dataset/jni_util.cc
rename to java/dataset/src/main/cpp/jni_util.cc
index 8979c327109..e54dcba32bd 100644
--- a/cpp/src/jni/dataset/jni_util.cc
+++ b/java/dataset/src/main/cpp/jni_util.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "jni/dataset/jni_util.h"
+#include "./jni_util.h"
 
 #include <memory>
 #include <mutex>
diff --git a/cpp/src/jni/dataset/jni_util.h b/java/dataset/src/main/cpp/jni_util.h
similarity index 100%
rename from cpp/src/jni/dataset/jni_util.h
rename to java/dataset/src/main/cpp/jni_util.h
diff --git a/cpp/src/jni/dataset/jni_util_test.cc b/java/dataset/src/main/cpp/jni_util_test.cc
similarity index 100%
rename from cpp/src/jni/dataset/jni_util_test.cc
rename to java/dataset/src/main/cpp/jni_util_test.cc
diff --git a/cpp/src/jni/dataset/jni_wrapper.cc b/java/dataset/src/main/cpp/jni_wrapper.cc
similarity index 99%
rename from cpp/src/jni/dataset/jni_wrapper.cc
rename to java/dataset/src/main/cpp/jni_wrapper.cc
index 1e5c7a8aa72..9312825e28d 100644
--- a/cpp/src/jni/dataset/jni_wrapper.cc
+++ b/java/dataset/src/main/cpp/jni_wrapper.cc
@@ -24,7 +24,7 @@
 #include "arrow/filesystem/localfs.h"
 #include "arrow/ipc/api.h"
 #include "arrow/util/iterator.h"
-#include "jni/dataset/jni_util.h"
+#include "./jni_util.h"
 #include "org_apache_arrow_dataset_file_JniWrapper.h"
 #include "org_apache_arrow_dataset_jni_JniWrapper.h"
 #include "org_apache_arrow_dataset_jni_NativeMemoryPool.h"