diff --git a/java/dataset/src/main/cpp/jni_wrapper.cc b/java/dataset/src/main/cpp/jni_wrapper.cc
index 19a43c8d2fa..79efbeb74fc 100644
--- a/java/dataset/src/main/cpp/jni_wrapper.cc
+++ b/java/dataset/src/main/cpp/jni_wrapper.cc
@@ -25,9 +25,8 @@
 #include "arrow/c/helpers.h"
 #include "arrow/dataset/api.h"
 #include "arrow/dataset/file_base.h"
-#include "arrow/filesystem/localfs.h"
+#include "arrow/filesystem/api.h"
 #include "arrow/filesystem/path_util.h"
-#include "arrow/filesystem/s3fs.h"
 #include "arrow/engine/substrait/util.h"
 #include "arrow/engine/substrait/serde.h"
 #include "arrow/engine/substrait/relation.h"
@@ -660,7 +659,9 @@ JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_releaseBuffe
 JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_ensureS3Finalized(
     JNIEnv* env, jobject) {
   JNI_METHOD_START
+#ifdef ARROW_S3
   JniAssertOkOrThrow(arrow::fs::EnsureS3Finalized());
+#endif
   JNI_METHOD_END()
 }