diff --git a/ci/docker/java-jni-manylinux-201x.dockerfile b/ci/docker/java-jni-manylinux-201x.dockerfile
index 479f4aa598b..d5adde07eb8 100644
--- a/ci/docker/java-jni-manylinux-201x.dockerfile
+++ b/ci/docker/java-jni-manylinux-201x.dockerfile
@@ -18,6 +18,8 @@
 ARG base
 FROM ${base}
 
+ARG llvm=16
+
 # Install the libraries required by the Gandiva to run
 # Use enable llvm[enable-rtti] in the vcpkg.json to avoid link problems in Gandiva
 RUN vcpkg install \
diff --git a/docker-compose.yml b/docker-compose.yml
index 14eeeeee6e5..e896b91806a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1204,6 +1204,8 @@ services:
       args:
         base: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
         java: 11
+        # GH-XXX: Gandiva throws error with LLVM 17+
+        llvm: 16
       context: .
       dockerfile: ci/docker/java-jni-manylinux-201x.dockerfile
       cache_from: