diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 58d669e94b0..b064ada54ce 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -47,15 +47,27 @@ env:
 jobs:
 
   debian:
-    name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
+    name: ${{ matrix.title }}
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        jdk: [11]
-        maven: [3.6.2]
+        jdk: [8, 11, 17]
+        include:
+        - jdk: 8
+          title: AMD64 Debian 9 Java JDK 8 Maven 3.5.4
+          maven: 3.5.4
+          image: debian-java
+        - jdk: 11
+          title: AMD64 Debian 9 Java JDK 11 Maven 3.6.2
+          maven: 3.6.2
+          image: debian-java
+        - jdk: 17
+          title: AMD64 Oracle Linux Server 8.5 Java JDK 17 Maven 3.8.5
+          maven: 3.8.5
+          image: oracle-java
     env:
       JDK: ${{ matrix.jdk }}
       MAVEN: ${{ matrix.maven }}
@@ -78,11 +90,11 @@ jobs:
       - name: Setup Archery
         run: pip install -e dev/archery[docker]
       - name: Execute Docker Build
-        run: archery docker run debian-java
+        run: archery docker run ${{ matrix.image }}
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
         continue-on-error: true
-        run: archery docker push debian-java
+        run: archery docker push ${{ matrix.image }}
 
   macos:
     name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }}
diff --git a/ci/docker/debian-9-java.dockerfile b/ci/docker/debian-9-java.dockerfile
deleted file mode 100644
index 2cc36e3eafb..00000000000
--- a/ci/docker/debian-9-java.dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-ARG arch=amd64
-ARG jdk=8
-ARG maven=3.5.4
-FROM ${arch}/maven:${maven}-jdk-${jdk}
-
-ENV ARROW_JAVA_SHADE_FLATBUFS=ON
-
-# TODO(kszucs):
-# 1. add the files required to install the dependencies to .dockerignore
-# 2. copy these files to their appropriate path
-# 3. download and compile the dependencies
diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile
index ddfa72e1774..6673b001d71 100644
--- a/ci/docker/linux-apt-jni.dockerfile
+++ b/ci/docker/linux-apt-jni.dockerfile
@@ -15,8 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG base
-FROM ${base}
+ARG arch=amd64
+ARG jdk=8
+ARG maven=3.5.4
+FROM ${arch}/maven:${maven}-jdk-${jdk}
 
 # pipefail is enabled for proper error detection in the `wget | apt-key add`
 # step
diff --git a/docker-compose.yml b/docker-compose.yml
index 8cae9a5ce1c..d7d5f6e3cfe 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -119,11 +119,12 @@ x-hierarchy:
   - debian-go:
     - debian-go-cgo
     - debian-go-cgo-python
-  - debian-java:
-    - debian-java-jni
+  - debian-java
+  - debian-java-jni
   - debian-js
   - fedora-cpp:
     - fedora-python
+  - oracle-java
   - python-sdist
   - ubuntu-cpp:
     - ubuntu-c-glib:
@@ -1440,16 +1441,10 @@ services:
     # Usage:
     #   docker-compose build debian-java
     #   docker-compose run debian-java
-    image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}
-    build:
-      context: .
-      dockerfile: ci/docker/debian-9-java.dockerfile
-      cache_from:
-        - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}
-      args:
-        arch: ${ARCH}
-        jdk: ${JDK}
-        maven: ${MAVEN}
+    # Parameters:
+    #   MAVEN: 3.5.4, 3.6.2
+    #   JDK: 8, 11
+    image: ${ARCH}/maven:${MAVEN}-jdk-${JDK}
     shm_size: *shm-size
     volumes: &java-volumes
       - .:/arrow:delegated
@@ -1472,7 +1467,6 @@ services:
       cache_from:
         - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
       args:
-        base: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}
         llvm: ${LLVM}
     shm_size: *shm-size
     environment:
@@ -1488,6 +1482,18 @@ services:
         /arrow/ci/scripts/java_build.sh /arrow /build &&
         /arrow/ci/scripts/java_test.sh /arrow /build"
 
+  oracle-java:
+    # Usage:
+    #   docker-compose build oracle-java
+    #   docker-compose run oracle-java
+    # Parameters:
+    #   MAVEN: 3.8.5
+    #   JDK: 17
+    image: ${ARCH}/maven:${MAVEN}-openjdk-${JDK}
+    shm_size: *shm-size
+    volumes: *java-volumes
+    command: *java-command
+
   ############################## Integration ##################################
 
   conda-integration:
diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst
index d675b8feac5..1bf10b24aae 100644
--- a/docs/source/developers/java/building.rst
+++ b/docs/source/developers/java/building.rst
@@ -32,7 +32,7 @@ Arrow Java uses the `Maven <https://maven.apache.org/>`_ build system.
 
 Building requires:
 
-* JDK 8, 9, 10, 11, 17, or 18, but only JDK 11 is tested in CI.
+* JDK 8, 9, 10, 11, 17, or 18, but only JDK 8, 11 and 17 are tested in CI.
 * Maven 3+
 
 Building