diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
index 845a58d3ddf8..b0963060402e 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
@@ -69,10 +69,6 @@ ENV CUDNN_VERSION=8.0.4.30
 COPY install/ubuntu_cudnn.sh /work/
 RUN /work/ubuntu_cudnn.sh
 
-# update the cuda compatibity package because cd host uses nvidia driver 460
-RUN apt-get update && apt-get install -y cuda-compat-11-2
-RUN ln -sfn /usr/local/cuda-11.2 /usr/local/cuda
-
 # Always last
 ARG USER_ID=0
 ARG GROUP_ID=0
@@ -82,4 +78,3 @@ RUN /work/ubuntu_adduser.sh
 COPY runtime_functions.sh /work/
 
 WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/compat
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu110 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
index db5138b11a61..923c06979d39 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
@@ -32,10 +32,6 @@ COPY install/ubuntu_python.sh /work/
 COPY install/requirements /work/
 RUN /work/ubuntu_python.sh
 
-# update the cuda compatibity package because cd host uses nvidia driver 460
-RUN apt-get update && apt-get install -y cuda-compat-11-2
-RUN ln -sfn /usr/local/cuda-11.2 /usr/local/cuda
-
 # Always last
 ARG USER_ID=0
 ARG GROUP_ID=0
@@ -45,4 +41,3 @@ RUN /work/ubuntu_adduser.sh
 COPY runtime_functions.sh /work/
 
 WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/compat
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu112 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
index 7fe0bceb6128..f45ceed73492 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
@@ -41,4 +41,3 @@ RUN /work/ubuntu_adduser.sh
 COPY runtime_functions.sh /work/
 
 WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/compat