diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/docker/install/ubuntu_caffe.sh
index 503b57f2ed7f..40ccc3c3015e 100755
--- a/ci/docker/install/ubuntu_caffe.sh
+++ b/ci/docker/install/ubuntu_caffe.sh
@@ -55,5 +55,6 @@ ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linu
 
 make all pycaffe -j$(nproc)
 
-cd python
-for req in $(cat requirements.txt); do pip3 install $req; done
+# Do not install caffe python dependencies, as it overwrites our specified versions in ci/docker/install/requirements
+#cd python
+#pip3 install -r requirements.txt
diff --git a/rat-excludes b/rat-excludes
index 56d6162142da..5fc635ce39c9 100644
--- a/rat-excludes
+++ b/rat-excludes
@@ -120,11 +120,14 @@ CODEOWNERS
 
 # Incorporated third-party source files that carry its own license, captured in licenses/
 pool.h
+pool.cuh
 erfinv-inl.h
 im2col.cuh
 im2col.h
 deformable_im2col.cuh
 deformable_im2col.h
+modulated_deformable_im2col.cuh
+modulated_deformable_im2col.h
 FindCUDAToolkit.cmake
 select_compute_arch.cmake
 
diff --git a/src/operator/contrib/nn/modulated_deformable_im2col.cuh b/src/operator/contrib/nn/modulated_deformable_im2col.cuh
index 16d9cef46d4e..74fc9944ecaa 100644
--- a/src/operator/contrib/nn/modulated_deformable_im2col.cuh
+++ b/src/operator/contrib/nn/modulated_deformable_im2col.cuh
@@ -1,22 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
 /*!
  ******************* BEGIN Caffe Copyright Notice and Disclaimer ****************
  *
diff --git a/src/operator/contrib/nn/modulated_deformable_im2col.h b/src/operator/contrib/nn/modulated_deformable_im2col.h
index b50eb13a8122..560856bdc478 100644
--- a/src/operator/contrib/nn/modulated_deformable_im2col.h
+++ b/src/operator/contrib/nn/modulated_deformable_im2col.h
@@ -1,22 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
 /*!
  ******************* BEGIN Caffe Copyright Notice and Disclaimer ****************
  *
diff --git a/src/operator/nn/pool.cuh b/src/operator/nn/pool.cuh
index e771b3681573..3810e2ccf390 100644
--- a/src/operator/nn/pool.cuh
+++ b/src/operator/nn/pool.cuh
@@ -1,22 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
 /*!
  ******************* BEGIN Caffe Copyright Notice and Disclaimer ****************
  *
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 24764fb1ae84..679d2aabd7d8 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -5,5 +5,5 @@ nose
 nose-timer
 ipython
 numpy>1.16.0,<1.19.0  # Restrict numpy version to < 1.19.0 due to https://github.com/apache/incubator-mxnet/issues/18600
-scipy
+scipy<1.7.0 # Restrict scipy version due to https://github.com/apache/incubator-mxnet/issues/20389
 onnxruntime
diff --git a/tools/license_header.py b/tools/license_header.py
index ccc45bdb3be1..e46934459ea9 100755
--- a/tools/license_header.py
+++ b/tools/license_header.py
@@ -95,9 +95,12 @@
 
                # Licensed under Caffe header
                'src/operator/nn/pool.h',
+               'src/operator/nn/pool.cuh',
                'src/operator/contrib/psroi_pooling-inl.h',
                'src/operator/contrib/nn/deformable_im2col.h',
                'src/operator/contrib/nn/deformable_im2col.cuh',
+               'src/operator/contrib/nn/modulated_deformable_im2col.h',
+               'src/operator/contrib/nn/modulated_deformable_im2col.cuh',
                'src/operator/nn/im2col.h',
                'src/operator/nn/im2col.cuh',