diff --git a/appveyor.yml b/appveyor.yml
index f2954a92e9e..802d43bad4e 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -32,4 +32,6 @@ init:
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
 
 build_script:
- - call ci\msvc-build.bat
+  - git config core.symlinks true
+  - git reset --hard
+  - call ci\msvc-build.bat
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
index 3ff9ef083e5..f225c66268a 100755
--- a/dev/release/run-rat.sh
+++ b/dev/release/run-rat.sh
@@ -34,7 +34,11 @@ $RAT $1 \
   -e status.h \
   -e asan_symbolize.py \
   -e cpplint.py \
+  -e BuildUtils.cmake \
   -e FindPythonLibsNew.cmake \
+  -e FindNumPy.cmake \
+  -e SetupCxxFlags.cmake \
+  -e CompilerInfo.cmake \
   -e pax_global_header \
   -e MANIFEST.in \
   -e __init__.pxd \
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 0d34bcdfa6e..a9db4d877fb 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -23,9 +23,6 @@ project(pyarrow)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
-# Use common cmake modules from Arrow C++ if available
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/../cpp/cmake_modules")
-
 include(CMakeParseArguments)
 
 # Compatibility with CMake 3.1
@@ -34,8 +31,6 @@ if(POLICY CMP0054)
   cmake_policy(SET CMP0054 NEW)
 endif()
 
-set(BUILD_SUPPORT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/build-support")
-
 # Allow "make install" to not depend on all targets.
 #
 # Must be declared in the top-level CMakeLists.txt.
diff --git a/python/cmake_modules/BuildUtils.cmake b/python/cmake_modules/BuildUtils.cmake
new file mode 120000
index 00000000000..e3c98c8fc53
--- /dev/null
+++ b/python/cmake_modules/BuildUtils.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/BuildUtils.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/CompilerInfo.cmake b/python/cmake_modules/CompilerInfo.cmake
new file mode 120000
index 00000000000..559ac3240c3
--- /dev/null
+++ b/python/cmake_modules/CompilerInfo.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/CompilerInfo.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/FindNumPy.cmake b/python/cmake_modules/FindNumPy.cmake
new file mode 120000
index 00000000000..767b320ec7e
--- /dev/null
+++ b/python/cmake_modules/FindNumPy.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/FindNumPy.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/FindPythonLibsNew.cmake b/python/cmake_modules/FindPythonLibsNew.cmake
new file mode 120000
index 00000000000..7087123cb23
--- /dev/null
+++ b/python/cmake_modules/FindPythonLibsNew.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/FindPythonLibsNew.cmake
\ No newline at end of file
diff --git a/python/cmake_modules/SetupCxxFlags.cmake b/python/cmake_modules/SetupCxxFlags.cmake
new file mode 120000
index 00000000000..3121a89926d
--- /dev/null
+++ b/python/cmake_modules/SetupCxxFlags.cmake
@@ -0,0 +1 @@
+../../cpp/cmake_modules/SetupCxxFlags.cmake
\ No newline at end of file