ARROW-16668: [CI] Add Substrait support to python wheels

ci/scripts/python_wheel_macos_build.sh

@@ -72,6 +72,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_PARQUET:=ON}
 : ${PARQUET_REQUIRE_ENCRYPTION:=ON}
 : ${ARROW_PLASMA:=ON}
+: ${ARROW_SUBSTRAIT:=ON}
 : ${ARROW_S3:=ON}
 : ${ARROW_SIMD_LEVEL:="SSE4_2"}
 : ${ARROW_TENSORFLOW:=ON}
@@ -111,6 +112,7 @@ cmake \
     -DARROW_PLASMA=${ARROW_PLASMA} \
     -DARROW_PYTHON=ON \
     -DARROW_RPATH_ORIGIN=ON \
+    -DARROW_SUBSTRAIT=${ARROW_SUBSTRAIT} \
     -DARROW_S3=${ARROW_S3} \
     -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL} \
     -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \
@@ -151,6 +153,7 @@ export PYARROW_WITH_ORC=${ARROW_ORC}
 export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
 export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
 export PYARROW_WITH_PLASMA=${ARROW_PLASMA}
+export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
 export PYARROW_WITH_S3=${ARROW_S3}
 export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}"
 # PyArrow build configuration

ci/scripts/python_wheel_manylinux_build.sh

@@ -59,6 +59,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_PARQUET:=ON}
 : ${PARQUET_REQUIRE_ENCRYPTION:=ON}
 : ${ARROW_PLASMA:=ON}
+: ${ARROW_SUBSTRAIT:=ON}
 : ${ARROW_S3:=ON}
 : ${ARROW_TENSORFLOW:=ON}
 : ${ARROW_WITH_BROTLI:=ON}
@@ -105,6 +106,7 @@ cmake \
     -DARROW_PLASMA=${ARROW_PLASMA} \
     -DARROW_PYTHON=ON \
     -DARROW_RPATH_ORIGIN=ON \
+    -DARROW_SUBSTRAIT=${ARROW_SUBSTRAIT} \
     -DARROW_S3=${ARROW_S3} \
     -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \
     -DARROW_USE_CCACHE=ON \
@@ -147,6 +149,7 @@ export PYARROW_WITH_ORC=${ARROW_ORC}
 export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
 export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
 export PYARROW_WITH_PLASMA=${ARROW_PLASMA}
+export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
 export PYARROW_WITH_S3=${ARROW_S3}
 # PyArrow build configuration
 export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/tmp/arrow-dist/lib/pkgconfig

ci/scripts/python_wheel_unix_test.sh

@@ -29,6 +29,7 @@ fi
 source_dir=${1}
 
 : ${ARROW_FLIGHT:=ON}
+: ${ARROW_SUBSTRAIT:=ON}
 : ${ARROW_S3:=ON}
 : ${CHECK_IMPORTS:=ON}
 : ${CHECK_UNITTESTS:=ON}
@@ -43,6 +44,7 @@ export PYARROW_TEST_ORC=ON
 export PYARROW_TEST_PANDAS=ON
 export PYARROW_TEST_PARQUET=ON
 export PYARROW_TEST_PLASMA=ON
+export PYARROW_TEST_SUBSTRAIT=${ARROW_SUBSTRAIT}
 export PYARROW_TEST_S3=${ARROW_S3}
 export PYARROW_TEST_TENSORFLOW=ON
 
@@ -73,6 +75,9 @@ import pyarrow.plasma
   if [ "${PYARROW_TEST_FLIGHT}" == "ON" ]; then
     python -c "import pyarrow.flight"
   fi
+  if [ "${PYARROW_TEST_SUBSTRAIT}" == "ON" ]; then
+    python -c "import pyarrow.substrait"
+  fi
 fi
 
 if [ "${CHECK_UNITTESTS}" == "ON" ]; then

ci/scripts/python_wheel_windows_build.bat

@@ -38,6 +38,7 @@ set ARROW_ORC=OFF
 set ARROW_PARQUET=ON
 set PARQUET_REQUIRE_ENCRYPTION=ON
 set ARROW_MIMALLOC=ON
+set ARROW_SUBSTRAIT=ON
 set ARROW_S3=ON
 set ARROW_TENSORFLOW=ON
 set ARROW_WITH_BROTLI=ON
@@ -71,6 +72,7 @@ cmake ^
     -DARROW_PARQUET=%ARROW_PARQUET% ^
     -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^
     -DARROW_PYTHON=ON ^
+    -DARROW_SUBSTRAIT=%ARROW_SUBSTRAIT% ^
     -DARROW_S3=%ARROW_S3% ^
     -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^
     -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^
@@ -105,6 +107,7 @@ set PYARROW_WITH_HDFS=%ARROW_HDFS%
 set PYARROW_WITH_ORC=%ARROW_ORC%
 set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
 set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
+set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
 set PYARROW_WITH_S3=%ARROW_S3%
 set ARROW_HOME=C:\arrow-dist
 

ci/scripts/python_wheel_windows_test.bat

@@ -25,6 +25,7 @@ set PYARROW_TEST_HDFS=ON
 set PYARROW_TEST_ORC=OFF
 set PYARROW_TEST_PARQUET=ON
 set PYARROW_TEST_PLASMA=OFF
+set PYARROW_TEST_SUBSTRAIT=ON
 set PYARROW_TEST_S3=OFF
 set PYARROW_TEST_TENSORFLOW=ON
 
@@ -50,6 +51,7 @@ python -c "import pyarrow.flight" || exit /B 1
 python -c "import pyarrow.fs" || exit /B 1
 python -c "import pyarrow.json" || exit /B 1
 python -c "import pyarrow.parquet" || exit /B 1
+python -c "import pyarrow.substrait" || exit /B 1
 
 @REM Execute unittest
 pytest -r s --pyargs pyarrow || exit /B 1

python/CMakeLists.txt

@@ -545,6 +545,9 @@ if(PYARROW_BUILD_SUBSTRAIT)
   find_package(ArrowSubstrait REQUIRED)
   if(PYARROW_BUNDLE_ARROW_CPP)
     bundle_arrow_lib(ARROW_SUBSTRAIT_SHARED_LIB SO_VERSION ${ARROW_SO_VERSION})
+    if(MSVC)
+      bundle_arrow_import_lib(ARROW_SUBSTRAIT_IMPORT_LIB)
+    endif()
   endif()
 
   set(SUBSTRAIT_LINK_LIBS arrow_substrait_shared)