GH-39352: [FS][Azure] Enable azure in builds

ci/docker/python-wheel-manylinux.dockerfile

@@ -82,6 +82,7 @@ RUN vcpkg install \
         --clean-after-build \
         --x-install-root=${VCPKG_ROOT}/installed \
         --x-manifest-root=/arrow/ci/vcpkg \
+        --x-feature=azure \ 
         --x-feature=flight \
         --x-feature=gcs \
         --x-feature=json \

ci/scripts/cpp_build.sh

@@ -54,6 +54,7 @@ if [ "${GITHUB_ACTIONS:-false}" = "true" ]; then
 fi
 
 if [ "${ARROW_ENABLE_THREADING:-ON}" = "OFF" ]; then
+  ARROW_AZURE=OFF
   ARROW_FLIGHT=OFF
   ARROW_FLIGHT_SQL=OFF
   ARROW_GCS=OFF

ci/scripts/python_build.sh

@@ -55,6 +55,7 @@ export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
 export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
 
 export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
+export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
 export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
 export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
 export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}

ci/scripts/python_test.sh

@@ -37,6 +37,7 @@ export ARROW_DEBUG_MEMORY_POOL=trap
 
 # By default, force-test all optional components
 : ${PYARROW_TEST_ACERO:=${ARROW_ACERO:-ON}}
+: ${PYARROW_TEST_AZURE:=${ARROW_AZURE:-ON}}
 : ${PYARROW_TEST_CUDA:=${ARROW_CUDA:-ON}}
 : ${PYARROW_TEST_DATASET:=${ARROW_DATASET:-ON}}
 : ${PYARROW_TEST_FLIGHT:=${ARROW_FLIGHT:-ON}}

ci/scripts/python_wheel_macos_build.sh

@@ -63,6 +63,7 @@ pip install "delocate>=0.10.3"
 
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_ACERO:=ON}
+: ${ARROW_AZURE:=ON}
 : ${ARROW_DATASET:=ON}
 : ${ARROW_FLIGHT:=ON}
 : ${ARROW_GANDIVA:=OFF}
@@ -95,6 +96,7 @@ pushd ${build_dir}/build
 
 cmake \
     -DARROW_ACERO=${ARROW_ACERO} \
+    -DARROW_AZURE=${ARROW_AZURE} \
     -DARROW_BUILD_SHARED=ON \
     -DARROW_BUILD_STATIC=OFF \
     -DARROW_BUILD_TESTS=OFF \
@@ -148,6 +150,7 @@ export PYARROW_BUNDLE_ARROW_CPP=1
 export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
 export PYARROW_INSTALL_TESTS=1
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
+export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}
 export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT}
 export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}

ci/scripts/python_wheel_manylinux_build.sh

@@ -49,6 +49,7 @@ rm -rf /arrow/python/pyarrow/*.so.*
 
 echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_ACERO:=ON}
+: ${ARROW_AZURE:=ON}
 : ${ARROW_DATASET:=ON}
 : ${ARROW_FLIGHT:=ON}
 : ${ARROW_GANDIVA:=OFF}
@@ -87,6 +88,7 @@ pushd /tmp/arrow-build
 
 cmake \
     -DARROW_ACERO=${ARROW_ACERO} \
+    -DARROW_AZURE=${ARROW_AZURE} \
     -DARROW_BUILD_SHARED=ON \
     -DARROW_BUILD_STATIC=OFF \
     -DARROW_BUILD_TESTS=OFF \
@@ -141,6 +143,7 @@ export PYARROW_BUNDLE_ARROW_CPP=1
 export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
 export PYARROW_INSTALL_TESTS=1
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
+export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}
 export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT}
 export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}

ci/scripts/python_wheel_unix_test.sh

@@ -28,15 +28,17 @@ fi
 
 source_dir=${1}
 
+: ${ARROW_AZURE:=ON}
 : ${ARROW_FLIGHT:=ON}
-: ${ARROW_SUBSTRAIT:=ON}
-: ${ARROW_S3:=ON}
 : ${ARROW_GCS:=ON}
+: ${ARROW_S3:=ON}
+: ${ARROW_SUBSTRAIT:=ON}
 : ${CHECK_IMPORTS:=ON}
 : ${CHECK_UNITTESTS:=ON}
 : ${INSTALL_PYARROW:=ON}
 
 export PYARROW_TEST_ACERO=ON
+export PYARROW_TEST_AZURE=${ARROW_AZURE}
 export PYARROW_TEST_CYTHON=OFF
 export PYARROW_TEST_DATASET=ON
 export PYARROW_TEST_FLIGHT=${ARROW_FLIGHT}

ci/vcpkg/vcpkg.json

@@ -105,6 +105,16 @@
         }
       ]
     },
+    "azure": {
+      "description": "Azure blob storage support",
+      "dependencies": [
+        "azure-core-cpp",
+        "azure-identity-cpp",
+        "azure-storage-blobs-cpp",
+        "azure-storage-common-cpp",
+        "azure-storage-files-datalake-cpp"
+      ]
+    },
     "orc": {
       "description": "ORC support",
       "dependencies": [

dev/tasks/python-wheels/github.osx.amd64.yml

@@ -85,6 +85,7 @@ jobs:
             --clean-after-build \
             --x-install-root=${VCPKG_ROOT}/installed \
             --x-manifest-root=arrow/ci/vcpkg \
+            --x-feature=azure \
             --x-feature=flight \
             --x-feature=gcs \
             --x-feature=json \

dev/tasks/python-wheels/github.osx.arm64.yml

@@ -71,6 +71,7 @@ jobs:
             --clean-after-build \
             --x-install-root=${VCPKG_ROOT}/installed \
             --x-manifest-root=arrow/ci/vcpkg \
+            --x-feature=azure \
             --x-feature=flight \
             --x-feature=gcs \
             --x-feature=json \