ARROW-15893: [CI][Python] Add python minimal builds to nightly builds

dev/tasks/macros.jinja

@@ -25,15 +25,15 @@ on:
       - "*-github-*"
 {% endmacro %}
 
-{%- macro github_checkout_arrow(fetch_depth=1) -%}
+{%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive") -%}
   - name: Checkout Arrow
     uses: actions/checkout@v3
     with:
       fetch-depth: {{ fetch_depth }}
       path: arrow
       repository: {{ arrow.github_repo }}
       ref: {{ arrow.head }}
-      submodules: recursive
+      submodules: {{ submodules }}
 {% endmacro %}
 
 {%- macro github_login_dockerhub() -%}

dev/tasks/python-minimal-build/github.linux.yml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+jobs:
+  test:
+    name: |
+      Docker Python Minimal Build {{ flags|default("") }} {{ image }} {{ command|default("") }}
+    runs-on: ubuntu-latest
+  {% if env is defined %}
+    env:
+    {% for key, value in env.items() %}
+      {{ key }}: "{{ value }}"
+    {% endfor %}
+  {% endif %}
+    steps:
+      {{ macros.github_checkout_arrow(submodules=false)|indent }}
+
+      - name: Run minimal build example
+        run: |
+          cd arrow/python/examples/{{ type }}
+          docker-compose run --rm {{ image }}

dev/tasks/tasks.yml

@@ -108,6 +108,9 @@ groups:
   example-cpp:
     - example-*cpp*
 
+  example-python:
+    - example-*python*
+
   fuzz:
     - test-*fuzz*
 
@@ -203,6 +206,17 @@ tasks:
         -e ARROW_CONAN_WITH_LZ4=True
       image: conan
 
+  ########################### Python Minimal ############################
+
+{% for kind in ["fedora-conda", "ubuntu-venv"] %}
+  example-python-minimal-build-{{ kind }}:
+    ci: github
+    template: python-minimal-build/github.linux.yml
+    params:
+      type: minimal_build
+      image: minimal-{{ kind }}
+{% endfor %}
+
   ############################## Conda Linux ############################
 
   conda-clean:

python/examples/minimal_build/README.md

@@ -34,10 +34,21 @@ docker build -t arrow_fedora_minimal -f Dockerfile.fedora .
 Then build PyArrow with conda or pip, respectively:
 ```
 # With pip
-docker run --rm -t -i -v $PWD:/io arrow_fedora_minimal /io/build_venv.sh
+docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_fedora_minimal /io/build_venv.sh
 
 # With conda
-docker run --rm -t -i -v $PWD:/io arrow_fedora_minimal /io/build_conda.sh
+docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_fedora_minimal /io/build_conda.sh
+```
+
+Alternatively you can use [Docker Compose][docker-compose] to build and run using:
+```
+docker-compose build
+
+# With conda
+docker-compose run --rm minimal-fedora-conda
+
+# With pip
+docker-compose run --rm minimal-fedora-venv
 ```
 
 ## Ubuntu 20.04
@@ -50,10 +61,21 @@ docker build -t arrow_ubuntu_minimal -f Dockerfile.ubuntu .
 Then build PyArrow with conda or pip, respectively:
 ```
 # With pip
-docker run --rm -t -i -v $PWD:/io arrow_ubuntu_minimal /io/build_venv.sh
+docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_ubuntu_minimal /io/build_venv.sh
 
 # With conda
-docker run --rm -t -i -v $PWD:/io arrow_ubuntu_minimal /io/build_conda.sh
+docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_ubuntu_minimal /io/build_conda.sh
+```
+
+Alternatively you can use [Docker Compose][docker-compose] to build and run using:
+```
+docker-compose build
+
+# With conda
+docker-compose run --rm minimal-ubuntu-conda
+
+# With pip
+docker-compose run --rm minimal-ubuntu-venv
 ```
 
 ## Building on Fedora - Podman and SELinux
@@ -69,5 +91,7 @@ podman build -t arrow_fedora_minimal -f Dockerfile.fedora
 Then build PyArrow with pip:
 ```
 # With pip
-podman run --rm -i -v $PWD:/io:Z -t arrow_fedora_minimal /io/build_venv.sh
+podman run --rm -i -v $PWD:/io:Z -v $PWD/../../..:/arrow:Z -t arrow_fedora_minimal /io/build_venv.sh
 ```
+
+[docker-compose]: https://docs.docker.com/compose/

python/examples/minimal_build/build_conda.sh

@@ -28,7 +28,7 @@ CPP_BUILD_DIR=$HOME/arrow-cpp-build
 ARROW_ROOT=/arrow
 PYTHON=3.10
 
-git clone --depth=100 https://github.com/apache/arrow.git /arrow
+git config --global --add safe.directory $ARROW_ROOT
 
 #----------------------------------------------------------------------
 # Run these only once
@@ -79,14 +79,12 @@ cmake -GNinja \
       -DCMAKE_BUILD_TYPE=DEBUG \
       -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
       -DCMAKE_INSTALL_LIBDIR=lib \
-      -DARROW_FLIGHT=ON \
       -DARROW_WITH_BZ2=ON \
       -DARROW_WITH_ZLIB=ON \
       -DARROW_WITH_ZSTD=ON \
       -DARROW_WITH_LZ4=ON \
       -DARROW_WITH_SNAPPY=ON \
       -DARROW_WITH_BROTLI=ON \
-      -DARROW_PARQUET=ON \
       -DARROW_PLASMA=ON \
       -DARROW_PYTHON=ON \
       $ARROW_ROOT/cpp
@@ -103,17 +101,10 @@ rm -rf build/  # remove any pesky pre-existing build directory
 
 export PYARROW_BUILD_TYPE=Debug
 export PYARROW_CMAKE_GENERATOR=Ninja
-export PYARROW_WITH_FLIGHT=1
-export PYARROW_WITH_PARQUET=1
 
 # You can run either "develop" or "build_ext --inplace". Your pick
 
 # python setup.py build_ext --inplace
 python setup.py develop
 
-# git submodules are required for unit tests
-git submodule update --init
-export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data"
-export ARROW_TEST_DATA="$ARROW_ROOT/testing/data"
-
 py.test pyarrow

python/examples/minimal_build/build_venv.sh

@@ -25,14 +25,14 @@ WORKDIR=${WORKDIR:-$HOME}
 MINICONDA=$WORKDIR/miniconda-for-arrow
 LIBRARY_INSTALL_DIR=$WORKDIR/local-libs
 CPP_BUILD_DIR=$WORKDIR/arrow-cpp-build
-ARROW_ROOT=$WORKDIR/arrow
+ARROW_ROOT=/arrow
 export ARROW_HOME=$WORKDIR/dist
 export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
 
 python3 -m venv $WORKDIR/venv
 source $WORKDIR/venv/bin/activate
 
-git clone --depth=100 https://github.com/apache/arrow.git $ARROW_ROOT
+git config --global --add safe.directory $ARROW_ROOT
 
 pip install -r $ARROW_ROOT/python/requirements-build.txt
 
@@ -53,7 +53,6 @@ cmake -GNinja \
       -DARROW_WITH_LZ4=ON \
       -DARROW_WITH_SNAPPY=ON \
       -DARROW_WITH_BROTLI=ON \
-      -DARROW_PARQUET=ON \
       -DARROW_PYTHON=ON \
       $ARROW_ROOT/cpp
 
@@ -69,18 +68,12 @@ rm -rf build/  # remove any pesky pre-existing build directory
 
 export PYARROW_BUILD_TYPE=Debug
 export PYARROW_CMAKE_GENERATOR=Ninja
-export PYARROW_WITH_PARQUET=1
 
 # You can run either "develop" or "build_ext --inplace". Your pick
 
 # python setup.py build_ext --inplace
 python setup.py develop
 
-# git submodules are required for unit tests
-git submodule update --init
-export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data"
-export ARROW_TEST_DATA="$ARROW_ROOT/testing/data"
-
 pip install -r $ARROW_ROOT/python/requirements-test.txt
 
 py.test pyarrow

python/examples/minimal_build/docker-compose.yml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3.5'
+
+services:
+  minimal-fedora-conda:
+    build:
+      context: .
+      dockerfile: Dockerfile.fedora
+    volumes: &minimal-volumes
+      - ../../../:/arrow:delegated
+      - .:/io:delegated
+    command:
+      - "/io/build_conda.sh"
+
+  minimal-fedora-venv:
+    build:
+      context: .
+      dockerfile: Dockerfile.fedora
+    volumes: *minimal-volumes
+    command:
+      - "/io/build_venv.sh"
+
+  minimal-ubuntu-conda:
+    build:
+      context: .
+      dockerfile: Dockerfile.ubuntu
+    volumes: *minimal-volumes
+    command:
+      - "/io/build_conda.sh"
+
+  minimal-ubuntu-venv:
+    build:
+      context: .
+      dockerfile: Dockerfile.ubuntu
+    volumes: *minimal-volumes
+    command:
+      - "/io/build_venv.sh"