diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index bd65f6488c9..0ecf2d44e0b 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -25,7 +25,7 @@ on: - "*-github-*" {% endmacro %} -{%- macro github_checkout_arrow(fetch_depth=1) -%} +{%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive") -%} - name: Checkout Arrow uses: actions/checkout@v3 with: @@ -33,7 +33,7 @@ on: path: arrow repository: {{ arrow.github_repo }} ref: {{ arrow.head }} - submodules: recursive + submodules: {{ submodules }} {% endmacro %} {%- macro github_login_dockerhub() -%} diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml new file mode 100644 index 00000000000..887197d17bf --- /dev/null +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + test: + name: | + Docker Python Minimal Build {{ flags|default("") }} {{ image }} {{ command|default("") }} + runs-on: ubuntu-latest + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + {% endif %} + steps: + {{ macros.github_checkout_arrow(submodules=false)|indent }} + + - name: Run minimal build example + run: | + cd arrow/python/examples/{{ type }} + docker-compose run --rm {{ image }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 93dd5229848..717c2f581b0 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -108,6 +108,9 @@ groups: example-cpp: - example-*cpp* + example-python: + - example-*python* + fuzz: - test-*fuzz* @@ -203,6 +206,17 @@ tasks: -e ARROW_CONAN_WITH_LZ4=True image: conan + ########################### Python Minimal ############################ + +{% for kind in ["fedora-conda", "ubuntu-venv"] %} + example-python-minimal-build-{{ kind }}: + ci: github + template: python-minimal-build/github.linux.yml + params: + type: minimal_build + image: minimal-{{ kind }} +{% endfor %} + ############################## Conda Linux ############################ conda-clean: diff --git a/python/examples/minimal_build/README.md b/python/examples/minimal_build/README.md index 57d08483861..d5cda25ce2b 100644 --- a/python/examples/minimal_build/README.md +++ b/python/examples/minimal_build/README.md @@ -34,10 +34,21 @@ docker build -t arrow_fedora_minimal -f Dockerfile.fedora . Then build PyArrow with conda or pip, respectively: ``` # With pip -docker run --rm -t -i -v $PWD:/io arrow_fedora_minimal /io/build_venv.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_fedora_minimal /io/build_venv.sh # With conda -docker run --rm -t -i -v $PWD:/io arrow_fedora_minimal /io/build_conda.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_fedora_minimal /io/build_conda.sh +``` + +Alternatively you can use [Docker Compose][docker-compose] to build and run using: +``` +docker-compose build + +# With conda +docker-compose run --rm minimal-fedora-conda + +# With pip +docker-compose run --rm minimal-fedora-venv ``` ## Ubuntu 20.04 @@ -50,10 +61,21 @@ docker build -t arrow_ubuntu_minimal -f Dockerfile.ubuntu . Then build PyArrow with conda or pip, respectively: ``` # With pip -docker run --rm -t -i -v $PWD:/io arrow_ubuntu_minimal /io/build_venv.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_ubuntu_minimal /io/build_venv.sh # With conda -docker run --rm -t -i -v $PWD:/io arrow_ubuntu_minimal /io/build_conda.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_ubuntu_minimal /io/build_conda.sh +``` + +Alternatively you can use [Docker Compose][docker-compose] to build and run using: +``` +docker-compose build + +# With conda +docker-compose run --rm minimal-ubuntu-conda + +# With pip +docker-compose run --rm minimal-ubuntu-venv ``` ## Building on Fedora - Podman and SELinux @@ -69,5 +91,7 @@ podman build -t arrow_fedora_minimal -f Dockerfile.fedora Then build PyArrow with pip: ``` # With pip -podman run --rm -i -v $PWD:/io:Z -t arrow_fedora_minimal /io/build_venv.sh +podman run --rm -i -v $PWD:/io:Z -v $PWD/../../..:/arrow:Z -t arrow_fedora_minimal /io/build_venv.sh ``` + +[docker-compose]: https://docs.docker.com/compose/ diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 5ab0b559c62..61378b9feca 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -28,7 +28,7 @@ CPP_BUILD_DIR=$HOME/arrow-cpp-build ARROW_ROOT=/arrow PYTHON=3.10 -git clone --depth=100 https://github.com/apache/arrow.git /arrow +git config --global --add safe.directory $ARROW_ROOT #---------------------------------------------------------------------- # Run these only once @@ -79,14 +79,12 @@ cmake -GNinja \ -DCMAKE_BUILD_TYPE=DEBUG \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_FLIGHT=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=ON \ -DARROW_PLASMA=ON \ -DARROW_PYTHON=ON \ $ARROW_ROOT/cpp @@ -103,17 +101,10 @@ rm -rf build/ # remove any pesky pre-existing build directory export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -export PYARROW_WITH_FLIGHT=1 -export PYARROW_WITH_PARQUET=1 # You can run either "develop" or "build_ext --inplace". Your pick # python setup.py build_ext --inplace python setup.py develop -# git submodules are required for unit tests -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" - py.test pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 828b72c9bcc..56dd54abbae 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -25,14 +25,14 @@ WORKDIR=${WORKDIR:-$HOME} MINICONDA=$WORKDIR/miniconda-for-arrow LIBRARY_INSTALL_DIR=$WORKDIR/local-libs CPP_BUILD_DIR=$WORKDIR/arrow-cpp-build -ARROW_ROOT=$WORKDIR/arrow +ARROW_ROOT=/arrow export ARROW_HOME=$WORKDIR/dist export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH python3 -m venv $WORKDIR/venv source $WORKDIR/venv/bin/activate -git clone --depth=100 https://github.com/apache/arrow.git $ARROW_ROOT +git config --global --add safe.directory $ARROW_ROOT pip install -r $ARROW_ROOT/python/requirements-build.txt @@ -53,7 +53,6 @@ cmake -GNinja \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=ON \ -DARROW_PYTHON=ON \ $ARROW_ROOT/cpp @@ -69,18 +68,12 @@ rm -rf build/ # remove any pesky pre-existing build directory export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -export PYARROW_WITH_PARQUET=1 # You can run either "develop" or "build_ext --inplace". Your pick # python setup.py build_ext --inplace python setup.py develop -# git submodules are required for unit tests -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" - pip install -r $ARROW_ROOT/python/requirements-test.txt py.test pyarrow diff --git a/python/examples/minimal_build/docker-compose.yml b/python/examples/minimal_build/docker-compose.yml new file mode 100644 index 00000000000..c9655d2aca6 --- /dev/null +++ b/python/examples/minimal_build/docker-compose.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.5' + +services: + minimal-fedora-conda: + build: + context: . + dockerfile: Dockerfile.fedora + volumes: &minimal-volumes + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/build_conda.sh" + + minimal-fedora-venv: + build: + context: . + dockerfile: Dockerfile.fedora + volumes: *minimal-volumes + command: + - "/io/build_venv.sh" + + minimal-ubuntu-conda: + build: + context: . + dockerfile: Dockerfile.ubuntu + volumes: *minimal-volumes + command: + - "/io/build_conda.sh" + + minimal-ubuntu-venv: + build: + context: . + dockerfile: Dockerfile.ubuntu + volumes: *minimal-volumes + command: + - "/io/build_venv.sh"