From c8229aa61e98879fb0dcf3bc129dc0d1967f6ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 11:51:40 +0200 Subject: [PATCH 01/14] ARROW-15893: [CI][Python] Add minimal python minimal builds to nightly builds --- .../python-minimal-build/github.linux.yml | 40 ++++++++++++++ dev/tasks/tasks.yml | 32 +++++++++++ python/examples/minimal_build/build_conda.sh | 2 - python/examples/minimal_build/build_venv.sh | 4 +- .../examples/minimal_build/docker-compose.yml | 53 +++++++++++++++++++ 5 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 dev/tasks/python-minimal-build/github.linux.yml create mode 100644 python/examples/minimal_build/docker-compose.yml diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml new file mode 100644 index 00000000000..b021499536e --- /dev/null +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + test: + name: | + Docker Test {{ flags|default("") }} {{ image }} {{ command|default("") }} + runs-on: ubuntu-latest + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + {% endif %} + steps: + {{ macros.github_checkout_arrow(fetch_depth=fetch_depth if fetch_depth is defined else 1)|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Run minimal build example + run: | + cd arrow/python/examples/minimal_build + docker-compose run --rm ${{ image }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 93dd5229848..ad29199e3d8 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -102,6 +102,10 @@ groups: - test-*spark* # - test-*turbodbc* + minimal: + - minimal-fedora* + - minimal-ubuntu* + example: - example-* @@ -143,6 +147,7 @@ groups: - centos-* - conan-* - conda-* + - minimal-* - java-jars # List the homebrews explicitly because we don't care about running homebrew-cpp-autobrew - homebrew-cpp @@ -157,6 +162,7 @@ groups: nightly-tests: - test-* - example-* + - minimal-* nightly-packaging: - almalinux-* @@ -202,6 +208,32 @@ tasks: flags: >- -e ARROW_CONAN_WITH_LZ4=True image: conan + + ############################## Minimal ################################## + + minimal-fedora-conda: + ci: github + template: python-minimal-build/github.linux.yml + params: + image: minimal-fedora-conda + + minimal-fedora-venv: + ci: github + template: python-minimal-build/github.linux.yml + params: + image: minimal-fedora-venv + + minimal-ubuntu-conda: + ci: github + template: python-minimal-build/github.linux.yml + params: + image: minimal-ubuntu-conda + + minimal-ubuntu-venv: + ci: github + template: python-minimal-build/github.linux.yml + params: + image: minimal-ubuntu-venv ############################## Conda Linux ############################ diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 5ab0b559c62..dec9f1b97bf 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -28,8 +28,6 @@ CPP_BUILD_DIR=$HOME/arrow-cpp-build ARROW_ROOT=/arrow PYTHON=3.10 -git clone --depth=100 https://github.com/apache/arrow.git /arrow - #---------------------------------------------------------------------- # Run these only once diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 828b72c9bcc..7baa6aa8d25 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -25,15 +25,13 @@ WORKDIR=${WORKDIR:-$HOME} MINICONDA=$WORKDIR/miniconda-for-arrow LIBRARY_INSTALL_DIR=$WORKDIR/local-libs CPP_BUILD_DIR=$WORKDIR/arrow-cpp-build -ARROW_ROOT=$WORKDIR/arrow +ARROW_ROOT=/arrow export ARROW_HOME=$WORKDIR/dist export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH python3 -m venv $WORKDIR/venv source $WORKDIR/venv/bin/activate -git clone --depth=100 https://github.com/apache/arrow.git $ARROW_ROOT - pip install -r $ARROW_ROOT/python/requirements-build.txt #---------------------------------------------------------------------- diff --git a/python/examples/minimal_build/docker-compose.yml b/python/examples/minimal_build/docker-compose.yml new file mode 100644 index 00000000000..c9655d2aca6 --- /dev/null +++ b/python/examples/minimal_build/docker-compose.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.5' + +services: + minimal-fedora-conda: + build: + context: . + dockerfile: Dockerfile.fedora + volumes: &minimal-volumes + - ../../../:/arrow:delegated + - .:/io:delegated + command: + - "/io/build_conda.sh" + + minimal-fedora-venv: + build: + context: . + dockerfile: Dockerfile.fedora + volumes: *minimal-volumes + command: + - "/io/build_venv.sh" + + minimal-ubuntu-conda: + build: + context: . + dockerfile: Dockerfile.ubuntu + volumes: *minimal-volumes + command: + - "/io/build_conda.sh" + + minimal-ubuntu-venv: + build: + context: . + dockerfile: Dockerfile.ubuntu + volumes: *minimal-volumes + command: + - "/io/build_venv.sh" From f718f7770fe2c71e6a0e680aa2f159e0784eb5c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 12:45:17 +0200 Subject: [PATCH 02/14] Fix minimal task and update minimal build readme examples --- .../python-minimal-build/github.linux.yml | 4 +-- python/examples/minimal_build/README.md | 34 ++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml index b021499536e..44d957d0663 100644 --- a/dev/tasks/python-minimal-build/github.linux.yml +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -22,7 +22,7 @@ jobs: test: name: | - Docker Test {{ flags|default("") }} {{ image }} {{ command|default("") }} + Docker Python Minimal Build {{ flags|default("") }} {{ image }} {{ command|default("") }} runs-on: ubuntu-latest {% if env is defined %} env: @@ -37,4 +37,4 @@ jobs: - name: Run minimal build example run: | cd arrow/python/examples/minimal_build - docker-compose run --rm ${{ image }} + docker-compose run --rm {{ image }} diff --git a/python/examples/minimal_build/README.md b/python/examples/minimal_build/README.md index 57d08483861..d5cda25ce2b 100644 --- a/python/examples/minimal_build/README.md +++ b/python/examples/minimal_build/README.md @@ -34,10 +34,21 @@ docker build -t arrow_fedora_minimal -f Dockerfile.fedora . Then build PyArrow with conda or pip, respectively: ``` # With pip -docker run --rm -t -i -v $PWD:/io arrow_fedora_minimal /io/build_venv.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_fedora_minimal /io/build_venv.sh # With conda -docker run --rm -t -i -v $PWD:/io arrow_fedora_minimal /io/build_conda.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_fedora_minimal /io/build_conda.sh +``` + +Alternatively you can use [Docker Compose][docker-compose] to build and run using: +``` +docker-compose build + +# With conda +docker-compose run --rm minimal-fedora-conda + +# With pip +docker-compose run --rm minimal-fedora-venv ``` ## Ubuntu 20.04 @@ -50,10 +61,21 @@ docker build -t arrow_ubuntu_minimal -f Dockerfile.ubuntu . Then build PyArrow with conda or pip, respectively: ``` # With pip -docker run --rm -t -i -v $PWD:/io arrow_ubuntu_minimal /io/build_venv.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_ubuntu_minimal /io/build_venv.sh # With conda -docker run --rm -t -i -v $PWD:/io arrow_ubuntu_minimal /io/build_conda.sh +docker run --rm -t -i -v $PWD:/io -v $PWD/../../..:/arrow arrow_ubuntu_minimal /io/build_conda.sh +``` + +Alternatively you can use [Docker Compose][docker-compose] to build and run using: +``` +docker-compose build + +# With conda +docker-compose run --rm minimal-ubuntu-conda + +# With pip +docker-compose run --rm minimal-ubuntu-venv ``` ## Building on Fedora - Podman and SELinux @@ -69,5 +91,7 @@ podman build -t arrow_fedora_minimal -f Dockerfile.fedora Then build PyArrow with pip: ``` # With pip -podman run --rm -i -v $PWD:/io:Z -t arrow_fedora_minimal /io/build_venv.sh +podman run --rm -i -v $PWD:/io:Z -v $PWD/../../..:/arrow:Z -t arrow_fedora_minimal /io/build_venv.sh ``` + +[docker-compose]: https://docs.docker.com/compose/ From 6e44e352c0d414f07e8c6ccb7273c24c5c47a550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 13:45:50 +0200 Subject: [PATCH 03/14] Add fix for git --- python/examples/minimal_build/build_conda.sh | 1 + python/examples/minimal_build/build_venv.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index dec9f1b97bf..b359ea1b633 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -28,6 +28,7 @@ CPP_BUILD_DIR=$HOME/arrow-cpp-build ARROW_ROOT=/arrow PYTHON=3.10 +git config --global --add safe.directory $ARROW_ROOT #---------------------------------------------------------------------- # Run these only once diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 7baa6aa8d25..ace42b62dca 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -32,6 +32,8 @@ export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH python3 -m venv $WORKDIR/venv source $WORKDIR/venv/bin/activate +git config --global --add safe.directory $ARROW_ROOT + pip install -r $ARROW_ROOT/python/requirements-build.txt #---------------------------------------------------------------------- From 335a3c7fc95b5cc9ff92a46c740b22977a653d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 14:42:37 +0200 Subject: [PATCH 04/14] Add submodule as git safe.directory --- python/examples/minimal_build/build_conda.sh | 11 ++++++----- python/examples/minimal_build/build_venv.sh | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index b359ea1b633..b3910e01861 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -29,6 +29,12 @@ ARROW_ROOT=/arrow PYTHON=3.10 git config --global --add safe.directory $ARROW_ROOT +# git submodules are required for unit tests +git submodule update --init +git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing +export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" +export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" + #---------------------------------------------------------------------- # Run these only once @@ -110,9 +116,4 @@ export PYARROW_WITH_PARQUET=1 # python setup.py build_ext --inplace python setup.py develop -# git submodules are required for unit tests -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" - py.test pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index ace42b62dca..edc268a0999 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -34,6 +34,12 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT +# git submodules are required for unit tests +git submodule update --init +git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing +export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" +export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" + pip install -r $ARROW_ROOT/python/requirements-build.txt #---------------------------------------------------------------------- @@ -76,11 +82,6 @@ export PYARROW_WITH_PARQUET=1 # python setup.py build_ext --inplace python setup.py develop -# git submodules are required for unit tests -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" - pip install -r $ARROW_ROOT/python/requirements-test.txt py.test pyarrow From 7b2fcf421654c993cc47913b1b7bd0e7611e5a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 16:05:02 +0200 Subject: [PATCH 05/14] Fix git submodule init directory --- python/examples/minimal_build/build_conda.sh | 3 +++ python/examples/minimal_build/build_venv.sh | 2 ++ 2 files changed, 5 insertions(+) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index b3910e01861..a78e45818ff 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -29,11 +29,14 @@ ARROW_ROOT=/arrow PYTHON=3.10 git config --global --add safe.directory $ARROW_ROOT + # git submodules are required for unit tests +pushd $ARROW_ROOT git submodule update --init git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" +popd #---------------------------------------------------------------------- # Run these only once diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index edc268a0999..2472e0cafb6 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -35,10 +35,12 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT # git submodules are required for unit tests +pushd $ARROW_ROOT git submodule update --init git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" +popd pip install -r $ARROW_ROOT/python/requirements-build.txt From 33651d43e17be838129a72188d683d054fd76f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 16:40:51 +0200 Subject: [PATCH 06/14] Remove duplicated configuration and use for loop on jinja template --- .../python-minimal-build/github.linux.yml | 5 ++-- dev/tasks/tasks.yml | 27 +++++-------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml index 44d957d0663..34b54c7d051 100644 --- a/dev/tasks/python-minimal-build/github.linux.yml +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -31,10 +31,9 @@ jobs: {% endfor %} {% endif %} steps: - {{ macros.github_checkout_arrow(fetch_depth=fetch_depth if fetch_depth is defined else 1)|indent }} - {{ macros.github_install_archery()|indent }} + {{ macros.github_checkout_arrow()|indent }} - name: Run minimal build example run: | - cd arrow/python/examples/minimal_build + cd arrow/python/examples/{{ type }} docker-compose run --rm {{ image }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ad29199e3d8..0c6c8091674 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -208,32 +208,17 @@ tasks: flags: >- -e ARROW_CONAN_WITH_LZ4=True image: conan - - ############################## Minimal ################################## - minimal-fedora-conda: - ci: github - template: python-minimal-build/github.linux.yml - params: - image: minimal-fedora-conda - - minimal-fedora-venv: - ci: github - template: python-minimal-build/github.linux.yml - params: - image: minimal-fedora-venv + ########################### Python Minimal ############################ - minimal-ubuntu-conda: +{% for kind in ["fedora-conda", "fedora-venv", "ubuntu-conda", "ubuntu-venv"] %} + example-python-minimal-build-{{ kind }}: ci: github template: python-minimal-build/github.linux.yml params: - image: minimal-ubuntu-conda - - minimal-ubuntu-venv: - ci: github - template: python-minimal-build/github.linux.yml - params: - image: minimal-ubuntu-venv + type: minimal_build + run: minimal-{{ kind }} +{% endfor %} ############################## Conda Linux ############################ From c6c8503e538f916ff3ff6aecabd34630af068bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 16:43:54 +0200 Subject: [PATCH 07/14] Fix task group definitions --- dev/tasks/tasks.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 0c6c8091674..8f435e8a529 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -102,16 +102,15 @@ groups: - test-*spark* # - test-*turbodbc* - minimal: - - minimal-fedora* - - minimal-ubuntu* - example: - example-* example-cpp: - example-*cpp* + example-python: + - example-*python* + fuzz: - test-*fuzz* @@ -147,7 +146,6 @@ groups: - centos-* - conan-* - conda-* - - minimal-* - java-jars # List the homebrews explicitly because we don't care about running homebrew-cpp-autobrew - homebrew-cpp @@ -162,7 +160,6 @@ groups: nightly-tests: - test-* - example-* - - minimal-* nightly-packaging: - almalinux-* From cb447cfdbe10346abe2d34d09e09fe5cfa56054a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 May 2022 17:40:21 +0200 Subject: [PATCH 08/14] Fix parameter name to match definition and submodules --- dev/tasks/tasks.yml | 2 +- python/examples/minimal_build/build_conda.sh | 4 +++- python/examples/minimal_build/build_venv.sh | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 8f435e8a529..14358bdc006 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -214,7 +214,7 @@ tasks: template: python-minimal-build/github.linux.yml params: type: minimal_build - run: minimal-{{ kind }} + image: minimal-{{ kind }} {% endfor %} ############################## Conda Linux ############################ diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index a78e45818ff..634928e3073 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -31,9 +31,11 @@ PYTHON=3.10 git config --global --add safe.directory $ARROW_ROOT # git submodules are required for unit tests +git config --global --add safe.directory $ARROW_ROOT/testing +git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing + pushd $ARROW_ROOT git submodule update --init -git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" popd diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 2472e0cafb6..c8a4402030c 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -35,9 +35,11 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT # git submodules are required for unit tests +git config --global --add safe.directory $ARROW_ROOT/testing +git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing + pushd $ARROW_ROOT git submodule update --init -git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" popd From 80a8876e2023478a80ada8ad043397ad82ce1328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 May 2022 12:55:04 +0200 Subject: [PATCH 09/14] Add dataset to minimal build and missing flight for venv --- python/examples/minimal_build/build_conda.sh | 2 ++ python/examples/minimal_build/build_venv.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 634928e3073..7aab4689f8c 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -89,6 +89,7 @@ cmake -GNinja \ -DCMAKE_BUILD_TYPE=DEBUG \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_DATASET=ON \ -DARROW_FLIGHT=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_ZLIB=ON \ @@ -113,6 +114,7 @@ rm -rf build/ # remove any pesky pre-existing build directory export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja +export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_PARQUET=1 diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index c8a4402030c..5f5bbe1303a 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -57,6 +57,8 @@ cmake -GNinja \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DCMAKE_INSTALL_LIBDIR=lib \ -DARROW_BUILD_STATIC=OFF \ + -DARROW_DATASET=ON \ + -DARROW_FLIGHT=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ @@ -79,6 +81,8 @@ rm -rf build/ # remove any pesky pre-existing build directory export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja +export PYARROW_WITH_DATASET=1 +export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_PARQUET=1 # You can run either "develop" or "build_ext --inplace". Your pick From c81c572fbee256372e6e891049be7a940249a25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 May 2022 14:39:50 +0200 Subject: [PATCH 10/14] Remove flight from venv to avoid dependency issues and add possibility of not retrieving submodules when checking out arrow --- dev/tasks/macros.jinja | 4 ++-- dev/tasks/python-minimal-build/github.linux.yml | 2 +- python/examples/minimal_build/build_venv.sh | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index bd65f6488c9..0ecf2d44e0b 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -25,7 +25,7 @@ on: - "*-github-*" {% endmacro %} -{%- macro github_checkout_arrow(fetch_depth=1) -%} +{%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive") -%} - name: Checkout Arrow uses: actions/checkout@v3 with: @@ -33,7 +33,7 @@ on: path: arrow repository: {{ arrow.github_repo }} ref: {{ arrow.head }} - submodules: recursive + submodules: {{ submodules }} {% endmacro %} {%- macro github_login_dockerhub() -%} diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml index 34b54c7d051..887197d17bf 100644 --- a/dev/tasks/python-minimal-build/github.linux.yml +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -31,7 +31,7 @@ jobs: {% endfor %} {% endif %} steps: - {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_checkout_arrow(submodules=false)|indent }} - name: Run minimal build example run: | diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 5f5bbe1303a..08687d5d2d1 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -58,7 +58,6 @@ cmake -GNinja \ -DCMAKE_INSTALL_LIBDIR=lib \ -DARROW_BUILD_STATIC=OFF \ -DARROW_DATASET=ON \ - -DARROW_FLIGHT=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ @@ -82,7 +81,6 @@ rm -rf build/ # remove any pesky pre-existing build directory export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja export PYARROW_WITH_DATASET=1 -export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_PARQUET=1 # You can run either "develop" or "build_ext --inplace". Your pick From f22aca9941c3e51187175cff0feae75737cfc5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 May 2022 15:38:01 +0200 Subject: [PATCH 11/14] Revert requirement for adding submodules as safe directories --- python/examples/minimal_build/build_conda.sh | 15 +++++---------- python/examples/minimal_build/build_venv.sh | 15 +++++---------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 7aab4689f8c..a3862bf9d78 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -30,16 +30,6 @@ PYTHON=3.10 git config --global --add safe.directory $ARROW_ROOT -# git submodules are required for unit tests -git config --global --add safe.directory $ARROW_ROOT/testing -git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing - -pushd $ARROW_ROOT -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" -popd - #---------------------------------------------------------------------- # Run these only once @@ -123,4 +113,9 @@ export PYARROW_WITH_PARQUET=1 # python setup.py build_ext --inplace python setup.py develop +# git submodules are required for unit tests +git submodule update --init +export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" +export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" + py.test pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 08687d5d2d1..82da65c50e6 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -34,16 +34,6 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT -# git submodules are required for unit tests -git config --global --add safe.directory $ARROW_ROOT/testing -git config --global --add safe.directory $ARROW_ROOT/cpp/submodules/parquet-testing - -pushd $ARROW_ROOT -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" -popd - pip install -r $ARROW_ROOT/python/requirements-build.txt #---------------------------------------------------------------------- @@ -88,6 +78,11 @@ export PYARROW_WITH_PARQUET=1 # python setup.py build_ext --inplace python setup.py develop +# git submodules are required for unit tests +git submodule update --init +export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" +export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" + pip install -r $ARROW_ROOT/python/requirements-test.txt py.test pyarrow From e0dbb47fb3a1045364bc0d265c76050160518f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 May 2022 17:05:09 +0200 Subject: [PATCH 12/14] Fix git safe.directory --- python/examples/minimal_build/build_conda.sh | 2 +- python/examples/minimal_build/build_venv.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index a3862bf9d78..0ab08ef69c3 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -28,7 +28,7 @@ CPP_BUILD_DIR=$HOME/arrow-cpp-build ARROW_ROOT=/arrow PYTHON=3.10 -git config --global --add safe.directory $ARROW_ROOT +git config --global --add safe.directory "*" #---------------------------------------------------------------------- # Run these only once diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 82da65c50e6..40fa98d7baf 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -32,7 +32,7 @@ export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH python3 -m venv $WORKDIR/venv source $WORKDIR/venv/bin/activate -git config --global --add safe.directory $ARROW_ROOT +git config --global --add safe.directory "*" pip install -r $ARROW_ROOT/python/requirements-build.txt From 6efaac2b6f04260bb06a4b5bc9964c260f36af95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 12 May 2022 15:48:28 +0200 Subject: [PATCH 13/14] Remove Dataset, Parquet and Flight from minimal builds --- python/examples/minimal_build/build_conda.sh | 13 +------------ python/examples/minimal_build/build_venv.sh | 11 +---------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 0ab08ef69c3..61378b9feca 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -28,7 +28,7 @@ CPP_BUILD_DIR=$HOME/arrow-cpp-build ARROW_ROOT=/arrow PYTHON=3.10 -git config --global --add safe.directory "*" +git config --global --add safe.directory $ARROW_ROOT #---------------------------------------------------------------------- # Run these only once @@ -79,15 +79,12 @@ cmake -GNinja \ -DCMAKE_BUILD_TYPE=DEBUG \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_DATASET=ON \ - -DARROW_FLIGHT=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=ON \ -DARROW_PLASMA=ON \ -DARROW_PYTHON=ON \ $ARROW_ROOT/cpp @@ -104,18 +101,10 @@ rm -rf build/ # remove any pesky pre-existing build directory export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -export PYARROW_WITH_DATASET=1 -export PYARROW_WITH_FLIGHT=1 -export PYARROW_WITH_PARQUET=1 # You can run either "develop" or "build_ext --inplace". Your pick # python setup.py build_ext --inplace python setup.py develop -# git submodules are required for unit tests -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" - py.test pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 40fa98d7baf..56dd54abbae 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -32,7 +32,7 @@ export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH python3 -m venv $WORKDIR/venv source $WORKDIR/venv/bin/activate -git config --global --add safe.directory "*" +git config --global --add safe.directory $ARROW_ROOT pip install -r $ARROW_ROOT/python/requirements-build.txt @@ -47,14 +47,12 @@ cmake -GNinja \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DCMAKE_INSTALL_LIBDIR=lib \ -DARROW_BUILD_STATIC=OFF \ - -DARROW_DATASET=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=ON \ -DARROW_PYTHON=ON \ $ARROW_ROOT/cpp @@ -70,19 +68,12 @@ rm -rf build/ # remove any pesky pre-existing build directory export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -export PYARROW_WITH_DATASET=1 -export PYARROW_WITH_PARQUET=1 # You can run either "develop" or "build_ext --inplace". Your pick # python setup.py build_ext --inplace python setup.py develop -# git submodules are required for unit tests -git submodule update --init -export PARQUET_TEST_DATA="$ARROW_ROOT/cpp/submodules/parquet-testing/data" -export ARROW_TEST_DATA="$ARROW_ROOT/testing/data" - pip install -r $ARROW_ROOT/python/requirements-test.txt py.test pyarrow From e383e7033e5a80a0bf17900415d4264bae0c1e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 17 May 2022 12:49:22 +0200 Subject: [PATCH 14/14] ARROW-15893: Remove one minimal conda and one minimal venv build to be run --- dev/tasks/tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 14358bdc006..717c2f581b0 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -208,7 +208,7 @@ tasks: ########################### Python Minimal ############################ -{% for kind in ["fedora-conda", "fedora-venv", "ubuntu-conda", "ubuntu-venv"] %} +{% for kind in ["fedora-conda", "ubuntu-venv"] %} example-python-minimal-build-{{ kind }}: ci: github template: python-minimal-build/github.linux.yml