From d1e2514b2593ab5992c69d6f14494441fc5a1119 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 23 May 2022 08:44:48 +0200 Subject: [PATCH 01/14] Initial commit for doctest CI job --- .github/workflows/python.yml | 27 ++++++++++++++++++++++ ci/docker/conda-python-doctest.dockerfile | 25 ++++++++++++++++++++ ci/scripts/install_doctest_cython.sh | 28 +++++++++++++++++++++++ docker-compose.yml | 28 +++++++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 ci/docker/conda-python-doctest.dockerfile create mode 100644 ci/scripts/install_doctest_cython.sh diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b14559d12a1..cf295c4f33a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -166,3 +166,30 @@ jobs: - name: Test shell: bash run: ci/scripts/python_test.sh $(pwd) $(pwd)/build + + doctests: + name: AMD64 Conda Python 3.9 Doctests + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + env: + PYTHON: "3.9" + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: conda-docs-${{ hashFiles('cpp/**') }} + restore-keys: conda-docs- + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run conda-python-doctest diff --git a/ci/docker/conda-python-doctest.dockerfile b/ci/docker/conda-python-doctest.dockerfile new file mode 100644 index 00000000000..9043aac3d1b --- /dev/null +++ b/ci/docker/conda-python-doctest.dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.9 +FROM ${repo}:${arch}-conda-python-${python} + +ARG doctest-cython=master +COPY ci/scripts/install_doctest_cython.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_doctest_cython.sh ${doctest-cython} diff --git a/ci/scripts/install_doctest_cython.sh b/ci/scripts/install_doctest_cython.sh new file mode 100644 index 00000000000..5934b195af7 --- /dev/null +++ b/ci/scripts/install_doctest_cython.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +doctest=$1 + +if [ "${dask}" = "master" ]; then + pip install git+https://github.com/lgpage/pytest-cython.git +else + pip install pytest-cython +fi diff --git a/docker-compose.yml b/docker-compose.yml index 406d310cbb0..91133325c63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1003,6 +1003,34 @@ services: /arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java-dist && /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"] + ############################## Doctest ################################# + + conda-python-doctest: + # Usage: + # docker-compose build conda-python-doctest + image: ${REPO}:${ARCH}-conda-python-doctest + build: + context: . + dockerfile: ci/docker/conda-python-doctest.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest + args: + repo: ${REPO} + arch: ${ARCH} + python: ${PYTHON} + shm_size: *shm-size + environment: + <<: *ccache + PARQUET_REQUIRE_ENCRYPTION: # inherit + PYTEST_ARGS: ["--doctest-modules --doctest-cython"] + HYPOTHESIS_PROFILE: # inherit + PYARROW_TEST_HYPOTHESIS: # inherit + volumes: *conda-volumes + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build && + /arrow/ci/scripts/python_build.sh /arrow /build && + /arrow/ci/scripts/python_test.sh /arrow"] + ############################## Integration ################################# conda-python-pandas: From b8aea0e627eae624067ff3834478a7a8e8ba94b9 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 23 May 2022 09:00:11 +0200 Subject: [PATCH 02/14] Correct PYTEST_ARGS --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 91133325c63..870376191c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1022,7 +1022,7 @@ services: environment: <<: *ccache PARQUET_REQUIRE_ENCRYPTION: # inherit - PYTEST_ARGS: ["--doctest-modules --doctest-cython"] + PYTEST_ARGS: "--doctest-modules --doctest-cython" HYPOTHESIS_PROFILE: # inherit PYARROW_TEST_HYPOTHESIS: # inherit volumes: *conda-volumes From 7da4eca6b8299275c97012b08a5d0d35c5c40fe8 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 23 May 2022 11:39:51 +0200 Subject: [PATCH 03/14] Try to fix importing cache manifest error --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 870376191c1..2d28873a8d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1008,7 +1008,7 @@ services: conda-python-doctest: # Usage: # docker-compose build conda-python-doctest - image: ${REPO}:${ARCH}-conda-python-doctest + image: ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest build: context: . dockerfile: ci/docker/conda-python-doctest.dockerfile From 85ffc383b09e9b6d15949e4426f0d7e097be4465 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 24 May 2022 09:22:57 +0200 Subject: [PATCH 04/14] Fix CI ValueError --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2d28873a8d5..a18ba745666 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,7 @@ x-hierarchy: - conda-python-kartothek - conda-python-spark - conda-python-turbodbc + - conda-python-doctest - conda-verify-rc - conan - debian-cpp: From 5bc5abe333762d98c2c5a31aaebd9c202cd7f9c5 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 24 May 2022 14:54:08 +0200 Subject: [PATCH 05/14] Remove extra build for doctest and instead add extra step to the sphinx doc build --- .github/workflows/python.yml | 27 ------------------- ci/conda_env_sphinx.txt | 2 ++ ci/docker/conda-python-doctest.dockerfile | 25 ----------------- ci/scripts/install_doctest_cython.sh | 28 ------------------- docker-compose.yml | 33 +++-------------------- 5 files changed, 5 insertions(+), 110 deletions(-) delete mode 100644 ci/docker/conda-python-doctest.dockerfile delete mode 100644 ci/scripts/install_doctest_cython.sh diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index cf295c4f33a..b14559d12a1 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -166,30 +166,3 @@ jobs: - name: Test shell: bash run: ci/scripts/python_test.sh $(pwd) $(pwd)/build - - doctests: - name: AMD64 Conda Python 3.9 Doctests - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - env: - PYTHON: "3.9" - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: conda-docs-${{ hashFiles('cpp/**') }} - restore-keys: conda-docs- - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run conda-python-doctest diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt index 1d5b5ab7db8..a7768085107 100644 --- a/ci/conda_env_sphinx.txt +++ b/ci/conda_env_sphinx.txt @@ -24,3 +24,5 @@ pydata-sphinx-theme sphinx-design sphinx>=4.2 sphinx-copybutton +# Requirement for doctest-cython +pytest-cython diff --git a/ci/docker/conda-python-doctest.dockerfile b/ci/docker/conda-python-doctest.dockerfile deleted file mode 100644 index 9043aac3d1b..00000000000 --- a/ci/docker/conda-python-doctest.dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.9 -FROM ${repo}:${arch}-conda-python-${python} - -ARG doctest-cython=master -COPY ci/scripts/install_doctest_cython.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_doctest_cython.sh ${doctest-cython} diff --git a/ci/scripts/install_doctest_cython.sh b/ci/scripts/install_doctest_cython.sh deleted file mode 100644 index 5934b195af7..00000000000 --- a/ci/scripts/install_doctest_cython.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -doctest=$1 - -if [ "${dask}" = "master" ]; then - pip install git+https://github.com/lgpage/pytest-cython.git -else - pip install pytest-cython -fi diff --git a/docker-compose.yml b/docker-compose.yml index a18ba745666..13d1133ab29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,7 +112,6 @@ x-hierarchy: - conda-python-kartothek - conda-python-spark - conda-python-turbodbc - - conda-python-doctest - conda-verify-rc - conan - debian-cpp: @@ -1004,34 +1003,6 @@ services: /arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java-dist && /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"] - ############################## Doctest ################################# - - conda-python-doctest: - # Usage: - # docker-compose build conda-python-doctest - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest - build: - context: . - dockerfile: ci/docker/conda-python-doctest.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest - args: - repo: ${REPO} - arch: ${ARCH} - python: ${PYTHON} - shm_size: *shm-size - environment: - <<: *ccache - PARQUET_REQUIRE_ENCRYPTION: # inherit - PYTEST_ARGS: "--doctest-modules --doctest-cython" - HYPOTHESIS_PROFILE: # inherit - PYARROW_TEST_HYPOTHESIS: # inherit - volumes: *conda-volumes - command: - ["/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/python_test.sh /arrow"] - ############################## Integration ################################# conda-python-pandas: @@ -1084,12 +1055,14 @@ services: LANG: "C.UTF-8" BUILD_DOCS_CPP: "ON" BUILD_DOCS_PYTHON: "ON" + PYTEST_ARGS: "--doctest-modules --doctest-cython" volumes: *conda-volumes command: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && pip install -e /arrow/dev/archery[numpydoc] && - archery numpydoc --allow-rule PR01,PR10"] + archery numpydoc --allow-rule PR01,PR10 + /arrow/ci/scripts/python_test.sh /arrow"] conda-python-dask: # Possible $DASK parameters: From cce055f2f9c3fe05ff55480e7a78fa481ff8fc30 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 25 May 2022 08:52:35 +0200 Subject: [PATCH 06/14] Update docker-compose.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Raúl Cumplido --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 13d1133ab29..db7f4f1dfd7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1061,7 +1061,7 @@ services: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && pip install -e /arrow/dev/archery[numpydoc] && - archery numpydoc --allow-rule PR01,PR10 + archery numpydoc --allow-rule PR01,PR10 && /arrow/ci/scripts/python_test.sh /arrow"] conda-python-dask: From 2269348e5c660e1b4be15dfb230c82aa54eea8af Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 25 May 2022 14:24:03 +0200 Subject: [PATCH 07/14] Remove --doctest-cython to be added in the last PR for doctest --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index db7f4f1dfd7..cfa70708bdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1055,7 +1055,7 @@ services: LANG: "C.UTF-8" BUILD_DOCS_CPP: "ON" BUILD_DOCS_PYTHON: "ON" - PYTEST_ARGS: "--doctest-modules --doctest-cython" + PYTEST_ARGS: "--doctest-modules" volumes: *conda-volumes command: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && From f666c54940b9ca474aefd934079c508f85b9b30b Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 25 May 2022 16:05:09 +0200 Subject: [PATCH 08/14] Doctest check corrections --- python/pyarrow/fs.py | 7 ++++--- python/pyarrow/parquet/__init__.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index e391e7d1ee2..f22eaf03041 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -224,12 +224,13 @@ def copy_files(source, destination, -------- Copy an S3 bucket's files to a local directory: - >>> copy_files("s3://your-bucket-name", "local-directory") + >>> copy_files("s3://your-bucket-name", + ... "local-directory") # doctest: +SKIP Using a FileSystem object: >>> copy_files("your-bucket-name", "local-directory", - ... source_filesystem=S3FileSystem(...)) + ... source_filesystem=S3FileSystem(...)) # doctest: +SKIP """ source_fs, source_path = _resolve_filesystem_and_path( @@ -263,7 +264,7 @@ class FSSpecHandler(FileSystemHandler): Examples -------- - >>> PyFileSystem(FSSpecHandler(fsspec_fs)) + >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP """ def __init__(self, fs): diff --git a/python/pyarrow/parquet/__init__.py b/python/pyarrow/parquet/__init__.py index 049f955f808..eb85509e177 100644 --- a/python/pyarrow/parquet/__init__.py +++ b/python/pyarrow/parquet/__init__.py @@ -1926,7 +1926,7 @@ def read_pandas(self, **kwargs): Select pandas metadata: >>> dataset.read_pandas(columns=["n_legs"]).schema.pandas_metadata - {'index_columns': [{'kind': 'range', ... 'pandas_version': '1.4.1'} + {'index_columns': [{'kind': 'range', ... } """ return self.read(use_pandas_metadata=True, **kwargs) @@ -2486,7 +2486,7 @@ def read_pandas(self, **kwargs): n_legs: [[2,2,4,4,5,100]] >>> dataset.read_pandas(columns=["n_legs"]).schema.pandas_metadata - {'index_columns': [{'kind': 'range', ... 'pandas_version': '1.4.1'} + {'index_columns': [{'kind': 'range', ... } """ return self.read(use_pandas_metadata=True, **kwargs) From 82093961123dc611a8728641ba7b1f7deac41e51 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 23 May 2022 08:44:48 +0200 Subject: [PATCH 09/14] Initial commit for doctest CI job --- .github/workflows/python.yml | 27 ++++++++++++++++++++++ ci/docker/conda-python-doctest.dockerfile | 25 ++++++++++++++++++++ ci/scripts/install_doctest_cython.sh | 28 +++++++++++++++++++++++ docker-compose.yml | 28 +++++++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 ci/docker/conda-python-doctest.dockerfile create mode 100644 ci/scripts/install_doctest_cython.sh diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b14559d12a1..cf295c4f33a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -166,3 +166,30 @@ jobs: - name: Test shell: bash run: ci/scripts/python_test.sh $(pwd) $(pwd)/build + + doctests: + name: AMD64 Conda Python 3.9 Doctests + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + env: + PYTHON: "3.9" + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: conda-docs-${{ hashFiles('cpp/**') }} + restore-keys: conda-docs- + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run conda-python-doctest diff --git a/ci/docker/conda-python-doctest.dockerfile b/ci/docker/conda-python-doctest.dockerfile new file mode 100644 index 00000000000..9043aac3d1b --- /dev/null +++ b/ci/docker/conda-python-doctest.dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.9 +FROM ${repo}:${arch}-conda-python-${python} + +ARG doctest-cython=master +COPY ci/scripts/install_doctest_cython.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_doctest_cython.sh ${doctest-cython} diff --git a/ci/scripts/install_doctest_cython.sh b/ci/scripts/install_doctest_cython.sh new file mode 100644 index 00000000000..5934b195af7 --- /dev/null +++ b/ci/scripts/install_doctest_cython.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +doctest=$1 + +if [ "${dask}" = "master" ]; then + pip install git+https://github.com/lgpage/pytest-cython.git +else + pip install pytest-cython +fi diff --git a/docker-compose.yml b/docker-compose.yml index cfa70708bdd..51f351ed01d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1003,6 +1003,34 @@ services: /arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java-dist && /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"] + ############################## Doctest ################################# + + conda-python-doctest: + # Usage: + # docker-compose build conda-python-doctest + image: ${REPO}:${ARCH}-conda-python-doctest + build: + context: . + dockerfile: ci/docker/conda-python-doctest.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest + args: + repo: ${REPO} + arch: ${ARCH} + python: ${PYTHON} + shm_size: *shm-size + environment: + <<: *ccache + PARQUET_REQUIRE_ENCRYPTION: # inherit + PYTEST_ARGS: ["--doctest-modules --doctest-cython"] + HYPOTHESIS_PROFILE: # inherit + PYARROW_TEST_HYPOTHESIS: # inherit + volumes: *conda-volumes + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build && + /arrow/ci/scripts/python_build.sh /arrow /build && + /arrow/ci/scripts/python_test.sh /arrow"] + ############################## Integration ################################# conda-python-pandas: From 6d39b5e8cfba00e55699dbbfd96448601dc1c9fb Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 23 May 2022 09:00:11 +0200 Subject: [PATCH 10/14] Correct PYTEST_ARGS --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 51f351ed01d..0cceebee7be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1022,7 +1022,7 @@ services: environment: <<: *ccache PARQUET_REQUIRE_ENCRYPTION: # inherit - PYTEST_ARGS: ["--doctest-modules --doctest-cython"] + PYTEST_ARGS: "--doctest-modules --doctest-cython" HYPOTHESIS_PROFILE: # inherit PYARROW_TEST_HYPOTHESIS: # inherit volumes: *conda-volumes From f416f4662376b649d1895990e3bc0e5d3b8bdc47 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 23 May 2022 11:39:51 +0200 Subject: [PATCH 11/14] Try to fix importing cache manifest error --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0cceebee7be..fe68dc0e523 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1008,7 +1008,7 @@ services: conda-python-doctest: # Usage: # docker-compose build conda-python-doctest - image: ${REPO}:${ARCH}-conda-python-doctest + image: ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest build: context: . dockerfile: ci/docker/conda-python-doctest.dockerfile From 1cdbf91f4bcc0605774f5d3311f783ab3cc647a2 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 24 May 2022 09:22:57 +0200 Subject: [PATCH 12/14] Fix CI ValueError --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index fe68dc0e523..47702636eab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,7 @@ x-hierarchy: - conda-python-kartothek - conda-python-spark - conda-python-turbodbc + - conda-python-doctest - conda-verify-rc - conan - debian-cpp: From 1df5c394813641e1726f62604a888d07b617a98d Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Tue, 24 May 2022 14:54:08 +0200 Subject: [PATCH 13/14] Remove extra build for doctest and instead add extra step to the sphinx doc build --- .github/workflows/python.yml | 27 --------------------- ci/docker/conda-python-doctest.dockerfile | 25 ------------------- ci/scripts/install_doctest_cython.sh | 28 ---------------------- docker-compose.yml | 29 ----------------------- 4 files changed, 109 deletions(-) delete mode 100644 ci/docker/conda-python-doctest.dockerfile delete mode 100644 ci/scripts/install_doctest_cython.sh diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index cf295c4f33a..b14559d12a1 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -166,30 +166,3 @@ jobs: - name: Test shell: bash run: ci/scripts/python_test.sh $(pwd) $(pwd)/build - - doctests: - name: AMD64 Conda Python 3.9 Doctests - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - env: - PYTHON: "3.9" - steps: - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Cache Docker Volumes - uses: actions/cache@v2 - with: - path: .docker - key: conda-docs-${{ hashFiles('cpp/**') }} - restore-keys: conda-docs- - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - run: archery docker run conda-python-doctest diff --git a/ci/docker/conda-python-doctest.dockerfile b/ci/docker/conda-python-doctest.dockerfile deleted file mode 100644 index 9043aac3d1b..00000000000 --- a/ci/docker/conda-python-doctest.dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.9 -FROM ${repo}:${arch}-conda-python-${python} - -ARG doctest-cython=master -COPY ci/scripts/install_doctest_cython.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_doctest_cython.sh ${doctest-cython} diff --git a/ci/scripts/install_doctest_cython.sh b/ci/scripts/install_doctest_cython.sh deleted file mode 100644 index 5934b195af7..00000000000 --- a/ci/scripts/install_doctest_cython.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -doctest=$1 - -if [ "${dask}" = "master" ]; then - pip install git+https://github.com/lgpage/pytest-cython.git -else - pip install pytest-cython -fi diff --git a/docker-compose.yml b/docker-compose.yml index 47702636eab..cfa70708bdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,7 +112,6 @@ x-hierarchy: - conda-python-kartothek - conda-python-spark - conda-python-turbodbc - - conda-python-doctest - conda-verify-rc - conan - debian-cpp: @@ -1004,34 +1003,6 @@ services: /arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java-dist && /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"] - ############################## Doctest ################################# - - conda-python-doctest: - # Usage: - # docker-compose build conda-python-doctest - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest - build: - context: . - dockerfile: ci/docker/conda-python-doctest.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-doctest - args: - repo: ${REPO} - arch: ${ARCH} - python: ${PYTHON} - shm_size: *shm-size - environment: - <<: *ccache - PARQUET_REQUIRE_ENCRYPTION: # inherit - PYTEST_ARGS: "--doctest-modules --doctest-cython" - HYPOTHESIS_PROFILE: # inherit - PYARROW_TEST_HYPOTHESIS: # inherit - volumes: *conda-volumes - command: - ["/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/python_test.sh /arrow"] - ############################## Integration ################################# conda-python-pandas: From 028d51d26c74e9bc4f6b84b92d7fbebee206af78 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Thu, 26 May 2022 05:55:38 +0200 Subject: [PATCH 14/14] Remove whitespace --- python/pyarrow/parquet/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/parquet/__init__.py b/python/pyarrow/parquet/__init__.py index eb85509e177..b4713a717c3 100644 --- a/python/pyarrow/parquet/__init__.py +++ b/python/pyarrow/parquet/__init__.py @@ -1926,7 +1926,7 @@ def read_pandas(self, **kwargs): Select pandas metadata: >>> dataset.read_pandas(columns=["n_legs"]).schema.pandas_metadata - {'index_columns': [{'kind': 'range', ... } + {'index_columns': [{'kind': 'range', 'name': None, 'start': 0, ...} """ return self.read(use_pandas_metadata=True, **kwargs) @@ -2486,7 +2486,7 @@ def read_pandas(self, **kwargs): n_legs: [[2,2,4,4,5,100]] >>> dataset.read_pandas(columns=["n_legs"]).schema.pandas_metadata - {'index_columns': [{'kind': 'range', ... } + {'index_columns': [{'kind': 'range', 'name': None, 'start': 0, ...} """ return self.read(use_pandas_metadata=True, **kwargs)