From a690ef45c8bc0bc6056f291eb766eeab84f67344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 12:44:07 +0200 Subject: [PATCH 01/16] GH-40216: [CI][Packaging][Python] Upload pyarrow nightly wheels to scientific python channel on Anaconda --- dev/tasks/macros.jinja | 11 +++++++++++ dev/tasks/python-wheels/github.osx.yml | 1 + 2 files changed, 12 insertions(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index df55f32222e..3bcb9e58d5e 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -183,6 +183,17 @@ env: {% endif %} {% endmacro %} +{%- macro github_upload_wheel_scientific_python(pattern) -%} + {%- if arrow.is_default_branch() -%} + - name: Upload wheel to Anaconda scientific-python + uses: scientific-python/upload-nightly-action@b67d7fcc0396e1128a474d1ab2b48aa94680f9fc # 0.5.0 + with: + artifacts_path: {{ pattern }} + anaconda_nightly_upload_token: {{ '${{secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN}}' }} + anaconda_nightly_upload_labels: dev + {% endif %} +{% endmacro %} + {%- macro azure_checkout_arrow() -%} - script: | git clone --no-checkout --branch {{ arrow.branch }} {{ arrow.remote }} arrow diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index b26aeba32b7..363cc790e5c 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -126,3 +126,4 @@ jobs: {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/")|indent }} From bc682b10e6c59487b377487f797c1ce0d806d331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 12:44:39 +0200 Subject: [PATCH 02/16] Temporarily remove check for default branch to test upload --- dev/tasks/macros.jinja | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3bcb9e58d5e..0de5ac9e1c4 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -184,14 +184,12 @@ env: {% endmacro %} {%- macro github_upload_wheel_scientific_python(pattern) -%} - {%- if arrow.is_default_branch() -%} - name: Upload wheel to Anaconda scientific-python uses: scientific-python/upload-nightly-action@b67d7fcc0396e1128a474d1ab2b48aa94680f9fc # 0.5.0 with: artifacts_path: {{ pattern }} anaconda_nightly_upload_token: {{ '${{secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN}}' }} anaconda_nightly_upload_labels: dev - {% endif %} {% endmacro %} {%- macro azure_checkout_arrow() -%} From e5f45ef5758eda2388acc7e103f3cfe6f462713a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 13:56:37 +0200 Subject: [PATCH 03/16] Try manual upload to anaconda --- dev/tasks/macros.jinja | 9 ++++----- dev/tasks/python-wheels/github.osx.yml | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 0de5ac9e1c4..a20a4fdca20 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -185,11 +185,10 @@ env: {%- macro github_upload_wheel_scientific_python(pattern) -%} - name: Upload wheel to Anaconda scientific-python - uses: scientific-python/upload-nightly-action@b67d7fcc0396e1128a474d1ab2b48aa94680f9fc # 0.5.0 - with: - artifacts_path: {{ pattern }} - anaconda_nightly_upload_token: {{ '${{secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN}}' }} - anaconda_nightly_upload_labels: dev + bash: | + conda create -y -n upload_wheel_scientific_python -c conda-forge anaconda-client + source activate upload_wheel_scientific_python + anaconda -t $(CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN) upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} {% endmacro %} {%- macro azure_checkout_arrow() -%} diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index 363cc790e5c..dbfbdc6b57e 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -126,4 +126,4 @@ jobs: {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} - {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/")|indent }} + {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }} From 5336576259a424107eaf07a819cfac59031b53ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 14:03:05 +0200 Subject: [PATCH 04/16] Add env for token --- dev/tasks/macros.jinja | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index a20a4fdca20..61829a4042b 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -188,7 +188,9 @@ env: bash: | conda create -y -n upload_wheel_scientific_python -c conda-forge anaconda-client source activate upload_wheel_scientific_python - anaconda -t $(CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN) upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} + anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} + env: + CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: {{ '${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}' }} {% endmacro %} {%- macro azure_checkout_arrow() -%} From 64a9de6ec1c72294852fb1347ca5b7e75c6fad06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 14:12:13 +0200 Subject: [PATCH 05/16] Fix yaml for GitHub syntax instead of Azure :) --- dev/tasks/macros.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 61829a4042b..4f05ea20d0e 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -185,7 +185,8 @@ env: {%- macro github_upload_wheel_scientific_python(pattern) -%} - name: Upload wheel to Anaconda scientific-python - bash: | + shell: bash + run: | conda create -y -n upload_wheel_scientific_python -c conda-forge anaconda-client source activate upload_wheel_scientific_python anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} From 27a7109d848d45204061557216e2a72e2b843e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 14:38:20 +0200 Subject: [PATCH 06/16] Install conda before uploading wheel --- dev/tasks/macros.jinja | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 4f05ea20d0e..92983821d53 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -184,6 +184,10 @@ env: {% endmacro %} {%- macro github_upload_wheel_scientific_python(pattern) -%} + - name: Install conda to upload wheel to scientific-python + shell: bash + run: | + arrow/ci/scripts/install_conda.sh mambaforge latest /opt/conda - name: Upload wheel to Anaconda scientific-python shell: bash run: | From 3aeef04b469ebcd071a820f42b88d86854f831e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 15:12:09 +0200 Subject: [PATCH 07/16] Use setup-miniconda action --- dev/tasks/macros.jinja | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 92983821d53..9a15c6ac0f3 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -184,10 +184,7 @@ env: {% endmacro %} {%- macro github_upload_wheel_scientific_python(pattern) -%} - - name: Install conda to upload wheel to scientific-python - shell: bash - run: | - arrow/ci/scripts/install_conda.sh mambaforge latest /opt/conda + - uses: conda-incubator/setup-miniconda@v3 - name: Upload wheel to Anaconda scientific-python shell: bash run: | From 641edf6260aefc15626f64a164600ca69f7e2f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 17:21:41 +0200 Subject: [PATCH 08/16] Fix activate command --- dev/tasks/macros.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 9a15c6ac0f3..7400b3762a5 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -189,7 +189,7 @@ env: shell: bash run: | conda create -y -n upload_wheel_scientific_python -c conda-forge anaconda-client - source activate upload_wheel_scientific_python + conda activate upload_wheel_scientific_python anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} env: CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: {{ '${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}' }} From 94d2e9dba2e9329074d962942428e0b5251f6114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 17:56:20 +0200 Subject: [PATCH 09/16] Use the existing environment created by action instead of creating new conda env --- dev/tasks/macros.jinja | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 7400b3762a5..f91ea425651 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -185,11 +185,12 @@ env: {%- macro github_upload_wheel_scientific_python(pattern) -%} - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: upload_wheel_scientific_python - name: Upload wheel to Anaconda scientific-python shell: bash run: | - conda create -y -n upload_wheel_scientific_python -c conda-forge anaconda-client - conda activate upload_wheel_scientific_python + conda install -y -c conda-forge anaconda-client anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} env: CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: {{ '${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}' }} From 7354834257d4ab3f57e446509c9c125013a67dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 18:29:41 +0200 Subject: [PATCH 10/16] Try with setup-micromamba instead of setup-miniconda --- dev/tasks/macros.jinja | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index f91ea425651..fb7822a6393 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -184,13 +184,15 @@ env: {% endmacro %} {%- macro github_upload_wheel_scientific_python(pattern) -%} - - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: upload_wheel_scientific_python + - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 + with: + init-shell: bash + environment-name: upload_wheel_scientific_python + create-args: >- + anaconda-client - name: Upload wheel to Anaconda scientific-python shell: bash run: | - conda install -y -c conda-forge anaconda-client anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} env: CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: {{ '${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}' }} From 3d96e6e0513bafe8669466263af0b079fe946a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 28 Aug 2024 18:36:02 +0200 Subject: [PATCH 11/16] Fix identation --- dev/tasks/macros.jinja | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index fb7822a6393..6a34198f417 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -184,12 +184,12 @@ env: {% endmacro %} {%- macro github_upload_wheel_scientific_python(pattern) -%} - - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 - with: - init-shell: bash - environment-name: upload_wheel_scientific_python - create-args: >- - anaconda-client + - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 + with: + init-shell: bash + environment-name: upload_wheel_scientific_python + create-args: >- + anaconda-client - name: Upload wheel to Anaconda scientific-python shell: bash run: | From b6502bc796dbb1875bb156a1a4023b46b42be5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 29 Aug 2024 10:47:47 +0200 Subject: [PATCH 12/16] Use pip instead of conda for anaconda-client --- dev/tasks/macros.jinja | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 6a34198f417..83d2f71224d 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -184,15 +184,10 @@ env: {% endmacro %} {%- macro github_upload_wheel_scientific_python(pattern) -%} - - uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822 - with: - init-shell: bash - environment-name: upload_wheel_scientific_python - create-args: >- - anaconda-client - name: Upload wheel to Anaconda scientific-python shell: bash run: | + python3 -m pip install anaconda-client anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} env: CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: {{ '${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}' }} From 14ea848aa53104d807fb9bc2d6046290d27f81ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 29 Aug 2024 11:22:05 +0200 Subject: [PATCH 13/16] Use newer version not available on pip --- dev/tasks/macros.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 83d2f71224d..c0f616c5000 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -187,7 +187,7 @@ env: - name: Upload wheel to Anaconda scientific-python shell: bash run: | - python3 -m pip install anaconda-client + python3 -m pip install git+https://github.com/Anaconda-Platform/anaconda-client.git@1.12.3 anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} env: CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: {{ '${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}' }} From 26d73011d600a9ca94398d9ab2795fb7540bdde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 29 Aug 2024 12:21:09 +0200 Subject: [PATCH 14/16] Add upload to scientific python to the rest of the wheels and sdist --- dev/tasks/python-sdist/github.yml | 1 + dev/tasks/python-wheels/github.linux.yml | 1 + dev/tasks/python-wheels/github.windows.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/dev/tasks/python-sdist/github.yml b/dev/tasks/python-sdist/github.yml index ef36e358aa9..ce41f437946 100644 --- a/dev/tasks/python-sdist/github.yml +++ b/dev/tasks/python-sdist/github.yml @@ -43,3 +43,4 @@ jobs: {{ macros.github_upload_releases("arrow/python/dist/*.tar.gz")|indent }} {{ macros.github_upload_gemfury("arrow/python/dist/*.tar.gz")|indent }} + {{ macros.github_upload_wheel_scientific_python("arrow/python/dist/*.tar.gz")|indent }} diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 97746ba3f9b..f9df27ba317 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -110,6 +110,7 @@ jobs: {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_wheel_scientific_python("arrow/python/repaired_wheels/*.whl")|indent }} {% if arrow.is_default_branch() %} - name: Push Docker Image diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index a40b9c0d651..3a943b6ae51 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -71,6 +71,7 @@ jobs: {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} + {{ macros.github_upload_wheel_scientific_python("arrow/python/dist/*.whl")|indent }} {% if arrow.is_default_branch() %} - name: Push Docker Image From 8662076458fcb583f995b48f4ece560480d774b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 29 Aug 2024 12:21:26 +0200 Subject: [PATCH 15/16] Add sdist to wheels group on archery tasks --- dev/tasks/tasks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index cae34c32313..a5046d9e054 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -29,6 +29,7 @@ groups: wheel: - wheel-* + - python-sdist linux: - almalinux-* From 6ecb968fc47b1decb58d7619ea852fa4a1473500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 29 Aug 2024 13:20:34 +0200 Subject: [PATCH 16/16] Add back guard to only upload if branch is main --- dev/tasks/macros.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index c0f616c5000..63cb2fc6dd1 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -184,6 +184,7 @@ env: {% endmacro %} {%- macro github_upload_wheel_scientific_python(pattern) -%} + {%- if arrow.is_default_branch() -%} - name: Upload wheel to Anaconda scientific-python shell: bash run: | @@ -191,6 +192,7 @@ env: anaconda -t ${CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN} upload --force -u scientific-python-nightly-wheels --label dev {{ pattern }} env: CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN: {{ '${{ secrets.CROSSBOW_SCIENTIFIC_PYTHON_UPLOAD_TOKEN }}' }} + {% endif %} {% endmacro %} {%- macro azure_checkout_arrow() -%}