From 5794118db7204ad9868991cc93737c5ca133fe24 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 29 Dec 2022 20:42:20 +0100 Subject: [PATCH 01/10] remove system python symlinks --- dev/tasks/r/github.macos.autobrew.yml | 3 +++ dev/tasks/r/github.macos.brew.yml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml index 67157e854ff..2113814c333 100644 --- a/dev/tasks/r/github.macos.autobrew.yml +++ b/dev/tasks/r/github.macos.autobrew.yml @@ -42,6 +42,9 @@ jobs: # install minio for tests brew install minio brew install sccache + # The github runners have a non-brew 3.11 installed and the existing symlinks cause + # the installation of brew 3.11 to fail + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete fi cd arrow/r {{ macros.pin_brew_formulae(is_fork)|indent }} diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index 5f426ab42ce..83f0a0f28dd 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -39,6 +39,10 @@ jobs: brew install -v --HEAD apache-arrow # for testing brew install minio + + # The github runners have a non-brew 3.11 installed and the existing symlinks cause + # the installation of brew 3.11 to fail + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - uses: r-lib/actions/setup-r@v2 - name: Install dependencies run: | From 221bfab523ad9b2a95608b05864307c1510c78c8 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 29 Dec 2022 20:47:37 +0100 Subject: [PATCH 02/10] also fix cpp homebrew --- dev/tasks/homebrew-formulae/github.macos.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/tasks/homebrew-formulae/github.macos.yml b/dev/tasks/homebrew-formulae/github.macos.yml index fd694737573..2400e579c9a 100644 --- a/dev/tasks/homebrew-formulae/github.macos.yml +++ b/dev/tasks/homebrew-formulae/github.macos.yml @@ -29,6 +29,10 @@ jobs: {{ macros.configure_homebrew_arrow(formula)|indent }} - name: Test formula run: | + # The github runners have a non-brew 3.11 installed and the existing symlinks cause + # the installation of brew 3.11 to fail + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete + brew install -v --HEAD apache-arrow brew test apache-arrow brew audit --strict apache-arrow From 95d6ff994aee7c77748df71a0067f5b04cc13c06 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 30 Dec 2022 16:18:20 +0100 Subject: [PATCH 03/10] use explict rm --- dev/tasks/homebrew-formulae/github.macos.yml | 9 +++++---- dev/tasks/r/github.macos.autobrew.yml | 8 +++++--- dev/tasks/r/github.macos.brew.yml | 8 +++++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/dev/tasks/homebrew-formulae/github.macos.yml b/dev/tasks/homebrew-formulae/github.macos.yml index 2400e579c9a..d50287bb02b 100644 --- a/dev/tasks/homebrew-formulae/github.macos.yml +++ b/dev/tasks/homebrew-formulae/github.macos.yml @@ -29,10 +29,11 @@ jobs: {{ macros.configure_homebrew_arrow(formula)|indent }} - name: Test formula run: | - # The github runners have a non-brew 3.11 installed and the existing symlinks cause - # the installation of brew 3.11 to fail - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - + # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause + # the installation of brew python@3.11 to fail + rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/2to3' || echo 'Nothing to delete' + brew install -v --HEAD apache-arrow brew test apache-arrow brew audit --strict apache-arrow diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml index 2113814c333..73e6a24541e 100644 --- a/dev/tasks/r/github.macos.autobrew.yml +++ b/dev/tasks/r/github.macos.autobrew.yml @@ -42,9 +42,11 @@ jobs: # install minio for tests brew install minio brew install sccache - # The github runners have a non-brew 3.11 installed and the existing symlinks cause - # the installation of brew 3.11 to fail - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete + # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause + # the installation of brew python@3.11 to fail + rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete + rm '/usr/local/bin/2to3' || echo 'Nothing to delete' + fi cd arrow/r {{ macros.pin_brew_formulae(is_fork)|indent }} diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index 83f0a0f28dd..0d50dad3ef9 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -40,9 +40,11 @@ jobs: # for testing brew install minio - # The github runners have a non-brew 3.11 installed and the existing symlinks cause - # the installation of brew 3.11 to fail - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete + # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause + # the installation of brew python@3.11 to fail + rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/2to3' || echo 'Nothing to delete' + - uses: r-lib/actions/setup-r@v2 - name: Install dependencies run: | From df735044b145eacdbf7741549f47f902993880ab Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 30 Dec 2022 17:15:41 +0100 Subject: [PATCH 04/10] remove additonal confilicting files --- dev/tasks/homebrew-formulae/github.macos.yml | 4 ++++ dev/tasks/r/github.macos.autobrew.yml | 7 +++++-- dev/tasks/r/github.macos.brew.yml | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dev/tasks/homebrew-formulae/github.macos.yml b/dev/tasks/homebrew-formulae/github.macos.yml index d50287bb02b..29cb807e776 100644 --- a/dev/tasks/homebrew-formulae/github.macos.yml +++ b/dev/tasks/homebrew-formulae/github.macos.yml @@ -33,6 +33,10 @@ jobs: # the installation of brew python@3.11 to fail rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' rm '/usr/local/bin/2to3' || echo 'Nothing to delete' + rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' brew install -v --HEAD apache-arrow brew test apache-arrow diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml index 73e6a24541e..7b675123024 100644 --- a/dev/tasks/r/github.macos.autobrew.yml +++ b/dev/tasks/r/github.macos.autobrew.yml @@ -44,9 +44,12 @@ jobs: brew install sccache # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause # the installation of brew python@3.11 to fail - rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete + rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' rm '/usr/local/bin/2to3' || echo 'Nothing to delete' - + rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' fi cd arrow/r {{ macros.pin_brew_formulae(is_fork)|indent }} diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index 0d50dad3ef9..6e330ba3a9f 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -44,6 +44,10 @@ jobs: # the installation of brew python@3.11 to fail rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' rm '/usr/local/bin/2to3' || echo 'Nothing to delete' + rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' + rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' - uses: r-lib/actions/setup-r@v2 - name: Install dependencies From 11f2feb52c79e79ad014382bf3771e646e247329 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 30 Dec 2022 18:14:03 +0100 Subject: [PATCH 05/10] delete conflicting files BEFORE trying to install arrow --- dev/tasks/r/github.macos.brew.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index 6e330ba3a9f..e43af8fd0f0 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -31,15 +31,6 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8)}} run: | - brew install sccache - # TODO: Update the TODO for ARROW-16907 below to refer to main instead of master - # after migrating the default branch to main. - # TODO(ARROW-16907): apache/arrow@master seems to be installed already - # so this does nothing on a branch/PR - brew install -v --HEAD apache-arrow - # for testing - brew install minio - # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause # the installation of brew python@3.11 to fail rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' @@ -48,6 +39,16 @@ jobs: rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' + + brew install sccache + # for testing + brew install minio + + # TODO: Update the TODO for ARROW-16907 below to refer to main instead of master + # after migrating the default branch to main. + # TODO(ARROW-16907): apache/arrow@master seems to be installed already + # so this does nothing on a branch/PR + brew install -v --HEAD apache-arrow - uses: r-lib/actions/setup-r@v2 - name: Install dependencies From 56352b6079e04bcccb981f10998591d376a7f3a0 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 30 Dec 2022 19:25:06 +0100 Subject: [PATCH 06/10] create macro --- dev/tasks/homebrew-formulae/github.macos.yml | 10 +--------- dev/tasks/macros.jinja | 16 ++++++++++++++++ dev/tasks/r/github.macos.autobrew.yml | 9 +-------- dev/tasks/r/github.macos.brew.yml | 9 +-------- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/dev/tasks/homebrew-formulae/github.macos.yml b/dev/tasks/homebrew-formulae/github.macos.yml index 29cb807e776..52aa355a335 100644 --- a/dev/tasks/homebrew-formulae/github.macos.yml +++ b/dev/tasks/homebrew-formulae/github.macos.yml @@ -29,15 +29,7 @@ jobs: {{ macros.configure_homebrew_arrow(formula)|indent }} - name: Test formula run: | - # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause - # the installation of brew python@3.11 to fail - rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/2to3' || echo 'Nothing to delete' - rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' - + {{ macros.github_cleanup_macos_python()|indent(10) }} brew install -v --HEAD apache-arrow brew test apache-arrow brew audit --strict apache-arrow diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 72f575a1885..f88d015c155 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -396,3 +396,19 @@ on: {{ key }}: "{{ value }}" {% endfor %} {% endmacro %} + +{%- macro github_cleanup_macos_python() %} + +# The github runners have a non-brew Python 3.11 installed and the existing symlinks cause +# the installation of brew python@3.11 to fail +rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' +rm '/usr/local/bin/2to3' || echo 'Nothing to delete' +rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' +rm '/usr/local/bin/idle3' || echo 'Nothing to delete' +rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' +rm '/usr/local/bin/pydoc3' || echo 'Nothing to delete' +rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' +rm '/usr/local/bin/python3' || echo 'Nothing to delete' +rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' + +{% endmacro %} diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml index 7b675123024..6e034a735f9 100644 --- a/dev/tasks/r/github.macos.autobrew.yml +++ b/dev/tasks/r/github.macos.autobrew.yml @@ -42,14 +42,7 @@ jobs: # install minio for tests brew install minio brew install sccache - # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause - # the installation of brew python@3.11 to fail - rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/2to3' || echo 'Nothing to delete' - rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' + {{ macros.github_cleanup_macos_python()|indent(12) }} fi cd arrow/r {{ macros.pin_brew_formulae(is_fork)|indent }} diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index e43af8fd0f0..6b5ccb2568d 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -31,14 +31,7 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8)}} run: | - # The github runners have a non-brew Python 3.11 installed and the existing symlinks cause - # the installation of brew python@3.11 to fail - rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/2to3' || echo 'Nothing to delete' - rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' - rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' + {{ macros.github_cleanup_macos_python()|indent(10) }} brew install sccache # for testing From f44b699c7653fb230f8babc709978762a7acdfd3 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 30 Dec 2022 20:51:39 +0100 Subject: [PATCH 07/10] another addition --- dev/tasks/macros.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index f88d015c155..4e2deda1eda 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -410,5 +410,6 @@ rm '/usr/local/bin/pydoc3' || echo 'Nothing to delete' rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' rm '/usr/local/bin/python3' || echo 'Nothing to delete' rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' +rm '/usr/local/bin/python3-config' || echo 'Nothing to delete' {% endmacro %} From 90c3b4542c5e521d1d258d83df43391bff8a51c6 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Mon, 2 Jan 2023 16:29:07 +0100 Subject: [PATCH 08/10] use existing macro to force install python@3.11 --- dev/tasks/homebrew-formulae/github.macos.yml | 1 - dev/tasks/macros.jinja | 17 +---------------- dev/tasks/r/github.macos.autobrew.yml | 1 - dev/tasks/r/github.macos.brew.yml | 1 - 4 files changed, 1 insertion(+), 19 deletions(-) diff --git a/dev/tasks/homebrew-formulae/github.macos.yml b/dev/tasks/homebrew-formulae/github.macos.yml index 52aa355a335..fd694737573 100644 --- a/dev/tasks/homebrew-formulae/github.macos.yml +++ b/dev/tasks/homebrew-formulae/github.macos.yml @@ -29,7 +29,6 @@ jobs: {{ macros.configure_homebrew_arrow(formula)|indent }} - name: Test formula run: | - {{ macros.github_cleanup_macos_python()|indent(10) }} brew install -v --HEAD apache-arrow brew test apache-arrow brew audit --strict apache-arrow diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 4e2deda1eda..cd62f29eb0a 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -235,6 +235,7 @@ on: brew unlink python@2 || true brew config brew doctor || true + brew install --overwrite python@3.11 ARROW_GLIB_FORMULA=$(echo ${ARROW_FORMULA} | sed -e 's/\.rb/-glib.rb/') echo "ARROW_GLIB_FORMULA=${ARROW_GLIB_FORMULA}" >> ${GITHUB_ENV} @@ -397,19 +398,3 @@ on: {% endfor %} {% endmacro %} -{%- macro github_cleanup_macos_python() %} - -# The github runners have a non-brew Python 3.11 installed and the existing symlinks cause -# the installation of brew python@3.11 to fail -rm '/usr/local/bin/2to3-3.11' || echo 'Nothing to delete' -rm '/usr/local/bin/2to3' || echo 'Nothing to delete' -rm '/usr/local/bin/idle3.11' || echo 'Nothing to delete' -rm '/usr/local/bin/idle3' || echo 'Nothing to delete' -rm '/usr/local/bin/pydoc3.11' || echo 'Nothing to delete' -rm '/usr/local/bin/pydoc3' || echo 'Nothing to delete' -rm '/usr/local/bin/python3.11' || echo 'Nothing to delete' -rm '/usr/local/bin/python3' || echo 'Nothing to delete' -rm '/usr/local/bin/python3.11-config' || echo 'Nothing to delete' -rm '/usr/local/bin/python3-config' || echo 'Nothing to delete' - -{% endmacro %} diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml index 6e034a735f9..67157e854ff 100644 --- a/dev/tasks/r/github.macos.autobrew.yml +++ b/dev/tasks/r/github.macos.autobrew.yml @@ -42,7 +42,6 @@ jobs: # install minio for tests brew install minio brew install sccache - {{ macros.github_cleanup_macos_python()|indent(12) }} fi cd arrow/r {{ macros.pin_brew_formulae(is_fork)|indent }} diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml index 6b5ccb2568d..7cf86d999db 100644 --- a/dev/tasks/r/github.macos.brew.yml +++ b/dev/tasks/r/github.macos.brew.yml @@ -31,7 +31,6 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8)}} run: | - {{ macros.github_cleanup_macos_python()|indent(10) }} brew install sccache # for testing From 368765cd0e4148c690b5e292f4041035463e03bd Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Mon, 2 Jan 2023 21:13:10 +0100 Subject: [PATCH 09/10] also install 3.10 --- dev/tasks/macros.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index cd62f29eb0a..fed31b8da43 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -235,7 +235,7 @@ on: brew unlink python@2 || true brew config brew doctor || true - brew install --overwrite python@3.11 + brew install --overwrite python@3.11 python@3.10 ARROW_GLIB_FORMULA=$(echo ${ARROW_FORMULA} | sed -e 's/\.rb/-glib.rb/') echo "ARROW_GLIB_FORMULA=${ARROW_GLIB_FORMULA}" >> ${GITHUB_ENV} From 4e7f8773747793cf54e7d096fdf41edf6f674ebb Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Tue, 3 Jan 2023 18:57:28 +0100 Subject: [PATCH 10/10] add comment --- dev/tasks/macros.jinja | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index fed31b8da43..9cb0c0f8a84 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -235,6 +235,9 @@ on: brew unlink python@2 || true brew config brew doctor || true + # The GHA runners install of python > 3.10 is incompatible with brew so we + # have to force overwritting of the symlinks + # see https://github.com/actions/runner-images/issues/6868 brew install --overwrite python@3.11 python@3.10 ARROW_GLIB_FORMULA=$(echo ${ARROW_FORMULA} | sed -e 's/\.rb/-glib.rb/')