From c66f19cc1c52c95a37af940c5eb96420a3f5b44f Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 8 Nov 2023 16:40:28 -0500 Subject: [PATCH 01/80] Add initial script for creating MLTBX file --- matlab/tools/packageMatlabInterface.m | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 matlab/tools/packageMatlabInterface.m diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m new file mode 100644 index 00000000000..88d401546ef --- /dev/null +++ b/matlab/tools/packageMatlabInterface.m @@ -0,0 +1,6 @@ + +toolboxFolder = getenv("ARROW_MATLAB_TOOLBOX_FOLDER"); +identifier = "ARROW-MATLAB-TOOLBOX"; +opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); + + From 18a0d78fbd3450383a3f16a0f8205ce6b5cf11d4 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 8 Nov 2023 16:56:27 -0500 Subject: [PATCH 02/80] Add basic `matlab` packging task to crossbow `tasks.yml` file. --- dev/tasks/tasks.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index b8d2f6c0a00..300fcf1cb33 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -59,6 +59,7 @@ groups: - conan-* - debian-* - java-jars + - matlab - nuget - python-sdist - r-binary-packages @@ -769,6 +770,12 @@ tasks: params: formula: apache-arrow.rb + ############################## MATLAB Packages ################################ + + matlab: + ci: github + template: matlab/github.yml + ############################## Arrow JAR's ################################## java-jars: From 647f2002e280411ba170f2823ddee378f55132a6 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 8 Nov 2023 16:59:42 -0500 Subject: [PATCH 03/80] Configure ToolboxOptions for packaging --- matlab/tools/packageMatlabInterface.m | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 88d401546ef..796e8735c83 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -1,6 +1,24 @@ +toolboxFolder = string(getenv("ARROW_MATLAB_TOOLBOX_FOLDER")); +outputFolder = string(getenv("ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER")); +toolboxVersion = string(getenv("ARROW_MATLAB_TOOLBOX_VERSION")); -toolboxFolder = getenv("ARROW_MATLAB_TOOLBOX_FOLDER"); identifier = "ARROW-MATLAB-TOOLBOX"; + opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); +opts.ToolboxName = "MATLAB Interface to Arrow"; +opts.ToolboxVersion = toolboxVersion; +opts.ToolboxMatlabPath = toolboxFolder; + +% Set the SupportedPlatforms +opts.SupportedPlatforms.Win64 = true; +opts.SupportedPlatforms.Maci64 = true; +opts.SupportedPlatforms.Glnxa64 = true; +opts.SupportedPlatforms.MatlabOnline = true; + +% TODO: Determine what to set the min/max release to +opts.MinimumMatlabRelease = "R2023a"; +opts.MaximumMatlabRelease = ""; +opts.OutputFile = fullfile(outputFolder, "arrow_matlab.mltbx"); +matlab.addons.toolbox.packageToolbox(opts); \ No newline at end of file From 89b79c572b5acc569da89d7b1362155742d1a023 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 8 Nov 2023 17:25:39 -0500 Subject: [PATCH 04/80] Add MATLAB packaging crossbow GitHub Actions workflow. --- dev/tasks/matlab/matlab.yml | 206 ++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 dev/tasks/matlab/matlab.yml diff --git a/dev/tasks/matlab/matlab.yml b/dev/tasks/matlab/matlab.yml new file mode 100644 index 00000000000..47c0d3dde8b --- /dev/null +++ b/dev/tasks/matlab/matlab.yml @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +name: MATLAB Packaging + +on: + push: + paths: + - '.github/workflows/matlab.yml' + - 'ci/scripts/matlab*.sh' + - 'matlab/**' + - 'cpp/src/arrow/**' + pull_request: + paths: + - '.github/workflows/matlab.yml' + - 'ci/scripts/matlab*.sh' + - 'matlab/**' + - 'cpp/src/arrow/**' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + ubuntu: + name: AMD64 Ubuntu 20.04 MATLAB + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install ninja-build + run: sudo apt-get install ninja-build + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Install ccache + run: sudo apt-get install ccache + - name: Setup ccache + shell: bash + run: ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: ${{ steps.ccache-info.outputs.cache-dir }} + key: matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} + restore-keys: matlab-ccache-ubuntu- + - name: Build MATLAB Interface + run: ci/scripts/matlab_build.sh $(pwd) + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-matlab-ubuntu.tar.gz matlab/install/arrow_matlab + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: arrow-matlab-ubuntu.tar.gz + path: arrow-matlab-ubuntu.tar.gz + + macos: + name: AMD64 macOS 11 MATLAB + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install ninja-build + run: brew install ninja + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Install ccache + run: brew install ccache + - name: Setup ccache + shell: bash + run: ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: ${{ steps.ccache-info.outputs.cache-dir }} + key: matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} + restore-keys: matlab-ccache-macos- + - name: Build MATLAB Interface + run: ci/scripts/matlab_build.sh $(pwd) + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-matlab-macos.tar.gz matlab/install/arrow_matlab + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: arrow-matlab-macos.tar.gz + path: arrow-matlab-macos.tar.gz + + windows: + name: AMD64 Windows 2022 MATLAB + runs-on: windows-2022 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Download Timezone Database + shell: bash + run: ci/scripts/download_tz_database.sh + - name: Install ccache + shell: bash + run: ci/scripts/install_ccache.sh 4.6.3 /usr + - name: Setup ccache + shell: bash + run: ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: | + ${{ steps.ccache-info.outputs.cache-dir }} + key: matlab-ccache-windows-${{ hashFiles('cpp/**', 'matlab/**') }} + restore-keys: matlab-ccache-windows- + - name: Build MATLAB Interface + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/matlab_build.sh $(pwd)" + - name: Compress into single artifact to keep directory structure + shell: bash + run: tar -cvzf arrow-matlab-windows.tar.gz matlab/install/arrow_matlab + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: arrow-matlab-windows.tar.gz + path: arrow-matlab-windows.tar.gz + + package-mltbx: + name: Package MATLAB Toolbox (MLTBX) Files + runs-on: ubuntu-latest + needs: + - ubuntu + - macos + - windows + steps: + {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts-downloaded + - name: Decompress Artifacts + run: | + mv artifacts-downloaded/*/*.tar.gz . + tar -xzvf arrow-matlab-ubuntu.tar.gz + tar -xzvf arrow-matlab-macos.tar.gz + tar -xzvf arrow-matlab-windows.tar.gz + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Run commands + env: + MATLABPATH: matlab/tools + ARROW_MATLAB_TOOLBOX_FOLDER: . + ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts + ARROW_MATLAB_TOOLBOX_VERSION: 15.0.0 + uses: matlab-actions/run-command@v1 + with: + command: packageMatlabInterface + {{ macros.github_upload_releases(["artifacts/arrow_matlab.mltbx"])|indent }} From 1d97302f7ed139aded222697ff50960a63a63cc1 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 8 Nov 2023 17:26:25 -0500 Subject: [PATCH 05/80] Add MATLAB packaging workflow to workflows folder. --- .github/workflows/matlab_package.yml | 206 +++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 .github/workflows/matlab_package.yml diff --git a/.github/workflows/matlab_package.yml b/.github/workflows/matlab_package.yml new file mode 100644 index 00000000000..47c0d3dde8b --- /dev/null +++ b/.github/workflows/matlab_package.yml @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +name: MATLAB Packaging + +on: + push: + paths: + - '.github/workflows/matlab.yml' + - 'ci/scripts/matlab*.sh' + - 'matlab/**' + - 'cpp/src/arrow/**' + pull_request: + paths: + - '.github/workflows/matlab.yml' + - 'ci/scripts/matlab*.sh' + - 'matlab/**' + - 'cpp/src/arrow/**' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + ubuntu: + name: AMD64 Ubuntu 20.04 MATLAB + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install ninja-build + run: sudo apt-get install ninja-build + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Install ccache + run: sudo apt-get install ccache + - name: Setup ccache + shell: bash + run: ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: ${{ steps.ccache-info.outputs.cache-dir }} + key: matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} + restore-keys: matlab-ccache-ubuntu- + - name: Build MATLAB Interface + run: ci/scripts/matlab_build.sh $(pwd) + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-matlab-ubuntu.tar.gz matlab/install/arrow_matlab + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: arrow-matlab-ubuntu.tar.gz + path: arrow-matlab-ubuntu.tar.gz + + macos: + name: AMD64 macOS 11 MATLAB + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install ninja-build + run: brew install ninja + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Install ccache + run: brew install ccache + - name: Setup ccache + shell: bash + run: ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: ${{ steps.ccache-info.outputs.cache-dir }} + key: matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} + restore-keys: matlab-ccache-macos- + - name: Build MATLAB Interface + run: ci/scripts/matlab_build.sh $(pwd) + - name: Compress into single artifact to keep directory structure + run: tar -cvzf arrow-matlab-macos.tar.gz matlab/install/arrow_matlab + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: arrow-matlab-macos.tar.gz + path: arrow-matlab-macos.tar.gz + + windows: + name: AMD64 Windows 2022 MATLAB + runs-on: windows-2022 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Download Timezone Database + shell: bash + run: ci/scripts/download_tz_database.sh + - name: Install ccache + shell: bash + run: ci/scripts/install_ccache.sh 4.6.3 /usr + - name: Setup ccache + shell: bash + run: ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: | + ${{ steps.ccache-info.outputs.cache-dir }} + key: matlab-ccache-windows-${{ hashFiles('cpp/**', 'matlab/**') }} + restore-keys: matlab-ccache-windows- + - name: Build MATLAB Interface + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/matlab_build.sh $(pwd)" + - name: Compress into single artifact to keep directory structure + shell: bash + run: tar -cvzf arrow-matlab-windows.tar.gz matlab/install/arrow_matlab + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: arrow-matlab-windows.tar.gz + path: arrow-matlab-windows.tar.gz + + package-mltbx: + name: Package MATLAB Toolbox (MLTBX) Files + runs-on: ubuntu-latest + needs: + - ubuntu + - macos + - windows + steps: + {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts-downloaded + - name: Decompress Artifacts + run: | + mv artifacts-downloaded/*/*.tar.gz . + tar -xzvf arrow-matlab-ubuntu.tar.gz + tar -xzvf arrow-matlab-macos.tar.gz + tar -xzvf arrow-matlab-windows.tar.gz + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Run commands + env: + MATLABPATH: matlab/tools + ARROW_MATLAB_TOOLBOX_FOLDER: . + ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts + ARROW_MATLAB_TOOLBOX_VERSION: 15.0.0 + uses: matlab-actions/run-command@v1 + with: + command: packageMatlabInterface + {{ macros.github_upload_releases(["artifacts/arrow_matlab.mltbx"])|indent }} From 249f0dee04d5c5586047812a1ce6d432e18fa0e8 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 8 Nov 2023 17:29:39 -0500 Subject: [PATCH 06/80] Remove Jinja2 macros. --- .github/workflows/matlab_package.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/matlab_package.yml b/.github/workflows/matlab_package.yml index 47c0d3dde8b..1d2e3c1f46b 100644 --- a/.github/workflows/matlab_package.yml +++ b/.github/workflows/matlab_package.yml @@ -15,10 +15,6 @@ # specific language governing permissions and limitations # under the License. -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - name: MATLAB Packaging on: @@ -179,7 +175,6 @@ jobs: - macos - windows steps: - {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} - name: Download Artifacts uses: actions/download-artifact@v3 with: @@ -203,4 +198,3 @@ jobs: uses: matlab-actions/run-command@v1 with: command: packageMatlabInterface - {{ macros.github_upload_releases(["artifacts/arrow_matlab.mltbx"])|indent }} From bf43c2fe3a4b31bc2d9d3cd0eb1846a46fbbcf4e Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 8 Nov 2023 17:33:15 -0500 Subject: [PATCH 07/80] Test --- .github/workflows/matlab_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/matlab_package.yml b/.github/workflows/matlab_package.yml index 1d2e3c1f46b..87c042fe0be 100644 --- a/.github/workflows/matlab_package.yml +++ b/.github/workflows/matlab_package.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: MATLAB Packaging +name: MATLAB Packaging Workflow on: push: From 6783d5a9ffa7172816436a454aa4a243ab75f332 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 8 Nov 2023 17:34:41 -0500 Subject: [PATCH 08/80] Update workflow trigers. --- .github/workflows/matlab_package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/matlab_package.yml b/.github/workflows/matlab_package.yml index 87c042fe0be..92ee0fb0227 100644 --- a/.github/workflows/matlab_package.yml +++ b/.github/workflows/matlab_package.yml @@ -20,13 +20,13 @@ name: MATLAB Packaging Workflow on: push: paths: - - '.github/workflows/matlab.yml' + - '.github/workflows/matlab_package.yml' - 'ci/scripts/matlab*.sh' - 'matlab/**' - 'cpp/src/arrow/**' pull_request: paths: - - '.github/workflows/matlab.yml' + - '.github/workflows/matlab_package.yml' - 'ci/scripts/matlab*.sh' - 'matlab/**' - 'cpp/src/arrow/**' From 4ad761dc7a145c86a6ad0768b091b500b7e0ae31 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 8 Nov 2023 17:36:37 -0500 Subject: [PATCH 09/80] Update indentation. --- .github/workflows/matlab_package.yml | 62 ++++++++++++++-------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/matlab_package.yml b/.github/workflows/matlab_package.yml index 92ee0fb0227..ea5e6ac8dc1 100644 --- a/.github/workflows/matlab_package.yml +++ b/.github/workflows/matlab_package.yml @@ -167,34 +167,34 @@ jobs: name: arrow-matlab-windows.tar.gz path: arrow-matlab-windows.tar.gz - package-mltbx: - name: Package MATLAB Toolbox (MLTBX) Files - runs-on: ubuntu-latest - needs: - - ubuntu - - macos - - windows - steps: - - name: Download Artifacts - uses: actions/download-artifact@v3 - with: - path: artifacts-downloaded - - name: Decompress Artifacts - run: | - mv artifacts-downloaded/*/*.tar.gz . - tar -xzvf arrow-matlab-ubuntu.tar.gz - tar -xzvf arrow-matlab-macos.tar.gz - tar -xzvf arrow-matlab-windows.tar.gz - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v1 - with: - release: R2023a - - name: Run commands - env: - MATLABPATH: matlab/tools - ARROW_MATLAB_TOOLBOX_FOLDER: . - ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts - ARROW_MATLAB_TOOLBOX_VERSION: 15.0.0 - uses: matlab-actions/run-command@v1 - with: - command: packageMatlabInterface + package-mltbx: + name: Package MATLAB Toolbox (MLTBX) Files + runs-on: ubuntu-latest + needs: + - ubuntu + - macos + - windows + steps: + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts-downloaded + - name: Decompress Artifacts + run: | + mv artifacts-downloaded/*/*.tar.gz . + tar -xzvf arrow-matlab-ubuntu.tar.gz + tar -xzvf arrow-matlab-macos.tar.gz + tar -xzvf arrow-matlab-windows.tar.gz + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v1 + with: + release: R2023a + - name: Run commands + env: + MATLABPATH: matlab/tools + ARROW_MATLAB_TOOLBOX_FOLDER: . + ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts + ARROW_MATLAB_TOOLBOX_VERSION: 15.0.0 + uses: matlab-actions/run-command@v1 + with: + command: packageMatlabInterface From fffeebb2215ebf99e840de57f3c36e3b97512522 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 11:22:01 -0500 Subject: [PATCH 10/80] Add debug output --- .github/workflows/matlab_package.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/matlab_package.yml b/.github/workflows/matlab_package.yml index ea5e6ac8dc1..d0351fd11a1 100644 --- a/.github/workflows/matlab_package.yml +++ b/.github/workflows/matlab_package.yml @@ -185,6 +185,9 @@ jobs: tar -xzvf arrow-matlab-ubuntu.tar.gz tar -xzvf arrow-matlab-macos.tar.gz tar -xzvf arrow-matlab-windows.tar.gz + - name: Print Current Directory + run: | + pwd && ls && tree / - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: From 2e0ffb839a31f5f288586dbecaa7db4d2148b6fe Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 12:09:38 -0500 Subject: [PATCH 11/80] Add post-release script to upload MATLAB interface MLTBX file to GitHub Releases. --- dev/release/post-16-matlab.sh | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 dev/release/post-16-matlab.sh diff --git a/dev/release/post-16-matlab.sh b/dev/release/post-16-matlab.sh new file mode 100755 index 00000000000..4f493a19d5c --- /dev/null +++ b/dev/release/post-16-matlab.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Use verbose debug output. +set -ex +set -o pipefail + +# Ensure that a version number has been supplied. +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit +fi + +# The version number to use will be the first input argument to the script. +version=$1 + +# Download the MATLAB MLTBX file from JFrog Artifactory to a temp directory. +tmp=$(mktemp -d -t "arrow-post-matlab.XXXXX") +base_url=https://apache.jfrog.io/artifactory/arrow/matlab/${version} +matlab_artifact="matlab-arrow-${version}.mltbx" +curl --fail --location --output ${tmp}/${matlab_artifact} ${base_url}/${matlab_artifact} +github_org="mathworks" +github_repo="arrow" +github_api_endpoint="https://api.github.com/repos/${github_org}/${github_repo}/releases" +github_api_params_template='{"tag_name":"%s","target_commitish":"main","name":"%s","body":"MATLAB Interface to Arrow version %s","draft":false,"prerelease":false,"generate_release_notes":false}' +github_api_params=$(printf "$github_api_params_template" "$version" "$version" "$version") + +# Upload MLTBX file to GitHub releases. +curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_ACCESS_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + ${github_api_endpoint} \ + -d "$github_api_params" + +# Clean up downloaded artifacts. +rm -rf "${tmp}" + +echo "Success! The released MATLAB packages are available here:" +echo " https://github.com/${github_org}/${github_repo}/releases/${version}" From f081f3818a922fb17c03839412f3f94a117ae9b9 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 13:34:57 -0500 Subject: [PATCH 12/80] Move MATLAB packging workflow to dev/tasks. --- .../workflows/matlab_package.yml => dev/tasks/matlab/github.yml | 0 dev/tasks/matlab/{matlab.yml => temp.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/matlab_package.yml => dev/tasks/matlab/github.yml (100%) rename dev/tasks/matlab/{matlab.yml => temp.yml} (100%) diff --git a/.github/workflows/matlab_package.yml b/dev/tasks/matlab/github.yml similarity index 100% rename from .github/workflows/matlab_package.yml rename to dev/tasks/matlab/github.yml diff --git a/dev/tasks/matlab/matlab.yml b/dev/tasks/matlab/temp.yml similarity index 100% rename from dev/tasks/matlab/matlab.yml rename to dev/tasks/matlab/temp.yml From 9006511041b86e7aa179b3de85a6336d574c3404 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 13:51:36 -0500 Subject: [PATCH 13/80] Updated environment variables and added macros to MATLAB packging workflow. --- dev/tasks/matlab/github.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index d0351fd11a1..86ff72b824d 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + name: MATLAB Packaging Workflow on: @@ -175,6 +179,7 @@ jobs: - macos - windows steps: + {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} - name: Download Artifacts uses: actions/download-artifact@v3 with: @@ -194,10 +199,11 @@ jobs: release: R2023a - name: Run commands env: - MATLABPATH: matlab/tools - ARROW_MATLAB_TOOLBOX_FOLDER: . + MATLABPATH: arrow/matlab/tools + ARROW_MATLAB_TOOLBOX_FOLDER: matlab/install/arrow_matlab ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts - ARROW_MATLAB_TOOLBOX_VERSION: 15.0.0 + ARROW_MATLAB_TOOLBOX_VERSION: {{ arrow.no_rc_version }} uses: matlab-actions/run-command@v1 with: command: packageMatlabInterface + {{ macros.github_upload_releases(["artifacts/matlab-arrow-$ARROW_MATLAB_TOOLBOX_VERSION.mltbx"])|indent }} From d23f6f2072bfeee76ed92a5c780876c981771734 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 13:55:36 -0500 Subject: [PATCH 14/80] Hard code version number. --- dev/tasks/matlab/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 86ff72b824d..9e5d12b5bd6 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -206,4 +206,4 @@ jobs: uses: matlab-actions/run-command@v1 with: command: packageMatlabInterface - {{ macros.github_upload_releases(["artifacts/matlab-arrow-$ARROW_MATLAB_TOOLBOX_VERSION.mltbx"])|indent }} + {{ macros.github_upload_releases(["artifacts/matlab-arrow-15.0.0.mltbx"])|indent }} From 1f605fc4c106b8994478ab5df884119309326bd1 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 13:57:07 -0500 Subject: [PATCH 15/80] Remove workflow triggers. --- dev/tasks/matlab/github.yml | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 9e5d12b5bd6..4e02b499ae9 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -19,29 +19,6 @@ {{ macros.github_header() }} -name: MATLAB Packaging Workflow - -on: - push: - paths: - - '.github/workflows/matlab_package.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - pull_request: - paths: - - '.github/workflows/matlab_package.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - jobs: ubuntu: @@ -206,4 +183,4 @@ jobs: uses: matlab-actions/run-command@v1 with: command: packageMatlabInterface - {{ macros.github_upload_releases(["artifacts/matlab-arrow-15.0.0.mltbx"])|indent }} + {{ macros.github_upload_releases(["artifacts/matlab-arrow-$ARROW_MATLAB_TOOLBOX_VERSION.mltbx"])|indent }} From 1f029c8461ea7b96645114d65cce8b3d6cffe53d Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 13:57:48 -0500 Subject: [PATCH 16/80] Remove WIP branch name check. --- dev/tasks/matlab/github.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 4e02b499ae9..852f1177555 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -24,7 +24,6 @@ jobs: ubuntu: name: AMD64 Ubuntu 20.04 MATLAB runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -64,7 +63,6 @@ jobs: macos: name: AMD64 macOS 11 MATLAB runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: - name: Check out repository uses: actions/checkout@v4 @@ -104,7 +102,6 @@ jobs: windows: name: AMD64 Windows 2022 MATLAB runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: - name: Check out repository uses: actions/checkout@v4 From eb37e3777e35a7318577e324abc520f87858f162 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 14:35:06 -0500 Subject: [PATCH 17/80] Escape curly braces --- dev/tasks/matlab/github.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 852f1177555..e11753e0748 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -47,8 +47,8 @@ jobs: - name: Cache ccache uses: actions/cache@v3 with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} + path: {% raw %} ${{ steps.ccache-info.outputs.cache-dir }} {% endraw %} + key: {% raw %} matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} {% endraw %} restore-keys: matlab-ccache-ubuntu- - name: Build MATLAB Interface run: ci/scripts/matlab_build.sh $(pwd) From a8f5599465ad70d4768da8615b300d9a17ea1a88 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 14:37:37 -0500 Subject: [PATCH 18/80] Escape curly braces --- dev/tasks/matlab/github.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index e11753e0748..5d9fdc65b63 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -86,8 +86,8 @@ jobs: - name: Cache ccache uses: actions/cache@v3 with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} + path: {% raw %} ${{ steps.ccache-info.outputs.cache-dir }} {% endraw %} + key: {% raw %} matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} {% endraw %} restore-keys: matlab-ccache-macos- - name: Build MATLAB Interface run: ci/scripts/matlab_build.sh $(pwd) @@ -128,8 +128,8 @@ jobs: uses: actions/cache@v3 with: path: | - ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-windows-${{ hashFiles('cpp/**', 'matlab/**') }} + {% raw %} ${{ steps.ccache-info.outputs.cache-dir }} {% endraw %} + key: {% raw %} matlab-ccache-windows-${{ hashFiles('cpp/**', 'matlab/**') }} {% endraw %} restore-keys: matlab-ccache-windows- - name: Build MATLAB Interface shell: cmd From 75c3aeecd54b911fb5140ebb0d9282143389b9ab Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 14:50:00 -0500 Subject: [PATCH 19/80] Add artifacts for MATLAB packaging task. --- dev/tasks/tasks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 300fcf1cb33..174b060c0d9 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -775,6 +775,8 @@ tasks: matlab: ci: github template: matlab/github.yml + artifacts: + - matlab-arrow-{no_rc_version}.mltbx ############################## Arrow JAR's ################################## From 84ad3957eac807a129aa42af514d4a412f1b4954 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 14:58:35 -0500 Subject: [PATCH 20/80] Remove debug print statements. --- dev/tasks/matlab/github.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 5d9fdc65b63..2137e925e4b 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -164,9 +164,6 @@ jobs: tar -xzvf arrow-matlab-ubuntu.tar.gz tar -xzvf arrow-matlab-macos.tar.gz tar -xzvf arrow-matlab-windows.tar.gz - - name: Print Current Directory - run: | - pwd && ls && tree / - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: From 54f771586f27e8c768e491c60c2546a463b8d122 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 14:58:52 -0500 Subject: [PATCH 21/80] Remove temp task file. --- dev/tasks/matlab/temp.yml | 206 -------------------------------------- 1 file changed, 206 deletions(-) delete mode 100644 dev/tasks/matlab/temp.yml diff --git a/dev/tasks/matlab/temp.yml b/dev/tasks/matlab/temp.yml deleted file mode 100644 index 47c0d3dde8b..00000000000 --- a/dev/tasks/matlab/temp.yml +++ /dev/null @@ -1,206 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -name: MATLAB Packaging - -on: - push: - paths: - - '.github/workflows/matlab.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - pull_request: - paths: - - '.github/workflows/matlab.yml' - - 'ci/scripts/matlab*.sh' - - 'matlab/**' - - 'cpp/src/arrow/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - - ubuntu: - name: AMD64 Ubuntu 20.04 MATLAB - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install ninja-build - run: sudo apt-get install ninja-build - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v1 - with: - release: R2023a - - name: Install ccache - run: sudo apt-get install ccache - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} - restore-keys: matlab-ccache-ubuntu- - - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) - - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-matlab-ubuntu.tar.gz matlab/install/arrow_matlab - - name: Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: arrow-matlab-ubuntu.tar.gz - path: arrow-matlab-ubuntu.tar.gz - - macos: - name: AMD64 macOS 11 MATLAB - runs-on: macos-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install ninja-build - run: brew install ninja - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v1 - with: - release: R2023a - - name: Install ccache - run: brew install ccache - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} - restore-keys: matlab-ccache-macos- - - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) - - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-matlab-macos.tar.gz matlab/install/arrow_matlab - - name: Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: arrow-matlab-macos.tar.gz - path: arrow-matlab-macos.tar.gz - - windows: - name: AMD64 Windows 2022 MATLAB - runs-on: windows-2022 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v1 - with: - release: R2023a - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Install ccache - shell: bash - run: ci/scripts/install_ccache.sh 4.6.3 /usr - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: | - ${{ steps.ccache-info.outputs.cache-dir }} - key: matlab-ccache-windows-${{ hashFiles('cpp/**', 'matlab/**') }} - restore-keys: matlab-ccache-windows- - - name: Build MATLAB Interface - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/matlab_build.sh $(pwd)" - - name: Compress into single artifact to keep directory structure - shell: bash - run: tar -cvzf arrow-matlab-windows.tar.gz matlab/install/arrow_matlab - - name: Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: arrow-matlab-windows.tar.gz - path: arrow-matlab-windows.tar.gz - - package-mltbx: - name: Package MATLAB Toolbox (MLTBX) Files - runs-on: ubuntu-latest - needs: - - ubuntu - - macos - - windows - steps: - {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} - - name: Download Artifacts - uses: actions/download-artifact@v3 - with: - path: artifacts-downloaded - - name: Decompress Artifacts - run: | - mv artifacts-downloaded/*/*.tar.gz . - tar -xzvf arrow-matlab-ubuntu.tar.gz - tar -xzvf arrow-matlab-macos.tar.gz - tar -xzvf arrow-matlab-windows.tar.gz - - name: Install MATLAB - uses: matlab-actions/setup-matlab@v1 - with: - release: R2023a - - name: Run commands - env: - MATLABPATH: matlab/tools - ARROW_MATLAB_TOOLBOX_FOLDER: . - ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts - ARROW_MATLAB_TOOLBOX_VERSION: 15.0.0 - uses: matlab-actions/run-command@v1 - with: - command: packageMatlabInterface - {{ macros.github_upload_releases(["artifacts/arrow_matlab.mltbx"])|indent }} From e7e73375502702e664b5630ce8278c76e06f3372 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:11:35 -0500 Subject: [PATCH 22/80] Use literal curly brace characters. --- dev/tasks/matlab/github.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 2137e925e4b..83b8ebbc14a 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -47,8 +47,8 @@ jobs: - name: Cache ccache uses: actions/cache@v3 with: - path: {% raw %} ${{ steps.ccache-info.outputs.cache-dir }} {% endraw %} - key: {% raw %} matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} {% endraw %} + path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} + key: matlab-ccache-ubuntu-${{ '{{' }} hashFiles('cpp/**', 'matlab/**') {{ '}}' }} restore-keys: matlab-ccache-ubuntu- - name: Build MATLAB Interface run: ci/scripts/matlab_build.sh $(pwd) @@ -86,8 +86,8 @@ jobs: - name: Cache ccache uses: actions/cache@v3 with: - path: {% raw %} ${{ steps.ccache-info.outputs.cache-dir }} {% endraw %} - key: {% raw %} matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} {% endraw %} + path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} + key: matlab-ccache-macos-${{ '{{' }} hashFiles('cpp/**', 'matlab/**') {{ '}}' }} restore-keys: matlab-ccache-macos- - name: Build MATLAB Interface run: ci/scripts/matlab_build.sh $(pwd) @@ -128,8 +128,8 @@ jobs: uses: actions/cache@v3 with: path: | - {% raw %} ${{ steps.ccache-info.outputs.cache-dir }} {% endraw %} - key: {% raw %} matlab-ccache-windows-${{ hashFiles('cpp/**', 'matlab/**') }} {% endraw %} + %{{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} + key: matlab-ccache-windows-${{ '{{' }} hashFiles('cpp/**', 'matlab/**') {{ '}}' }} restore-keys: matlab-ccache-windows- - name: Build MATLAB Interface shell: cmd From 698385f54ea88ddc4c1ca755af5af77fe0673577 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:17:38 -0500 Subject: [PATCH 23/80] Fix typo --- dev/tasks/matlab/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 83b8ebbc14a..b555f2a65f4 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -128,7 +128,7 @@ jobs: uses: actions/cache@v3 with: path: | - %{{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} + ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} key: matlab-ccache-windows-${{ '{{' }} hashFiles('cpp/**', 'matlab/**') {{ '}}' }} restore-keys: matlab-ccache-windows- - name: Build MATLAB Interface From 02253b78f98ebfb10c9863f786d8c4637e18938d Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:27:43 -0500 Subject: [PATCH 24/80] Modify indentation --- dev/tasks/matlab/github.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index b555f2a65f4..51a4e44180a 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -157,24 +157,24 @@ jobs: - name: Download Artifacts uses: actions/download-artifact@v3 with: - path: artifacts-downloaded + path: artifacts-downloaded - name: Decompress Artifacts run: | - mv artifacts-downloaded/*/*.tar.gz . - tar -xzvf arrow-matlab-ubuntu.tar.gz - tar -xzvf arrow-matlab-macos.tar.gz - tar -xzvf arrow-matlab-windows.tar.gz + mv artifacts-downloaded/*/*.tar.gz . + tar -xzvf arrow-matlab-ubuntu.tar.gz + tar -xzvf arrow-matlab-macos.tar.gz + tar -xzvf arrow-matlab-windows.tar.gz - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: - release: R2023a + release: R2023a - name: Run commands env: - MATLABPATH: arrow/matlab/tools - ARROW_MATLAB_TOOLBOX_FOLDER: matlab/install/arrow_matlab - ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts - ARROW_MATLAB_TOOLBOX_VERSION: {{ arrow.no_rc_version }} + MATLABPATH: arrow/matlab/tools + ARROW_MATLAB_TOOLBOX_FOLDER: matlab/install/arrow_matlab + ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts + ARROW_MATLAB_TOOLBOX_VERSION: {{ arrow.no_rc_version }} uses: matlab-actions/run-command@v1 with: - command: packageMatlabInterface - {{ macros.github_upload_releases(["artifacts/matlab-arrow-$ARROW_MATLAB_TOOLBOX_VERSION.mltbx"])|indent }} + command: packageMatlabInterface + {{ macros.github_upload_releases(["artifacts/matlab-arrow-$ARROW_MATLAB_TOOLBOX_VERSION.mltbx"])|indent }} From 45db0635c762b4ad12d698abfe028a34fccc1fef Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:33:30 -0500 Subject: [PATCH 25/80] Remove ccache steps. --- dev/tasks/matlab/github.yml | 50 ------------------------------------- 1 file changed, 50 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 51a4e44180a..a9a69e87a7e 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -35,21 +35,6 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a - - name: Install ccache - run: sudo apt-get install ccache - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} - key: matlab-ccache-ubuntu-${{ '{{' }} hashFiles('cpp/**', 'matlab/**') {{ '}}' }} - restore-keys: matlab-ccache-ubuntu- - name: Build MATLAB Interface run: ci/scripts/matlab_build.sh $(pwd) - name: Compress into single artifact to keep directory structure @@ -74,21 +59,6 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a - - name: Install ccache - run: brew install ccache - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} - key: matlab-ccache-macos-${{ '{{' }} hashFiles('cpp/**', 'matlab/**') {{ '}}' }} - restore-keys: matlab-ccache-macos- - name: Build MATLAB Interface run: ci/scripts/matlab_build.sh $(pwd) - name: Compress into single artifact to keep directory structure @@ -111,26 +81,6 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Install ccache - shell: bash - run: ci/scripts/install_ccache.sh 4.6.3 /usr - - name: Setup ccache - shell: bash - run: ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: | - ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} - key: matlab-ccache-windows-${{ '{{' }} hashFiles('cpp/**', 'matlab/**') {{ '}}' }} - restore-keys: matlab-ccache-windows- - name: Build MATLAB Interface shell: cmd run: | From 481bc0c8a8f077cabe414845a2ccfe2a61a676b2 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:40:12 -0500 Subject: [PATCH 26/80] Fix path to ci/scripts. --- dev/tasks/matlab/github.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index a9a69e87a7e..44ee796f193 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -36,7 +36,7 @@ jobs: with: release: R2023a - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) + run: arrow/ci/scripts/matlab_build.sh $(pwd) - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-ubuntu.tar.gz matlab/install/arrow_matlab - name: Upload artifacts @@ -60,7 +60,7 @@ jobs: with: release: R2023a - name: Build MATLAB Interface - run: ci/scripts/matlab_build.sh $(pwd) + run: arrow/ci/scripts/matlab_build.sh $(pwd) - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-macos.tar.gz matlab/install/arrow_matlab - name: Upload artifacts @@ -85,7 +85,7 @@ jobs: shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/matlab_build.sh $(pwd)" + bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)" - name: Compress into single artifact to keep directory structure shell: bash run: tar -cvzf arrow-matlab-windows.tar.gz matlab/install/arrow_matlab From 1eb348a8acdc2ff55155d45e6562fa47a494d580 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:43:33 -0500 Subject: [PATCH 27/80] Update MLTBX filename to include version. --- matlab/tools/packageMatlabInterface.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 796e8735c83..2e35743677f 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -20,5 +20,5 @@ opts.MinimumMatlabRelease = "R2023a"; opts.MaximumMatlabRelease = ""; -opts.OutputFile = fullfile(outputFolder, "arrow_matlab.mltbx"); -matlab.addons.toolbox.packageToolbox(opts); \ No newline at end of file +opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersion)); +matlab.addons.toolbox.packageToolbox(opts); From 6f8246dceb50ec388425e1eaeb743b5c42d8effd Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:46:24 -0500 Subject: [PATCH 28/80] Use checkout macro. --- dev/tasks/matlab/github.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 44ee796f193..8132bfdaa8c 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -25,10 +25,7 @@ jobs: name: AMD64 Ubuntu 20.04 MATLAB runs-on: ubuntu-latest steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 + {{ macros.github_checkout_arrow()|indent }} - name: Install ninja-build run: sudo apt-get install ninja-build - name: Install MATLAB @@ -49,10 +46,7 @@ jobs: name: AMD64 macOS 11 MATLAB runs-on: macos-latest steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 + {{ macros.github_checkout_arrow()|indent }} - name: Install ninja-build run: brew install ninja - name: Install MATLAB @@ -73,10 +67,7 @@ jobs: name: AMD64 Windows 2022 MATLAB runs-on: windows-2022 steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 + {{ macros.github_checkout_arrow()|indent }} - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: From b100c5f73a5fd8bdbde94717df8edcdc6656b12f Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 15:52:43 -0500 Subject: [PATCH 29/80] Fix path to Arrow source code. --- dev/tasks/matlab/github.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 8132bfdaa8c..89ec632823b 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -33,7 +33,7 @@ jobs: with: release: R2023a - name: Build MATLAB Interface - run: arrow/ci/scripts/matlab_build.sh $(pwd) + run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-ubuntu.tar.gz matlab/install/arrow_matlab - name: Upload artifacts @@ -54,7 +54,7 @@ jobs: with: release: R2023a - name: Build MATLAB Interface - run: arrow/ci/scripts/matlab_build.sh $(pwd) + run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-macos.tar.gz matlab/install/arrow_matlab - name: Upload artifacts @@ -76,7 +76,7 @@ jobs: shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)" + bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)/arrow" - name: Compress into single artifact to keep directory structure shell: bash run: tar -cvzf arrow-matlab-windows.tar.gz matlab/install/arrow_matlab From 4f7365f3ed3d2bfed3310536a6e5ae96b708c5e0 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 16:05:38 -0500 Subject: [PATCH 30/80] Fix paths to compress tarball files. --- dev/tasks/matlab/github.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 89ec632823b..af71c17b891 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -35,7 +35,7 @@ jobs: - name: Build MATLAB Interface run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-matlab-ubuntu.tar.gz matlab/install/arrow_matlab + run: tar -cvzf arrow-matlab-ubuntu.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 with: @@ -56,7 +56,7 @@ jobs: - name: Build MATLAB Interface run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-matlab-macos.tar.gz matlab/install/arrow_matlab + run: tar -cvzf arrow-matlab-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 with: @@ -79,7 +79,7 @@ jobs: bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)/arrow" - name: Compress into single artifact to keep directory structure shell: bash - run: tar -cvzf arrow-matlab-windows.tar.gz matlab/install/arrow_matlab + run: tar -cvzf arrow-matlab-windows.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 with: @@ -112,7 +112,7 @@ jobs: - name: Run commands env: MATLABPATH: arrow/matlab/tools - ARROW_MATLAB_TOOLBOX_FOLDER: matlab/install/arrow_matlab + ARROW_MATLAB_TOOLBOX_FOLDER: arrow/matlab/install/arrow_matlab ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts ARROW_MATLAB_TOOLBOX_VERSION: {{ arrow.no_rc_version }} uses: matlab-actions/run-command@v1 From 2cff4db0d37cb344383c049cac1a12c8a6f7d634 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 16:10:14 -0500 Subject: [PATCH 31/80] Re-enable ccache. --- dev/tasks/matlab/github.yml | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index af71c17b891..69c477cacf8 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -32,6 +32,21 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a + - name: Install ccache + run: sudo apt-get install ccache + - name: Setup ccache + shell: bash + run: arrow/ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} + key: matlab-ccache-ubuntu-${{ '{{' }} hashFiles('arrow/cpp/**', 'arrow/matlab/**') {{ '}}' }} + restore-keys: matlab-ccache-ubuntu- - name: Build MATLAB Interface run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure @@ -53,6 +68,21 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a + - name: Install ccache + run: brew install ccache + - name: Setup ccache + shell: bash + run: arrow/ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} + key: matlab-ccache-macos-${{ '{{' }} hashFiles('arrow/cpp/**', 'arrow/matlab/**') {{ '}}' }} + restore-keys: matlab-ccache-macos- - name: Build MATLAB Interface run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure @@ -72,6 +102,26 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a + - name: Download Timezone Database + shell: bash + run: arrow/ci/scripts/download_tz_database.sh + - name: Install ccache + shell: bash + run: arrow/ci/scripts/install_ccache.sh 4.6.3 /usr + - name: Setup ccache + shell: bash + run: arrow/ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + shell: bash + run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT + - name: Cache ccache + uses: actions/cache@v3 + with: + path: | + ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} + key: matlab-ccache-windows-${{ '{{' }} hashFiles('arrow/cpp/**', 'arrow/matlab/**') {{ '}}' }} + restore-keys: matlab-ccache-windows- - name: Build MATLAB Interface shell: cmd run: | From 35598744b784da8aa014bedd83446beccb47b51e Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 16:33:39 -0500 Subject: [PATCH 32/80] 1. Add missing license header to packageMatlabInterface. 2. Extract Major.Minor.Patch version from toolbox version string. --- matlab/tools/packageMatlabInterface.m | 29 ++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 2e35743677f..c25fe3e0360 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -1,6 +1,33 @@ +% Licensed to the Apache Software Foundation (ASF) under one +% or more contributor license agreements. See the NOTICE file +% distributed with this work for additional information +% regarding copyright ownership. The ASF licenses this file +% to you under the Apache License, Version 2.0 (the +% "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, +% software distributed under the License is distributed on an +% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +% KIND, either express or implied. See the License for the +% specific language governing permissions and limitations +% under the License. + toolboxFolder = string(getenv("ARROW_MATLAB_TOOLBOX_FOLDER")); outputFolder = string(getenv("ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER")); -toolboxVersion = string(getenv("ARROW_MATLAB_TOOLBOX_VERSION")); +toolboxVersionRaw = string(getenv("ARROW_MATLAB_TOOLBOX_VERSION")); + +% Note: This string processing heuristic may not be robust to future +% changes in the Arrow versioning scheme. +dotIdx = strfind(toolboxVersionRaw, "."); +numDots = numel(dotIdx); +if numDots >= 3 + toolboxVersion = extractBefore(toolboxVersionRaw, dotIdx(3)); +else + toolboxVersion = toolboxVersionRaw; +end identifier = "ARROW-MATLAB-TOOLBOX"; From fa6d1b2ff6b854fd2aefbd3e808d09c3b1171116 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Thu, 9 Nov 2023 17:07:50 -0500 Subject: [PATCH 33/80] Create output folder in packageMatlabInterface. --- matlab/tools/packageMatlabInterface.m | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index c25fe3e0360..010be8888b3 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -19,6 +19,9 @@ outputFolder = string(getenv("ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER")); toolboxVersionRaw = string(getenv("ARROW_MATLAB_TOOLBOX_VERSION")); +% Output folder must exist. +mkdir(outputFolder); + % Note: This string processing heuristic may not be robust to future % changes in the Arrow versioning scheme. dotIdx = strfind(toolboxVersionRaw, "."); From 38878462ae45dfc8daada0311fe494b03490b0a6 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 13:06:13 -0500 Subject: [PATCH 34/80] Fix output MLTBX file name in packageMatlabInterface --- matlab/tools/packageMatlabInterface.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 010be8888b3..3139747f07e 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -50,5 +50,5 @@ opts.MinimumMatlabRelease = "R2023a"; opts.MaximumMatlabRelease = ""; -opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersion)); +opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); matlab.addons.toolbox.packageToolbox(opts); From fd5743ae1287806eb4aa2082c8a71432ddf76ece Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 13:24:26 -0500 Subject: [PATCH 35/80] Use sccache instead of ccache --- dev/tasks/matlab/github.yml | 54 ++++++------------------------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 69c477cacf8..d15404c20fe 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -32,22 +32,9 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a - - name: Install ccache - run: sudo apt-get install ccache - - name: Setup ccache - shell: bash - run: arrow/ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} - key: matlab-ccache-ubuntu-${{ '{{' }} hashFiles('arrow/cpp/**', 'arrow/matlab/**') {{ '}}' }} - restore-keys: matlab-ccache-ubuntu- - name: Build MATLAB Interface + env: + {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-ubuntu.tar.gz arrow/matlab/install/arrow_matlab @@ -68,22 +55,9 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a - - name: Install ccache - run: brew install ccache - - name: Setup ccache - shell: bash - run: arrow/ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} - key: matlab-ccache-macos-${{ '{{' }} hashFiles('arrow/cpp/**', 'arrow/matlab/**') {{ '}}' }} - restore-keys: matlab-ccache-macos- - name: Build MATLAB Interface + env: + {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-macos.tar.gz arrow/matlab/install/arrow_matlab @@ -105,25 +79,13 @@ jobs: - name: Download Timezone Database shell: bash run: arrow/ci/scripts/download_tz_database.sh - - name: Install ccache - shell: bash - run: arrow/ci/scripts/install_ccache.sh 4.6.3 /usr - - name: Setup ccache + - name: Install sccache shell: bash - run: arrow/ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v3 - with: - path: | - ${{ '{{' }} steps.ccache-info.outputs.cache-dir {{ '}}' }} - key: matlab-ccache-windows-${{ '{{' }} hashFiles('arrow/cpp/**', 'arrow/matlab/**') {{ '}}' }} - restore-keys: matlab-ccache-windows- + run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache - name: Build MATLAB Interface shell: cmd + env: + {{ macros.github_set_sccache_envvars()|indent(8) }} run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)/arrow" From 2c7f85bb7e08987c59f41993e090be098737475b Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 13:45:50 -0500 Subject: [PATCH 36/80] Add debug print statements to packageMatlabInterface --- matlab/tools/packageMatlabInterface.m | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 3139747f07e..af23738375d 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -22,6 +22,8 @@ % Output folder must exist. mkdir(outputFolder); +disp("Output Folder: " + outputFolder); + % Note: This string processing heuristic may not be robust to future % changes in the Arrow versioning scheme. dotIdx = strfind(toolboxVersionRaw, "."); @@ -51,4 +53,7 @@ opts.MaximumMatlabRelease = ""; opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); +disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts); + +disp("Is File " + isfile(opts.OutputFile)); \ No newline at end of file From 0ef30fd99d09352d639401ca9f8053c74ee3f46b Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 14:03:45 -0500 Subject: [PATCH 37/80] Update MLTBX output folder --- dev/tasks/matlab/github.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index d15404c20fe..71cc8d1b68c 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -125,9 +125,9 @@ jobs: env: MATLABPATH: arrow/matlab/tools ARROW_MATLAB_TOOLBOX_FOLDER: arrow/matlab/install/arrow_matlab - ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts + ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts/matlab-dist ARROW_MATLAB_TOOLBOX_VERSION: {{ arrow.no_rc_version }} uses: matlab-actions/run-command@v1 with: command: packageMatlabInterface - {{ macros.github_upload_releases(["artifacts/matlab-arrow-$ARROW_MATLAB_TOOLBOX_VERSION.mltbx"])|indent }} + {{ macros.github_upload_releases(["artifacts/matlab-dist/*.mltbx"])|indent }} From dc256eafd14ecb7184d48a29fd63c7062cb07a3a Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 15:14:01 -0500 Subject: [PATCH 38/80] 1. Call sudo apt-get update 2. Update debug print statements in packageMatlabInterface --- dev/tasks/matlab/github.yml | 2 +- matlab/tools/packageMatlabInterface.m | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 71cc8d1b68c..826a4df1b47 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -27,7 +27,7 @@ jobs: steps: {{ macros.github_checkout_arrow()|indent }} - name: Install ninja-build - run: sudo apt-get install ninja-build + run: sudo apt-get update && apt-get install ninja-build - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index af23738375d..9a8a4bf571a 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -22,7 +22,9 @@ % Output folder must exist. mkdir(outputFolder); +disp("Toolbox Folder: " + toolboxFolder); disp("Output Folder: " + outputFolder); +disp("Toolbox Version Raw: " + toolboxVersionRaw); % Note: This string processing heuristic may not be robust to future % changes in the Arrow versioning scheme. @@ -34,6 +36,8 @@ toolboxVersion = toolboxVersionRaw; end +disp("Toolbox Version:" + toolboxVersion); + identifier = "ARROW-MATLAB-TOOLBOX"; opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); @@ -55,5 +59,3 @@ opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts); - -disp("Is File " + isfile(opts.OutputFile)); \ No newline at end of file From d19db713144792a6ee9069aba80dd7a5cfd21305 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 15:34:44 -0500 Subject: [PATCH 39/80] Include sudo in apt-get update call --- dev/tasks/matlab/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 826a4df1b47..053d2190304 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -27,7 +27,7 @@ jobs: steps: {{ macros.github_checkout_arrow()|indent }} - name: Install ninja-build - run: sudo apt-get update && apt-get install ninja-build + run: sudo apt-get update && sudo apt-get install ninja-build - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: From 0137b0b32ce9026ca089108d5d6423425d4b3f78 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 16:42:15 -0500 Subject: [PATCH 40/80] Add debug print statements to packageMatlabInterfae --- matlab/tools/packageMatlabInterface.m | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 9a8a4bf571a..7567acfea68 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -41,11 +41,14 @@ identifier = "ARROW-MATLAB-TOOLBOX"; opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); +disp("ToolboxMatlabPath Post Construction: " + opts.ToolboxMatlabPath); opts.ToolboxName = "MATLAB Interface to Arrow"; opts.ToolboxVersion = toolboxVersion; opts.ToolboxMatlabPath = toolboxFolder; +disp("ToolboxMatlabPath Post Set: " + opts.ToolboxMatlabPath); + % Set the SupportedPlatforms opts.SupportedPlatforms.Win64 = true; opts.SupportedPlatforms.Maci64 = true; From 0d46d1684d8bb3b9313feafd28ab5481a8a3b7e0 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 10 Nov 2023 16:59:18 -0500 Subject: [PATCH 41/80] Don't set ToolboxMatlabPath post construction --- matlab/tools/packageMatlabInterface.m | 5 ----- 1 file changed, 5 deletions(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 7567acfea68..553733b6a0a 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -41,13 +41,8 @@ identifier = "ARROW-MATLAB-TOOLBOX"; opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); -disp("ToolboxMatlabPath Post Construction: " + opts.ToolboxMatlabPath); - opts.ToolboxName = "MATLAB Interface to Arrow"; opts.ToolboxVersion = toolboxVersion; -opts.ToolboxMatlabPath = toolboxFolder; - -disp("ToolboxMatlabPath Post Set: " + opts.ToolboxMatlabPath); % Set the SupportedPlatforms opts.SupportedPlatforms.Win64 = true; From 079505cd7ee256c83fec0bc134d24f595af5c618 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Mon, 13 Nov 2023 10:38:25 -0500 Subject: [PATCH 42/80] Work on symbolic link issue --- matlab/tools/packageMatlabInterface.m | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 553733b6a0a..d5d1dc70bd7 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -54,6 +54,19 @@ opts.MinimumMatlabRelease = "R2023a"; opts.MaximumMatlabRelease = ""; -opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); +opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", "test")); disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts); + +% Copy symlinks to package. +dylib = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.dylib"); +so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so*."); +%tmpFolder = fullfile(tempdir, "arrow-matlab"); +%cmd = compose("pushd %s; zip -u %s %s; popd", + +%sharedLibraryTargetFolder = fullfile(tmpFolder, "fsroot", "+libmexclass", "+proxy"); +%copyfile(dylib, sharedLibraryTargetFolder); +%copyfile(so, sharedLibraryTargetFolder); + +%system(compose("zip %s zip(opts.OutputFile, fullfile(tmpFolder, "/")); +%movefile(opts.OutputFile + ".zip", opts.OutputFile); \ No newline at end of file From 2d041912c4a8c5f35ab9d0f842a20ff5bea3646d Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Mon, 13 Nov 2023 15:30:54 -0500 Subject: [PATCH 43/80] Fix packaging script to include missing shared library files --- matlab/tools/packageMatlabInterface.m | 77 +++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index d5d1dc70bd7..a5f8d4a4508 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -54,19 +54,78 @@ opts.MinimumMatlabRelease = "R2023a"; opts.MaximumMatlabRelease = ""; -opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", "test")); +opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts); % Copy symlinks to package. dylib = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.dylib"); -so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so*."); -%tmpFolder = fullfile(tempdir, "arrow-matlab"); -%cmd = compose("pushd %s; zip -u %s %s; popd", +so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so"); -%sharedLibraryTargetFolder = fullfile(tmpFolder, "fsroot", "+libmexclass", "+proxy"); -%copyfile(dylib, sharedLibraryTargetFolder); -%copyfile(so, sharedLibraryTargetFolder); +% Get the filenames and file paths of all dylibs within +proxy +dylibFiles = dir(dylib); +dylibFileNames = convertCharsToStrings({dylibFiles.name}); +dylibFilePaths = fullfile({dylibFiles.folder}, dylibFileNames); -%system(compose("zip %s zip(opts.OutputFile, fullfile(tmpFolder, "/")); -%movefile(opts.OutputFile + ".zip", opts.OutputFile); \ No newline at end of file +% Get the filenames and file paths of all sos within +proxy +soFiles = dir(so); +soFileNames = convertCharsToStrings({soFiles.name}); +soFilePaths = fullfile({soFiles.folder}, soFileNames); + +sharedLibFileNames = [dylibFileNames soFileNames]; +sharedLibFilePaths = [dylibFilePaths soFilePaths]; + +% Determine which dylibs and sos were not included in the MLTBX file +[~, name, ext] = fileparts(opts.ToolboxFiles); +idx = ~ismember(sharedLibFileNames, name + ext); +sharedLibrariesToCopy = sharedLibFilePaths(idx); + +tmpFolder = fullfile(tempdir, "arrow-matlab"); +unzip(opts.OutputFile, tmpFolder); + +% Get top-level directories and files +dirContents = dir(tmpFolder); +dotOrDotDotIdx = ismember({dirContents.name}, [".", ".."]); +dirContents(dotOrDotDotIdx) = []; +rootPaths = fullfile(tmpFolder, {dirContents.name}); + +delete(opts.OutputFile); + +% Copy missing shared libraries to the proper subfolder within fsroot +sharedLibraryTargetFolder = fullfile(tmpFolder, "fsroot", "+libmexclass", "+proxy"); +for ii = 1:numel(sharedLibrariesToCopy) + copyfile(sharedLibrariesToCopy(ii), sharedLibraryTargetFolder); +end + +manifestFilename = fullfile(tmpFolder, "metadata", "filesystemManifest.xml"); +parser = matlab.io.xml.dom.Parser; +manifestXDoc = parser.parseFile(manifestFilename); + +fileEntriesNode = manifestXDoc.Children; +date = datetime("now", TimeZone="UTC", Format="yyyy-MM-dd'T'hh:mm:ss"); +date = string(date) + "Z"; +permissions = "0644"; + +% Use the first element as a template for the new nodes +firstElement = fileEntriesNode.getFirstElementChild; + +for ii = 1:numel(sharedLibrariesToCopy) + % Clone the first node because of handle copy-semantics + fileEntryNode = firstElement.cloneNode(false); + nameAttributeValue = extractAfter(sharedLibrariesToCopy(ii), toolboxFolder); + contentAttributeValue = fullfile("/fsroot", nameAttributeValue); + fileEntryNode.setAttribute("name", nameAttributeValue); + fileEntryNode.setAttribute("content", contentAttributeValue); + fileEntryNode.setAttribute("date", date); + fileEntryNode.setAttribute("permissions", permissions); + fileEntryNode.setAttribute("type", "File"); + fileEntriesNode.appendChild(fileEntryNode); +end + +% Export the updated DOM to an XML file +writer = matlab.io.xml.dom.DOMWriter; +writeToFile(writer, manifestXDoc, manifestFilename); + +% Re-zip the MLTBX source files and remove ".zip" from the archive filename +zip(opts.OutputFile, rootPaths); +movefile(opts.OutputFile + ".zip", opts.OutputFile); From d754fb46f036c73fe8315d28b93362331f138fa0 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Mon, 13 Nov 2023 15:51:53 -0500 Subject: [PATCH 44/80] Fix shared object wildcard pattern --- matlab/tools/packageMatlabInterface.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index a5f8d4a4508..228fa0dc604 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -60,7 +60,7 @@ % Copy symlinks to package. dylib = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.dylib"); -so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so"); +so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so*"); % Get the filenames and file paths of all dylibs within +proxy dylibFiles = dir(dylib); From 9d948b8e14720e5a82bf891da436c26834894272 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 14 Nov 2023 16:29:42 -0500 Subject: [PATCH 45/80] Update shared library install name --- dev/tasks/matlab/github.yml | 2 + matlab/tools/packageMatlabInterface.m | 143 +++++++++++++------------- 2 files changed, 74 insertions(+), 71 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 053d2190304..f9ea11c9155 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -59,6 +59,8 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow + - name: change shared library install name + run: install_name_tool -change @rpath/libarrow.1500.dylib @rpath/libarrow.1500.0.0.dylib libarrowproxy.dylib - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 228fa0dc604..b594a39b19a 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -58,74 +58,75 @@ disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts); -% Copy symlinks to package. -dylib = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.dylib"); -so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so*"); - -% Get the filenames and file paths of all dylibs within +proxy -dylibFiles = dir(dylib); -dylibFileNames = convertCharsToStrings({dylibFiles.name}); -dylibFilePaths = fullfile({dylibFiles.folder}, dylibFileNames); - -% Get the filenames and file paths of all sos within +proxy -soFiles = dir(so); -soFileNames = convertCharsToStrings({soFiles.name}); -soFilePaths = fullfile({soFiles.folder}, soFileNames); - -sharedLibFileNames = [dylibFileNames soFileNames]; -sharedLibFilePaths = [dylibFilePaths soFilePaths]; - -% Determine which dylibs and sos were not included in the MLTBX file -[~, name, ext] = fileparts(opts.ToolboxFiles); -idx = ~ismember(sharedLibFileNames, name + ext); -sharedLibrariesToCopy = sharedLibFilePaths(idx); - -tmpFolder = fullfile(tempdir, "arrow-matlab"); -unzip(opts.OutputFile, tmpFolder); - -% Get top-level directories and files -dirContents = dir(tmpFolder); -dotOrDotDotIdx = ismember({dirContents.name}, [".", ".."]); -dirContents(dotOrDotDotIdx) = []; -rootPaths = fullfile(tmpFolder, {dirContents.name}); - -delete(opts.OutputFile); - -% Copy missing shared libraries to the proper subfolder within fsroot -sharedLibraryTargetFolder = fullfile(tmpFolder, "fsroot", "+libmexclass", "+proxy"); -for ii = 1:numel(sharedLibrariesToCopy) - copyfile(sharedLibrariesToCopy(ii), sharedLibraryTargetFolder); -end - -manifestFilename = fullfile(tmpFolder, "metadata", "filesystemManifest.xml"); -parser = matlab.io.xml.dom.Parser; -manifestXDoc = parser.parseFile(manifestFilename); - -fileEntriesNode = manifestXDoc.Children; -date = datetime("now", TimeZone="UTC", Format="yyyy-MM-dd'T'hh:mm:ss"); -date = string(date) + "Z"; -permissions = "0644"; - -% Use the first element as a template for the new nodes -firstElement = fileEntriesNode.getFirstElementChild; - -for ii = 1:numel(sharedLibrariesToCopy) - % Clone the first node because of handle copy-semantics - fileEntryNode = firstElement.cloneNode(false); - nameAttributeValue = extractAfter(sharedLibrariesToCopy(ii), toolboxFolder); - contentAttributeValue = fullfile("/fsroot", nameAttributeValue); - fileEntryNode.setAttribute("name", nameAttributeValue); - fileEntryNode.setAttribute("content", contentAttributeValue); - fileEntryNode.setAttribute("date", date); - fileEntryNode.setAttribute("permissions", permissions); - fileEntryNode.setAttribute("type", "File"); - fileEntriesNode.appendChild(fileEntryNode); -end - -% Export the updated DOM to an XML file -writer = matlab.io.xml.dom.DOMWriter; -writeToFile(writer, manifestXDoc, manifestFilename); - -% Re-zip the MLTBX source files and remove ".zip" from the archive filename -zip(opts.OutputFile, rootPaths); -movefile(opts.OutputFile + ".zip", opts.OutputFile); +% % Copy symlinks to package. +% dylib = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.dylib"); +% so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so*"); +% +% % Get the filenames and file paths of all dylibs within +proxy +% dylibFiles = dir(dylib); +% dylibFileNames = convertCharsToStrings({dylibFiles.name}); +% dylibFilePaths = fullfile({dylibFiles.folder}, dylibFileNames); +% +% % Get the filenames and file paths of all sos within +proxy +% soFiles = dir(so); +% soFileNames = convertCharsToStrings({soFiles.name}); +% soFilePaths = fullfile({soFiles.folder}, soFileNames); +% +% sharedLibFileNames = [dylibFileNames soFileNames]; +% sharedLibFilePaths = [dylibFilePaths soFilePaths]; +% +% % Determine which dylibs and sos were not included in the MLTBX file +% [~, name, ext] = fileparts(opts.ToolboxFiles); +% idx = ~ismember(sharedLibFileNames, name + ext); +% sharedLibrariesToCopy = sharedLibFilePaths(idx); +% +% tmpFolder = fullfile(tempdir, "arrow-matlab"); +% unzip(opts.OutputFile, tmpFolder); +% +% % Get top-level directories and files +% dirContents = dir(tmpFolder); +% dotOrDotDotIdx = ismember({dirContents.name}, [".", ".."]); +% dirContents(dotOrDotDotIdx) = []; +% rootPaths = convertCharsToStrings({dirContents.name}); +% +% delete(opts.OutputFile); +% +% % Copy missing shared libraries to the proper subfolder within fsroot +% sharedLibraryTargetFolder = fullfile(tmpFolder, "fsroot", "+libmexclass", "+proxy"); +% for ii = 1:numel(sharedLibrariesToCopy) +% cmd = compose("cp -P %s %s", sharedLibrariesToCopy(ii), sharedLibraryTargetFolder); +% system(cmd); +% end +% +% manifestFilename = fullfile(tmpFolder, "metadata", "filesystemManifest.xml"); +% parser = matlab.io.xml.dom.Parser; +% manifestXDoc = parser.parseFile(manifestFilename); +% +% fileEntriesNode = manifestXDoc.Children; +% date = datetime("now", TimeZone="UTC", Format="yyyy-MM-dd'T'hh:mm:ss"); +% date = string(date) + "Z"; +% permissions = "0644"; +% +% % Use the first element as a template for the new nodes +% firstElement = fileEntriesNode.getFirstElementChild; +% +% for ii = 1:numel(sharedLibrariesToCopy) +% % Clone the first node because of handle copy-semantics +% fileEntryNode = firstElement.cloneNode(false); +% nameAttributeValue = extractAfter(sharedLibrariesToCopy(ii), toolboxFolder); +% contentAttributeValue = fullfile("/fsroot", nameAttributeValue); +% fileEntryNode.setAttribute("name", nameAttributeValue); +% fileEntryNode.setAttribute("content", contentAttributeValue); +% fileEntryNode.setAttribute("date", date); +% fileEntryNode.setAttribute("permissions", permissions); +% fileEntryNode.setAttribute("type", "File"); +% fileEntriesNode.appendChild(fileEntryNode); +% end +% +% % Export the updated DOM to an XML file +% writer = matlab.io.xml.dom.DOMWriter; +% writeToFile(writer, manifestXDoc, manifestFilename); +% +% % Re-zip the MLTBX source files +% cmd = compose("pushd %s; zip -y -r %s %s; popd", tmpFolder, opts.OutputFile, strjoin(rootPaths)); +% system(cmd); From 98a655dbeda6d142e10381b46cfe9a2d9777ef10 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 14 Nov 2023 16:32:07 -0500 Subject: [PATCH 46/80] Update file path --- dev/tasks/matlab/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index f9ea11c9155..64965bd7278 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -60,7 +60,7 @@ jobs: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: change shared library install name - run: install_name_tool -change @rpath/libarrow.1500.dylib @rpath/libarrow.1500.0.0.dylib libarrowproxy.dylib + run: install_name_tool -change @rpath/libarrow.1500.dylib @rpath/libarrow.1500.0.0.dylib arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/libarrowproxy.dylib - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts From 47dd3ab7341782b471de2e20d0401b4c16eb8c26 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Tue, 14 Nov 2023 17:00:58 -0500 Subject: [PATCH 47/80] Dynamically create library names --- dev/tasks/matlab/github.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 64965bd7278..cd049124d49 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -60,7 +60,10 @@ jobs: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: change shared library install name - run: install_name_tool -change @rpath/libarrow.1500.dylib @rpath/libarrow.1500.0.0.dylib arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/libarrowproxy.dylib + env: + TEST_VERSION: {{ arrow.version }} + TEST_R_VERSION: {{ arrow.r_version }} + run: install_name_tool -change @rpath/libarrow.$TEST_VERSION.dylib @rpath/libarrow.$TEST_R_VERSION.dylib arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/libarrowproxy.dylib - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts From c9a77e849807989ee5bb1a0f084999f13d4e8450 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 10:50:28 -0500 Subject: [PATCH 48/80] Use regexp to find the versioned libarrow symlink and the regular libarrow dylib --- dev/tasks/matlab/github.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index cd049124d49..77eac0ec3ca 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -60,10 +60,12 @@ jobs: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: change shared library install name - env: - TEST_VERSION: {{ arrow.version }} - TEST_R_VERSION: {{ arrow.r_version }} - run: install_name_tool -change @rpath/libarrow.$TEST_VERSION.dylib @rpath/libarrow.$TEST_R_VERSION.dylib arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/libarrowproxy.dylib + run: | + pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy + SYMLINK_ARROW_LIB="$(find . -regex "./libarrow.[0-9][0-9][0-9][0-9].dylib" | xargs basename)" + REGULAR_ARROW_LIB="$(find . -regex "./libarrow.[0-9][0-9][0-9][0-9].[0-9].[0-9].dylib" | xargs basename)" + install_name_tool -change @rpath/$SYMLINK_ARROW_LIB @rpath/$REGULAR_ARROW_LIB libarrowproxy.dylib + popd - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts From 931f54eee800255dafa139db4a90757778eb030f Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 12:17:43 -0500 Subject: [PATCH 49/80] Update needed shared library name in libarrowproxy.so --- dev/tasks/matlab/github.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 77eac0ec3ca..9c25f1a64e0 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -36,6 +36,13 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow + - name: change shared library install name + run: | + pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ + SYMLINK_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9]" | xargs basename)" + REGULAR_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9].[0-9].[0-9]" | xargs basename)" + patchelf --replace-needed $SYMLINK_ARROW_LIB $REGULAR_ARROW_LIB + popd - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-ubuntu.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts From 706f7b5369b886780018f691453ef3e4a5066def Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 12:30:26 -0500 Subject: [PATCH 50/80] Add missing filename to patchelf command --- dev/tasks/matlab/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 9c25f1a64e0..317087bb2a1 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -41,7 +41,7 @@ jobs: pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ SYMLINK_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9]" | xargs basename)" REGULAR_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9].[0-9].[0-9]" | xargs basename)" - patchelf --replace-needed $SYMLINK_ARROW_LIB $REGULAR_ARROW_LIB + patchelf --replace-needed $SYMLINK_ARROW_LIB $REGULAR_ARROW_LIB libarrowproxy.so popd - name: Compress into single artifact to keep directory structure run: tar -cvzf arrow-matlab-ubuntu.tar.gz arrow/matlab/install/arrow_matlab From d254e81b3ff3ad0163f2813338c7de5d7293cd7e Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 14:57:09 -0500 Subject: [PATCH 51/80] Change runs-on to ubuntu-20.04 from ubuntu-latest --- dev/tasks/matlab/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 317087bb2a1..4384fb207e2 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -23,7 +23,7 @@ jobs: ubuntu: name: AMD64 Ubuntu 20.04 MATLAB - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: {{ macros.github_checkout_arrow()|indent }} - name: Install ninja-build From 609bafbc564338dfaee3b90d94d934e762ee215b Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 15:03:48 -0500 Subject: [PATCH 52/80] Add ls command to find libstdc++ --- dev/tasks/matlab/github.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 4384fb207e2..8b02a942913 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -32,6 +32,11 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a + - name: find system libstdc++ + run: | + popd /usr/lib/ + ls */** + pushd - name: Build MATLAB Interface env: {{ macros.github_set_sccache_envvars()|indent(8) }} From d7f369d17440d4389f1176a4c5d6b6377534fc3e Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 15:07:32 -0500 Subject: [PATCH 53/80] Fix syntax error --- dev/tasks/matlab/github.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 8b02a942913..09bb57981c3 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -34,9 +34,9 @@ jobs: release: R2023a - name: find system libstdc++ run: | - popd /usr/lib/ + pushd /usr/lib/ ls */** - pushd + popd - name: Build MATLAB Interface env: {{ macros.github_set_sccache_envvars()|indent(8) }} From bb1de7b5d40cfac260db5d76fbc5539a12f5e697 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 15:16:35 -0500 Subject: [PATCH 54/80] Add debug print statements --- dev/tasks/matlab/github.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 09bb57981c3..ab12e3f3fe7 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -35,7 +35,10 @@ jobs: - name: find system libstdc++ run: | pushd /usr/lib/ - ls */** + ls -la x86_64-linux-gnu/libstdc++* + ls -la x86_64-linux-gnu/libc.* + strings x86_64-linux-gnu/libstdc++.so.6 | grep -i GLIBC + strings x86_64-linux-gnu/libc.so | grep -i GLIBC popd - name: Build MATLAB Interface env: From ee96e8c4a11f748688075b337852bbb5ac4014ca Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 16:14:08 -0500 Subject: [PATCH 55/80] Delete commented out code --- matlab/tools/packageMatlabInterface.m | 73 --------------------------- 1 file changed, 73 deletions(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index b594a39b19a..553733b6a0a 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -57,76 +57,3 @@ opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts); - -% % Copy symlinks to package. -% dylib = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.dylib"); -% so = fullfile(toolboxFolder, "+libmexclass", "+proxy", "*.so*"); -% -% % Get the filenames and file paths of all dylibs within +proxy -% dylibFiles = dir(dylib); -% dylibFileNames = convertCharsToStrings({dylibFiles.name}); -% dylibFilePaths = fullfile({dylibFiles.folder}, dylibFileNames); -% -% % Get the filenames and file paths of all sos within +proxy -% soFiles = dir(so); -% soFileNames = convertCharsToStrings({soFiles.name}); -% soFilePaths = fullfile({soFiles.folder}, soFileNames); -% -% sharedLibFileNames = [dylibFileNames soFileNames]; -% sharedLibFilePaths = [dylibFilePaths soFilePaths]; -% -% % Determine which dylibs and sos were not included in the MLTBX file -% [~, name, ext] = fileparts(opts.ToolboxFiles); -% idx = ~ismember(sharedLibFileNames, name + ext); -% sharedLibrariesToCopy = sharedLibFilePaths(idx); -% -% tmpFolder = fullfile(tempdir, "arrow-matlab"); -% unzip(opts.OutputFile, tmpFolder); -% -% % Get top-level directories and files -% dirContents = dir(tmpFolder); -% dotOrDotDotIdx = ismember({dirContents.name}, [".", ".."]); -% dirContents(dotOrDotDotIdx) = []; -% rootPaths = convertCharsToStrings({dirContents.name}); -% -% delete(opts.OutputFile); -% -% % Copy missing shared libraries to the proper subfolder within fsroot -% sharedLibraryTargetFolder = fullfile(tmpFolder, "fsroot", "+libmexclass", "+proxy"); -% for ii = 1:numel(sharedLibrariesToCopy) -% cmd = compose("cp -P %s %s", sharedLibrariesToCopy(ii), sharedLibraryTargetFolder); -% system(cmd); -% end -% -% manifestFilename = fullfile(tmpFolder, "metadata", "filesystemManifest.xml"); -% parser = matlab.io.xml.dom.Parser; -% manifestXDoc = parser.parseFile(manifestFilename); -% -% fileEntriesNode = manifestXDoc.Children; -% date = datetime("now", TimeZone="UTC", Format="yyyy-MM-dd'T'hh:mm:ss"); -% date = string(date) + "Z"; -% permissions = "0644"; -% -% % Use the first element as a template for the new nodes -% firstElement = fileEntriesNode.getFirstElementChild; -% -% for ii = 1:numel(sharedLibrariesToCopy) -% % Clone the first node because of handle copy-semantics -% fileEntryNode = firstElement.cloneNode(false); -% nameAttributeValue = extractAfter(sharedLibrariesToCopy(ii), toolboxFolder); -% contentAttributeValue = fullfile("/fsroot", nameAttributeValue); -% fileEntryNode.setAttribute("name", nameAttributeValue); -% fileEntryNode.setAttribute("content", contentAttributeValue); -% fileEntryNode.setAttribute("date", date); -% fileEntryNode.setAttribute("permissions", permissions); -% fileEntryNode.setAttribute("type", "File"); -% fileEntriesNode.appendChild(fileEntryNode); -% end -% -% % Export the updated DOM to an XML file -% writer = matlab.io.xml.dom.DOMWriter; -% writeToFile(writer, manifestXDoc, manifestFilename); -% -% % Re-zip the MLTBX source files -% cmd = compose("pushd %s; zip -y -r %s %s; popd", tmpFolder, opts.OutputFile, strjoin(rootPaths)); -% system(cmd); From 8291eee0f84eb34129c7a30089420039d7bb5cf5 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 16:14:56 -0500 Subject: [PATCH 56/80] Delete debug print statements in github workflow file --- dev/tasks/matlab/github.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index ab12e3f3fe7..4384fb207e2 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -32,14 +32,6 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a - - name: find system libstdc++ - run: | - pushd /usr/lib/ - ls -la x86_64-linux-gnu/libstdc++* - ls -la x86_64-linux-gnu/libc.* - strings x86_64-linux-gnu/libstdc++.so.6 | grep -i GLIBC - strings x86_64-linux-gnu/libc.so | grep -i GLIBC - popd - name: Build MATLAB Interface env: {{ macros.github_set_sccache_envvars()|indent(8) }} From 83f1b0e9538866b74dfa5bd94cf6eb77a67d7630 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 16:16:58 -0500 Subject: [PATCH 57/80] Set ubuntu-20.04 as the runner instead of ubuntu-latest --- .github/workflows/matlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index 6921e12213b..b25748d92c5 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -42,7 +42,7 @@ jobs: ubuntu: name: AMD64 Ubuntu 20.04 MATLAB - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: - name: Check out repository From 9dc8940443a4a958b1b6bb58cfe50233563d4656 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 16:17:49 -0500 Subject: [PATCH 58/80] Specify the MaximumMatlabRelease to R2023a --- matlab/tools/packageMatlabInterface.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 553733b6a0a..ddd5b2b6178 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -50,9 +50,9 @@ opts.SupportedPlatforms.Glnxa64 = true; opts.SupportedPlatforms.MatlabOnline = true; -% TODO: Determine what to set the min/max release to +% Interface is only qualified against R2023a at the moment opts.MinimumMatlabRelease = "R2023a"; -opts.MaximumMatlabRelease = ""; +opts.MaximumMatlabRelease = "R2023a"; opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); disp("Output File: " + opts.OutputFile); From 6d057b3a1121015e4ebd6dad34bc424d6c82e377 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 16:19:36 -0500 Subject: [PATCH 59/80] Don't set LD_PRELOAD prior to starting up MATLAB --- .github/workflows/matlab.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index b25748d92c5..ddc759fef1c 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -74,14 +74,6 @@ jobs: run: ci/scripts/matlab_build.sh $(pwd) - name: Run MATLAB Tests env: - # libarrow.so requires a more recent version of libstdc++.so - # than is bundled with MATLAB under /sys/os/glnxa64. - # Therefore, if a MEX function that depends on libarrow.so - # is executed within the MATLAB address space, runtime linking - # errors will occur. To work around this issue, we can explicitly - # force MATLAB to use the system libstdc++.so via LD_PRELOAD. - LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libstdc++.so.6 - # Add the installation directory to the MATLAB Search Path by # setting the MATLABPATH environment variable. MATLABPATH: matlab/install/arrow_matlab From 64a13f6b345af6dbbdd434bf3e5a037a01c26cff Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 15 Nov 2023 16:20:24 -0500 Subject: [PATCH 60/80] Delete post-16-matlab.sh. This script will be added in a separate PR. --- dev/release/post-16-matlab.sh | 57 ----------------------------------- 1 file changed, 57 deletions(-) delete mode 100755 dev/release/post-16-matlab.sh diff --git a/dev/release/post-16-matlab.sh b/dev/release/post-16-matlab.sh deleted file mode 100755 index 4f493a19d5c..00000000000 --- a/dev/release/post-16-matlab.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Use verbose debug output. -set -ex -set -o pipefail - -# Ensure that a version number has been supplied. -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit -fi - -# The version number to use will be the first input argument to the script. -version=$1 - -# Download the MATLAB MLTBX file from JFrog Artifactory to a temp directory. -tmp=$(mktemp -d -t "arrow-post-matlab.XXXXX") -base_url=https://apache.jfrog.io/artifactory/arrow/matlab/${version} -matlab_artifact="matlab-arrow-${version}.mltbx" -curl --fail --location --output ${tmp}/${matlab_artifact} ${base_url}/${matlab_artifact} -github_org="mathworks" -github_repo="arrow" -github_api_endpoint="https://api.github.com/repos/${github_org}/${github_repo}/releases" -github_api_params_template='{"tag_name":"%s","target_commitish":"main","name":"%s","body":"MATLAB Interface to Arrow version %s","draft":false,"prerelease":false,"generate_release_notes":false}' -github_api_params=$(printf "$github_api_params_template" "$version" "$version" "$version") - -# Upload MLTBX file to GitHub releases. -curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_ACCESS_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - ${github_api_endpoint} \ - -d "$github_api_params" - -# Clean up downloaded artifacts. -rm -rf "${tmp}" - -echo "Success! The released MATLAB packages are available here:" -echo " https://github.com/${github_org}/${github_repo}/releases/${version}" From e0f6339c0093cde311bbc276fb6610353a7b66e9 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 16 Nov 2023 10:59:58 -0500 Subject: [PATCH 61/80] Change MLTBX identifer to a UUID --- matlab/tools/packageMatlabInterface.m | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index ddd5b2b6178..9324e4e5248 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -38,8 +38,7 @@ disp("Toolbox Version:" + toolboxVersion); -identifier = "ARROW-MATLAB-TOOLBOX"; - +identifier = "ad1d0fe6-22d1-4969-9e6f-0ab5d0f12ce3"; opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); opts.ToolboxName = "MATLAB Interface to Arrow"; opts.ToolboxVersion = toolboxVersion; From e08319cd13aae44942466d7f917ade45b46c9827 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 16 Nov 2023 11:05:15 -0500 Subject: [PATCH 62/80] 1. Update MLTBX Toolbox Name to MATLAB Arrow Interface 2. Update MLTBX filename to matlab-arrow-{version}.mltbx 3. Change tar filenames from arrow-matlab to matlab-arrow --- dev/tasks/matlab/github.yml | 24 ++++++++++++------------ matlab/tools/packageMatlabInterface.m | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 4384fb207e2..bbdebe10073 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -44,12 +44,12 @@ jobs: patchelf --replace-needed $SYMLINK_ARROW_LIB $REGULAR_ARROW_LIB libarrowproxy.so popd - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-matlab-ubuntu.tar.gz arrow/matlab/install/arrow_matlab + run: tar -cvzf matlab-arrow-ubuntu.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: arrow-matlab-ubuntu.tar.gz - path: arrow-matlab-ubuntu.tar.gz + name: matlab-arrow-ubuntu.tar.gz + path: matlab-arrow-ubuntu.tar.gz macos: name: AMD64 macOS 11 MATLAB @@ -74,12 +74,12 @@ jobs: install_name_tool -change @rpath/$SYMLINK_ARROW_LIB @rpath/$REGULAR_ARROW_LIB libarrowproxy.dylib popd - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-matlab-macos.tar.gz arrow/matlab/install/arrow_matlab + run: tar -cvzf matlab-arrow-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: arrow-matlab-macos.tar.gz - path: arrow-matlab-macos.tar.gz + name: matlab-arrow-macos.tar.gz + path: matlab-arrow-macos.tar.gz windows: name: AMD64 Windows 2022 MATLAB @@ -105,12 +105,12 @@ jobs: bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)/arrow" - name: Compress into single artifact to keep directory structure shell: bash - run: tar -cvzf arrow-matlab-windows.tar.gz arrow/matlab/install/arrow_matlab + run: tar -cvzf matlab-arrow-windows.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: arrow-matlab-windows.tar.gz - path: arrow-matlab-windows.tar.gz + name: matlab-arrow-windows.tar.gz + path: matlab-arrow-windows.tar.gz package-mltbx: name: Package MATLAB Toolbox (MLTBX) Files @@ -128,9 +128,9 @@ jobs: - name: Decompress Artifacts run: | mv artifacts-downloaded/*/*.tar.gz . - tar -xzvf arrow-matlab-ubuntu.tar.gz - tar -xzvf arrow-matlab-macos.tar.gz - tar -xzvf arrow-matlab-windows.tar.gz + tar -xzvf matlab-arrow-ubuntu.tar.gz + tar -xzvf matlab-arrow-macos.tar.gz + tar -xzvf matlab-arrow-windows.tar.gz - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 9324e4e5248..c16255b817a 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -40,7 +40,7 @@ identifier = "ad1d0fe6-22d1-4969-9e6f-0ab5d0f12ce3"; opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); -opts.ToolboxName = "MATLAB Interface to Arrow"; +opts.ToolboxName = "MATLAB Arrow Interface"; opts.ToolboxVersion = toolboxVersion; % Set the SupportedPlatforms From 5d61b571fa9300479b840bfeaf209287117d9151 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 16 Nov 2023 11:16:59 -0500 Subject: [PATCH 63/80] Update step names in dev/tasks/matlab/github.yml file --- dev/tasks/matlab/github.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index bbdebe10073..e37d2fcf8e8 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -36,14 +36,14 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - - name: change shared library install name + - name: Change shared library dependency name run: | pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ SYMLINK_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9]" | xargs basename)" REGULAR_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9].[0-9].[0-9]" | xargs basename)" patchelf --replace-needed $SYMLINK_ARROW_LIB $REGULAR_ARROW_LIB libarrowproxy.so popd - - name: Compress into single artifact to keep directory structure + - name: Compress into single artifact run: tar -cvzf matlab-arrow-ubuntu.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 @@ -66,14 +66,14 @@ jobs: env: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - - name: change shared library install name + - name: Change shared library dependency name run: | pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy SYMLINK_ARROW_LIB="$(find . -regex "./libarrow.[0-9][0-9][0-9][0-9].dylib" | xargs basename)" REGULAR_ARROW_LIB="$(find . -regex "./libarrow.[0-9][0-9][0-9][0-9].[0-9].[0-9].dylib" | xargs basename)" install_name_tool -change @rpath/$SYMLINK_ARROW_LIB @rpath/$REGULAR_ARROW_LIB libarrowproxy.dylib popd - - name: Compress into single artifact to keep directory structure + - name: Compress into single artifact run: tar -cvzf matlab-arrow-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts uses: actions/upload-artifact@v2 @@ -103,7 +103,7 @@ jobs: run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)/arrow" - - name: Compress into single artifact to keep directory structure + - name: Compress into single artifact shell: bash run: tar -cvzf matlab-arrow-windows.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts From 834fd9452597acd31c2f35c87df7e3fad04734dd Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 17 Nov 2023 10:23:09 -0500 Subject: [PATCH 64/80] Simplify expressions setting SYMLINK_ARROW_LIB and REGULAR_ARROR_LIB variables --- dev/tasks/matlab/github.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index e37d2fcf8e8..ebce108f01d 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -39,8 +39,10 @@ jobs: - name: Change shared library dependency name run: | pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ - SYMLINK_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9]" | xargs basename)" - REGULAR_ARROW_LIB="$(find . -regex "./libarrow.so.[0-9][0-9][0-9][0-9].[0-9].[0-9]" | xargs basename)" + SYMLINK_ARROW_LIB="$(find . -name 'libarrow.so.*' -type l | xargs basename)" + REGULAR_ARROW_LIB="$(echo libarrow.so.*.*)" + echo "SYMLINK_ARROW_LIB = ${SYMLINK_ARROW_LIB}" + echo "REGULAR_ARROW_LIB = ${REGULAR_ARROW_LIB}" patchelf --replace-needed $SYMLINK_ARROW_LIB $REGULAR_ARROW_LIB libarrowproxy.so popd - name: Compress into single artifact @@ -69,8 +71,10 @@ jobs: - name: Change shared library dependency name run: | pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy - SYMLINK_ARROW_LIB="$(find . -regex "./libarrow.[0-9][0-9][0-9][0-9].dylib" | xargs basename)" - REGULAR_ARROW_LIB="$(find . -regex "./libarrow.[0-9][0-9][0-9][0-9].[0-9].[0-9].dylib" | xargs basename)" + SYMLINK_ARROW_LIB="$(find . -name 'libarrow.*.dylib' -type l | xargs basename)" + REGULAR_ARROW_LIB="$(echo libarrow.*.*.dylib)" + echo "SYMLINK_ARROW_LIB = ${SYMLINK_ARROW_LIB}" + echo "REGULAR_ARROW_LIB = ${REGULAR_ARROW_LIB}" install_name_tool -change @rpath/$SYMLINK_ARROW_LIB @rpath/$REGULAR_ARROW_LIB libarrowproxy.dylib popd - name: Compress into single artifact From dfd1de38392f0348f84818e95f546a28280dc175 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 17 Nov 2023 10:25:17 -0500 Subject: [PATCH 65/80] Set Toolbox AuthorName to 'Apache Arrow' and AuthorEmail to 'dev@arrow.apache.org' --- matlab/tools/packageMatlabInterface.m | 2 ++ 1 file changed, 2 insertions(+) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index c16255b817a..5832d2fd683 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -42,6 +42,8 @@ opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); opts.ToolboxName = "MATLAB Arrow Interface"; opts.ToolboxVersion = toolboxVersion; +opts.AuthorName = "Apache Arrow"; +opts.AuthorEmail = "dev@arrow.apache.org"; % Set the SupportedPlatforms opts.SupportedPlatforms.Win64 = true; From 45dafd15dffffa2b6ad821da93a16cc73aa12c2a Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 17 Nov 2023 10:32:01 -0500 Subject: [PATCH 66/80] Add comment explaining the intall_name_tool and patchelf commands --- dev/tasks/matlab/github.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index ebce108f01d..357c0b0144c 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -38,6 +38,11 @@ jobs: run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Change shared library dependency name run: | + # MATLAB's programmatic packaging interface does not properly + # include symbolic link files in the package MLTBX - this is a + # bug. As a temporary workaround, change the expected name of the + # Arrow C++ library which libarrowproxy.so depends on. For example, + # change libarrow.so.1500.0.0 to libarrow.so.1500. pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ SYMLINK_ARROW_LIB="$(find . -name 'libarrow.so.*' -type l | xargs basename)" REGULAR_ARROW_LIB="$(echo libarrow.so.*.*)" @@ -70,6 +75,11 @@ jobs: run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Change shared library dependency name run: | + # MATLAB's programmatic packaging interface does not properly + # include symbolic link files in the package MLTBX - this is a + # bug. As a temporary workaround, change the expected name of the + # Arrow C++ library which libarrowproxy.dylib depends on. + # For example, change to libarrow.1500.0.0.dylib to libarrow.1500.dylib. pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy SYMLINK_ARROW_LIB="$(find . -name 'libarrow.*.dylib' -type l | xargs basename)" REGULAR_ARROW_LIB="$(echo libarrow.*.*.dylib)" From ac1c36fed7a1977a1beb6dd36b886d77b451297d Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 17 Nov 2023 10:49:46 -0500 Subject: [PATCH 67/80] Add step to copy LICENSE.txt to toolbox root folder --- dev/tasks/matlab/github.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 357c0b0144c..41f3339b009 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -145,6 +145,8 @@ jobs: tar -xzvf matlab-arrow-ubuntu.tar.gz tar -xzvf matlab-arrow-macos.tar.gz tar -xzvf matlab-arrow-windows.tar.gz + - name: Copy LICENSE.txt to toolbox root folder + run: cp arrow/LICENSE.txt arrow/matlab/install/arrow_matlab/LICENSE.txt - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: From ce52a230bd4365c6dec2be9d23f18580e0c22343 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 17 Nov 2023 11:57:51 -0500 Subject: [PATCH 68/80] Fix comments in dev/tasks/matlab/github.yml file --- dev/tasks/matlab/github.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 41f3339b009..6bf0d2bcdb6 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -37,12 +37,12 @@ jobs: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Change shared library dependency name + # MATLAB's programmatic packaging interface does not properly + # include symbolic link files in the package MLTBX - this is a + # bug. As a temporary workaround, change the expected name of the + # Arrow C++ library which libarrowproxy.so depends on. For example, + # change libarrow.so.1500 to libarrow.so.1500.0.0. run: | - # MATLAB's programmatic packaging interface does not properly - # include symbolic link files in the package MLTBX - this is a - # bug. As a temporary workaround, change the expected name of the - # Arrow C++ library which libarrowproxy.so depends on. For example, - # change libarrow.so.1500.0.0 to libarrow.so.1500. pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ SYMLINK_ARROW_LIB="$(find . -name 'libarrow.so.*' -type l | xargs basename)" REGULAR_ARROW_LIB="$(echo libarrow.so.*.*)" @@ -74,12 +74,12 @@ jobs: {{ macros.github_set_sccache_envvars()|indent(8) }} run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow - name: Change shared library dependency name + # MATLAB's programmatic packaging interface does not properly + # include symbolic link files in the package MLTBX - this is a + # bug. As a temporary workaround, change the expected name of the + # Arrow C++ library which libarrowproxy.dylib depends on. + # For example, change libarrow.1500.dylib to libarrow.1500.0.0.dylib. run: | - # MATLAB's programmatic packaging interface does not properly - # include symbolic link files in the package MLTBX - this is a - # bug. As a temporary workaround, change the expected name of the - # Arrow C++ library which libarrowproxy.dylib depends on. - # For example, change to libarrow.1500.0.0.dylib to libarrow.1500.dylib. pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy SYMLINK_ARROW_LIB="$(find . -name 'libarrow.*.dylib' -type l | xargs basename)" REGULAR_ARROW_LIB="$(echo libarrow.*.*.dylib)" @@ -145,8 +145,6 @@ jobs: tar -xzvf matlab-arrow-ubuntu.tar.gz tar -xzvf matlab-arrow-macos.tar.gz tar -xzvf matlab-arrow-windows.tar.gz - - name: Copy LICENSE.txt to toolbox root folder - run: cp arrow/LICENSE.txt arrow/matlab/install/arrow_matlab/LICENSE.txt - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: From a9cfe644567cdf60383c963136ec04cd2dbd6cbc Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 17 Nov 2023 12:00:53 -0500 Subject: [PATCH 69/80] Remove Download Timezone Database step from workflow --- dev/tasks/matlab/github.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 6bf0d2bcdb6..815b7c642d0 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -104,9 +104,6 @@ jobs: uses: matlab-actions/setup-matlab@v1 with: release: R2023a - - name: Download Timezone Database - shell: bash - run: arrow/ci/scripts/download_tz_database.sh - name: Install sccache shell: bash run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache From 385a524463ae44b3f6811c7d5c6bd010bdfda88f Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 13 Mar 2024 12:16:39 -0400 Subject: [PATCH 70/80] Add pointer to mathworks/libmexclass to LICENSE.txt. Co-authored-by: Sarah Gilmore --- LICENSE.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/LICENSE.txt b/LICENSE.txt index 0423854567b..5ec92626f8d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2252,3 +2252,13 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency mathworks/libmexclass is redistributed as a dynamically +linked shared library in certain binary distributions, like the MATLAB +distribution. + +Copyright: 2022-2024 The MathWorks, Inc. All rights reserved. +Homepage: https://github.com/mathworks/libmexclass +License: 3-clause BSD From f309dd02ea1b3df3299e6029548d0ed8ffe74574 Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 13 Mar 2024 13:54:07 -0400 Subject: [PATCH 71/80] Add note about software from The MathWorks, Inc. being included in the project to the NOTICE.txt file. --- NOTICE.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NOTICE.txt b/NOTICE.txt index a609791374c..f0562362db8 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -82,3 +82,8 @@ its NOTICE file: This product includes software developed by Hewlett-Packard: (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +--------------------------------------------------------------------------------- + +This product includes software from The MathWorks, Inc. (Apache 2.0) + * Copyright (C) 2024 The MathWorks, Inc. From 7d3b429ecdf2b4f6ebf0344cdc959ac1928866ed Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Wed, 13 Mar 2024 14:27:08 -0400 Subject: [PATCH 72/80] Include LICENSE.txt and NOTICE.txt in the MLTBX file --- dev/tasks/matlab/github.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 815b7c642d0..2bc814b5b42 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -142,6 +142,10 @@ jobs: tar -xzvf matlab-arrow-ubuntu.tar.gz tar -xzvf matlab-arrow-macos.tar.gz tar -xzvf matlab-arrow-windows.tar.gz + - name: Copy LICENSE.txt and NOTICE.txt for packaging + run: | + cp arrow/LICENSE.txt arrow/matlab/install/arrow_matlab/LICENSE.txt + cp arrow/NOTICE.txt arrow/matlab/install/arrow_matlab/NOTICE.txt - name: Install MATLAB uses: matlab-actions/setup-matlab@v1 with: From bb0ef54e47e8eb8618afa09a9ada144b1612b50e Mon Sep 17 00:00:00 2001 From: Kevin Gurney Date: Wed, 13 Mar 2024 15:01:05 -0400 Subject: [PATCH 73/80] Add comment to matlab.yml CI workflow explaining why it is useful to explicitly pin the operating system version to `ubuntu-20.04 --- .github/workflows/matlab.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index ddc759fef1c..731c7aca2c2 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -42,6 +42,22 @@ jobs: ubuntu: name: AMD64 Ubuntu 20.04 MATLAB + # Explicitly pin the Ubuntu version to 20.04 for the time being because: + # + # 1. The version of GLIBCXX shipped with Ubuntu 22.04 is not binary compatible + # with the GLIBCXX bundled with MATLAB R2023a. This is a relatively common + # issue. + # + # For example, see: + # + # https://www.mathworks.com/matlabcentral/answers/1907290-how-to-manually-select-the-libstdc-library-to-use-to-resolve-a-version-glibcxx_-not-found + # + # 2. The version of GLIBCXX shipped with Ubuntu 22.04 is not binary compatible with + # the version of GLIBCXX shipped with Debian 11. Several of the Arrow community + # members who work on the MATLAB bindings use Debian 11 locally for qualification. + # Using Ubuntu 20.04 eases development workflows for these community members. + # + # In the future, we can investigate adding support for building against more Linux (e.g. `ubuntu-22.04`) and MATLAB versions (e.g. R2023b). runs-on: ubuntu-20.04 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: From 9d67a2466530c2c384bfdc1f5ab927494780c802 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 14 Mar 2024 09:41:47 -0400 Subject: [PATCH 74/80] Set the MLTBX's AuthorName to The Apache Software Foundation --- matlab/tools/packageMatlabInterface.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 5832d2fd683..25667e7a725 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -42,7 +42,7 @@ opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); opts.ToolboxName = "MATLAB Arrow Interface"; opts.ToolboxVersion = toolboxVersion; -opts.AuthorName = "Apache Arrow"; +opts.AuthorName = "The Apache Software Foundation"; opts.AuthorEmail = "dev@arrow.apache.org"; % Set the SupportedPlatforms From 2c9940623c7eabf03f93b2a18936e1b2bb95dd55 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 14 Mar 2024 09:45:06 -0400 Subject: [PATCH 75/80] Use latest version (v4) of actions/upload-artifacts --- dev/tasks/matlab/github.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 2bc814b5b42..40dd1d119fb 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -53,7 +53,7 @@ jobs: - name: Compress into single artifact run: tar -cvzf matlab-arrow-ubuntu.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: matlab-arrow-ubuntu.tar.gz path: matlab-arrow-ubuntu.tar.gz @@ -90,7 +90,7 @@ jobs: - name: Compress into single artifact run: tar -cvzf matlab-arrow-macos.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: matlab-arrow-macos.tar.gz path: matlab-arrow-macos.tar.gz @@ -118,7 +118,7 @@ jobs: shell: bash run: tar -cvzf matlab-arrow-windows.tar.gz arrow/matlab/install/arrow_matlab - name: Upload artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: matlab-arrow-windows.tar.gz path: matlab-arrow-windows.tar.gz From 9d5d01b525634cb0327c4f062da3d5ad1998a5ed Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 14 Mar 2024 09:47:03 -0400 Subject: [PATCH 76/80] Update macOS job name to AMD64 macOS 12 MATLAB --- .github/workflows/matlab.yml | 2 +- dev/tasks/matlab/github.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index 731c7aca2c2..2c1e69e580b 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -97,7 +97,7 @@ jobs: with: select-by-folder: matlab/test macos: - name: AMD64 macOS 11 MATLAB + name: AMD64 macOS 12 MATLAB runs-on: macos-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} steps: diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 40dd1d119fb..6dc5bd8309b 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -59,7 +59,7 @@ jobs: path: matlab-arrow-ubuntu.tar.gz macos: - name: AMD64 macOS 11 MATLAB + name: AMD64 macOS 12 MATLAB runs-on: macos-latest steps: {{ macros.github_checkout_arrow()|indent }} From 4aae606db1e035de44d986999a701cc2948796b5 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 14 Mar 2024 10:16:38 -0400 Subject: [PATCH 77/80] Dynamically append required text to LICENSE.txt and NOTICE.txt when creating the MLTBX file --- LICENSE.txt | 10 ---------- NOTICE.txt | 5 ----- matlab/tools/packageMatlabInterface.m | 24 ++++++++++++++++++++++++ 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 5ec92626f8d..0423854567b 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2252,13 +2252,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - --------------------------------------------------------------------------------- - -3rdparty dependency mathworks/libmexclass is redistributed as a dynamically -linked shared library in certain binary distributions, like the MATLAB -distribution. - -Copyright: 2022-2024 The MathWorks, Inc. All rights reserved. -Homepage: https://github.com/mathworks/libmexclass -License: 3-clause BSD diff --git a/NOTICE.txt b/NOTICE.txt index f0562362db8..a609791374c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -82,8 +82,3 @@ its NOTICE file: This product includes software developed by Hewlett-Packard: (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P - ---------------------------------------------------------------------------------- - -This product includes software from The MathWorks, Inc. (Apache 2.0) - * Copyright (C) 2024 The MathWorks, Inc. diff --git a/matlab/tools/packageMatlabInterface.m b/matlab/tools/packageMatlabInterface.m index 25667e7a725..55b4d4241a5 100644 --- a/matlab/tools/packageMatlabInterface.m +++ b/matlab/tools/packageMatlabInterface.m @@ -19,6 +19,9 @@ outputFolder = string(getenv("ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER")); toolboxVersionRaw = string(getenv("ARROW_MATLAB_TOOLBOX_VERSION")); +appendLicenseText(fullfile(toolboxFolder, "LICENSE.txt")); +appendNoticeText(fullfile(toolboxFolder, "NOTICE.txt")); + % Output folder must exist. mkdir(outputFolder); @@ -26,6 +29,7 @@ disp("Output Folder: " + outputFolder); disp("Toolbox Version Raw: " + toolboxVersionRaw); + % Note: This string processing heuristic may not be robust to future % changes in the Arrow versioning scheme. dotIdx = strfind(toolboxVersionRaw, "."); @@ -58,3 +62,23 @@ opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); disp("Output File: " + opts.OutputFile); matlab.addons.toolbox.packageToolbox(opts); + +function appendLicenseText(filename) + licenseText = [ ... + newline + "--------------------------------------------------------------------------------" + newline + "3rdparty dependency mathworks/libmexclass is redistributed as a dynamically" + "linked shared library in certain binary distributions, like the MATLAB" + "distribution." + newline + "Copyright: 2022-2024 The MathWorks, Inc. All rights reserved." + "Homepage: https://github.com/mathworks/libmexclass" + "License: 3-clause BSD" ]; + writelines(licenseText, filename, WriteMode="append"); +end + +function appendNoticeText(filename) + noticeText = [ ... + newline + "---------------------------------------------------------------------------------" + newline + "This product includes software from The MathWorks, Inc. (Apache 2.0)" + " * Copyright (C) 2024 The MathWorks, Inc."]; + writelines(noticeText, filename, WriteMode="append"); +end \ No newline at end of file From 43d82780e5ad51edf78e70f909efda59c3b6c05f Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Thu, 14 Mar 2024 13:37:27 -0400 Subject: [PATCH 78/80] Use version v4 for actions/download-artifact --- dev/tasks/matlab/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 6dc5bd8309b..1cd3949efbc 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -133,7 +133,7 @@ jobs: steps: {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts-downloaded - name: Decompress Artifacts From d3026b676b20b70b78ebf35660712b85f21be6e4 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 29 Mar 2024 14:50:55 -0400 Subject: [PATCH 79/80] Remove unnecessary install step that copies libarrow.* to fsroot/+arrow --- matlab/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 47d2acd613f..33b570dda8c 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -232,14 +232,14 @@ if(NOT Arrow_FOUND) # folders that would be empty after installation: # https://gitlab.kitware.com/cmake/cmake/-/issues/17122 - set(CMAKE_PACKAGED_INSTALL_DIR "${CMAKE_INSTALL_DIR}/+arrow") - - install(DIRECTORY "${ARROW_SHARED_LIB_DIR}/" - DESTINATION ${CMAKE_PACKAGED_INSTALL_DIR} - FILES_MATCHING - REGEX ${SHARED_LIBRARY_VERSION_REGEX} - PATTERN "cmake" EXCLUDE - PATTERN "pkgconfig" EXCLUDE) +# set(CMAKE_PACKAGED_INSTALL_DIR "${CMAKE_INSTALL_DIR}/+arrow") + +# install(DIRECTORY "${ARROW_SHARED_LIB_DIR}/" +# DESTINATION ${CMAKE_PACKAGED_INSTALL_DIR} +# FILES_MATCHING +# REGEX ${SHARED_LIBRARY_VERSION_REGEX} +# PATTERN "cmake" EXCLUDE +# PATTERN "pkgconfig" EXCLUDE) endif() # MATLAB_ADD_INSTALL_DIR_TO_STARTUP_FILE toggles whether an addpath command to add the install From 4c4511b6333ffee9fa70cf33ed64e0fe98f1e36b Mon Sep 17 00:00:00 2001 From: Sarah Gilmore Date: Fri, 29 Mar 2024 15:05:05 -0400 Subject: [PATCH 80/80] Delete unnecessary comments in CMakeLists.txt --- matlab/CMakeLists.txt | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 33b570dda8c..bcdde324090 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -201,9 +201,6 @@ get_filename_component(ARROW_SHARED_LIB_DIR ${ARROW_SHARED_LIB} DIRECTORY) get_filename_component(ARROW_SHARED_LIB_FILENAME ${ARROW_SHARED_LIB} NAME_WE) if(NOT Arrow_FOUND) - # If Arrow_FOUND is false, Arrow is built by the arrow_shared target and needs - # to be copied to CMAKE_PACKAGED_INSTALL_DIR. - if(APPLE) # Install libarrow.dylib (symlink) and the real files it points to. # on macOS, we need to match these files: libarrow.dylib @@ -226,20 +223,6 @@ if(NOT Arrow_FOUND) set(SHARED_LIBRARY_VERSION_REGEX ${ARROW_SHARED_LIB_FILENAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) endif() - - # The subfolders cmake and pkgconfig are excluded as they will be empty. - # Note: The following CMake Issue suggests enabling an option to exclude all - # folders that would be empty after installation: - # https://gitlab.kitware.com/cmake/cmake/-/issues/17122 - -# set(CMAKE_PACKAGED_INSTALL_DIR "${CMAKE_INSTALL_DIR}/+arrow") - -# install(DIRECTORY "${ARROW_SHARED_LIB_DIR}/" -# DESTINATION ${CMAKE_PACKAGED_INSTALL_DIR} -# FILES_MATCHING -# REGEX ${SHARED_LIBRARY_VERSION_REGEX} -# PATTERN "cmake" EXCLUDE -# PATTERN "pkgconfig" EXCLUDE) endif() # MATLAB_ADD_INSTALL_DIR_TO_STARTUP_FILE toggles whether an addpath command to add the install