GH-38659: [CI][MATLAB][Packaging] Add MATLAB packaging task to crossbow tasks.yml
#38660
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| {% import 'macros.jinja' as macros with context %} | ||
|
|
||
| {{ macros.github_header() }} | ||
|
|
||
| jobs: | ||
|
|
||
| ubuntu: | ||
| name: AMD64 Ubuntu 20.04 MATLAB | ||
| runs-on: ubuntu-20.04 | ||
| steps: | ||
| {{ macros.github_checkout_arrow()|indent }} | ||
| - name: Install ninja-build | ||
| run: sudo apt-get update && sudo apt-get install ninja-build | ||
| - name: Install MATLAB | ||
| uses: matlab-actions/setup-matlab@v1 | ||
| with: | ||
| release: R2023a | ||
| - name: Build MATLAB Interface | ||
| env: | ||
| {{ macros.github_set_sccache_envvars()|indent(8) }} | ||
| run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow | ||
| - name: Change shared library dependency name | ||
| # MATLAB's programmatic packaging interface does not properly | ||
| # include symbolic link files in the package MLTBX - this is a | ||
| # bug. As a temporary workaround, change the expected name of the | ||
| # Arrow C++ library which libarrowproxy.so depends on. For example, | ||
| # change libarrow.so.1500 to libarrow.so.1500.0.0. | ||
| run: | | ||
| pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy/ | ||
kevingurney marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| SYMLINK_ARROW_LIB="$(find . -name 'libarrow.so.*' -type l | xargs basename)" | ||
| REGULAR_ARROW_LIB="$(echo libarrow.so.*.*)" | ||
| echo "SYMLINK_ARROW_LIB = ${SYMLINK_ARROW_LIB}" | ||
| echo "REGULAR_ARROW_LIB = ${REGULAR_ARROW_LIB}" | ||
| patchelf --replace-needed $SYMLINK_ARROW_LIB $REGULAR_ARROW_LIB libarrowproxy.so | ||
| popd | ||
| - name: Compress into single artifact | ||
| run: tar -cvzf matlab-arrow-ubuntu.tar.gz arrow/matlab/install/arrow_matlab | ||
| - name: Upload artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: matlab-arrow-ubuntu.tar.gz | ||
| path: matlab-arrow-ubuntu.tar.gz | ||
|
|
||
| macos: | ||
| name: AMD64 macOS 12 MATLAB | ||
| runs-on: macos-latest | ||
| steps: | ||
| {{ macros.github_checkout_arrow()|indent }} | ||
| - name: Install ninja-build | ||
| run: brew install ninja | ||
| - name: Install MATLAB | ||
| uses: matlab-actions/setup-matlab@v1 | ||
| with: | ||
| release: R2023a | ||
| - name: Build MATLAB Interface | ||
| env: | ||
| {{ macros.github_set_sccache_envvars()|indent(8) }} | ||
| run: arrow/ci/scripts/matlab_build.sh $(pwd)/arrow | ||
| - name: Change shared library dependency name | ||
| # MATLAB's programmatic packaging interface does not properly | ||
| # include symbolic link files in the package MLTBX - this is a | ||
| # bug. As a temporary workaround, change the expected name of the | ||
| # Arrow C++ library which libarrowproxy.dylib depends on. | ||
| # For example, change libarrow.1500.dylib to libarrow.1500.0.0.dylib. | ||
| run: | | ||
| pushd arrow/matlab/install/arrow_matlab/+libmexclass/+proxy | ||
| SYMLINK_ARROW_LIB="$(find . -name 'libarrow.*.dylib' -type l | xargs basename)" | ||
| REGULAR_ARROW_LIB="$(echo libarrow.*.*.dylib)" | ||
| echo "SYMLINK_ARROW_LIB = ${SYMLINK_ARROW_LIB}" | ||
| echo "REGULAR_ARROW_LIB = ${REGULAR_ARROW_LIB}" | ||
| install_name_tool -change @rpath/$SYMLINK_ARROW_LIB @rpath/$REGULAR_ARROW_LIB libarrowproxy.dylib | ||
| popd | ||
| - name: Compress into single artifact | ||
| run: tar -cvzf matlab-arrow-macos.tar.gz arrow/matlab/install/arrow_matlab | ||
| - name: Upload artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: matlab-arrow-macos.tar.gz | ||
| path: matlab-arrow-macos.tar.gz | ||
|
|
||
| windows: | ||
| name: AMD64 Windows 2022 MATLAB | ||
| runs-on: windows-2022 | ||
| steps: | ||
| {{ macros.github_checkout_arrow()|indent }} | ||
| - name: Install MATLAB | ||
| uses: matlab-actions/setup-matlab@v1 | ||
| with: | ||
| release: R2023a | ||
| - name: Install sccache | ||
| shell: bash | ||
| run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache | ||
| - name: Build MATLAB Interface | ||
| shell: cmd | ||
| env: | ||
| {{ macros.github_set_sccache_envvars()|indent(8) }} | ||
| run: | | ||
| call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 | ||
| bash -c "arrow/ci/scripts/matlab_build.sh $(pwd)/arrow" | ||
| - name: Compress into single artifact | ||
| shell: bash | ||
| run: tar -cvzf matlab-arrow-windows.tar.gz arrow/matlab/install/arrow_matlab | ||
| - name: Upload artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: matlab-arrow-windows.tar.gz | ||
| path: matlab-arrow-windows.tar.gz | ||
|
|
||
| package-mltbx: | ||
| name: Package MATLAB Toolbox (MLTBX) Files | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - ubuntu | ||
| - macos | ||
| - windows | ||
| steps: | ||
| {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} | ||
| - name: Download Artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| path: artifacts-downloaded | ||
| - name: Decompress Artifacts | ||
| run: | | ||
| mv artifacts-downloaded/*/*.tar.gz . | ||
| tar -xzvf matlab-arrow-ubuntu.tar.gz | ||
| tar -xzvf matlab-arrow-macos.tar.gz | ||
| tar -xzvf matlab-arrow-windows.tar.gz | ||
| - name: Copy LICENSE.txt and NOTICE.txt for packaging | ||
| run: | | ||
| cp arrow/LICENSE.txt arrow/matlab/install/arrow_matlab/LICENSE.txt | ||
| cp arrow/NOTICE.txt arrow/matlab/install/arrow_matlab/NOTICE.txt | ||
| - name: Install MATLAB | ||
| uses: matlab-actions/setup-matlab@v1 | ||
| with: | ||
| release: R2023a | ||
| - name: Run commands | ||
| env: | ||
| MATLABPATH: arrow/matlab/tools | ||
| ARROW_MATLAB_TOOLBOX_FOLDER: arrow/matlab/install/arrow_matlab | ||
| ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER: artifacts/matlab-dist | ||
| ARROW_MATLAB_TOOLBOX_VERSION: {{ arrow.no_rc_version }} | ||
| uses: matlab-actions/run-command@v1 | ||
| with: | ||
| command: packageMatlabInterface | ||
| {{ macros.github_upload_releases(["artifacts/matlab-dist/*.mltbx"])|indent }} | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| % Licensed to the Apache Software Foundation (ASF) under one | ||
| % or more contributor license agreements. See the NOTICE file | ||
| % distributed with this work for additional information | ||
| % regarding copyright ownership. The ASF licenses this file | ||
| % to you under the Apache License, Version 2.0 (the | ||
| % "License"); you may not use this file except in compliance | ||
| % with the License. You may obtain a copy of the License at | ||
| % | ||
| % http://www.apache.org/licenses/LICENSE-2.0 | ||
| % | ||
| % Unless required by applicable law or agreed to in writing, | ||
| % software distributed under the License is distributed on an | ||
| % "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| % KIND, either express or implied. See the License for the | ||
| % specific language governing permissions and limitations | ||
| % under the License. | ||
|
|
||
| toolboxFolder = string(getenv("ARROW_MATLAB_TOOLBOX_FOLDER")); | ||
| outputFolder = string(getenv("ARROW_MATLAB_TOOLBOX_OUTPUT_FOLDER")); | ||
| toolboxVersionRaw = string(getenv("ARROW_MATLAB_TOOLBOX_VERSION")); | ||
|
|
||
| appendLicenseText(fullfile(toolboxFolder, "LICENSE.txt")); | ||
| appendNoticeText(fullfile(toolboxFolder, "NOTICE.txt")); | ||
|
|
||
| % Output folder must exist. | ||
| mkdir(outputFolder); | ||
|
|
||
| disp("Toolbox Folder: " + toolboxFolder); | ||
| disp("Output Folder: " + outputFolder); | ||
| disp("Toolbox Version Raw: " + toolboxVersionRaw); | ||
|
|
||
|
|
||
| % Note: This string processing heuristic may not be robust to future | ||
| % changes in the Arrow versioning scheme. | ||
| dotIdx = strfind(toolboxVersionRaw, "."); | ||
| numDots = numel(dotIdx); | ||
| if numDots >= 3 | ||
| toolboxVersion = extractBefore(toolboxVersionRaw, dotIdx(3)); | ||
| else | ||
| toolboxVersion = toolboxVersionRaw; | ||
| end | ||
|
|
||
| disp("Toolbox Version:" + toolboxVersion); | ||
|
|
||
| identifier = "ad1d0fe6-22d1-4969-9e6f-0ab5d0f12ce3"; | ||
| opts = matlab.addons.toolbox.ToolboxOptions(toolboxFolder, identifier); | ||
|
Member
There was a problem hiding this comment. Can we set Can we set license information too?
Member
There was a problem hiding this comment. Hi @kou, I'll update the script to set Regarding including the license, the following repository lists best practices for MATLAB Toolboxes: mathworks/toolboxdesign. The recommendation is to put a Best,
Member
There was a problem hiding this comment. Yes. We must add See also:
Member
There was a problem hiding this comment. Hi @kou, Thanks for linking all of this information to us! We'll read through these documents and respond on the PR as soon as we can. Best,
Member
There was a problem hiding this comment. Hi @kou, Just wanted to provide a quick status update on this. We've been working on trying to understand what we need to include in LICENSE.txt/NOTICE.txt from a MEX perspective. We know we need to include a pointer about libmexclass, but we're still evaluating if there is more we should be including relating to MEX build artifacts. Note, we are approaching the holiday season in the US, so there may be a bit of a delay in moving this PR forward. Apologies for any inconvenience this delay causes. Thanks for helping us with this PR!
Member
There was a problem hiding this comment. Hi Everyone, Thanks very much for patiently waiting on us to move this forward. We realized that we need to ensure that the licenses of any MEX artifacts bundled with the released MLTBX file are compatible with the ASF 3rd Party License Policy. After several rounds of discussion with some colleagues at MathWorks (unfortunately delayed due to the holiday season in the US), we came up with a license (attached to this comment) that we hope will allow us to ensure that we are abiding by the ASF's 3rd Party License Policy. The attached license is a modified version of the standard MathWorks license that applies to MEX files. The intent of these modifications is to meet the requirements for inclusion as a "Category B" license according to the ASF 3rd Party License Policy. Our goal is to make sure we are doing the right thing here, and we understand this may need further input from ASF Legal and the rest of the community. Please let us know what we can do to help move this forward. We sincerely appreciate everyone's support as we navigate these licensing requirements. Link to proposed license: license.txt Best, Sarah and Kevin
Member
There was a problem hiding this comment. Thanks for sharing this! Let's discuss this on
Member
There was a problem hiding this comment. I agree! I'll start a discussion on the mailing list. Thanks for the suggestion!
Member
There was a problem hiding this comment. Just an fyi, we now have a path forward. We've determined that we can license MEX binaries and all contents included within the MLTBX file under the Apache V2 license. Here's a link to the mailing list discussion. Now that we are unblocked, Kevin and I will start working on this pull request again. It took some time for us to work through the details with our colleagues at MathWorks and we appreciate everyone's patience.
Member
Author
There was a problem hiding this comment. After reviewing the relevant sections from https://www.apache.org/legal/release-policy.html and https://infra.apache.org/licensing-howto.html, we updated the
We believe these updates should be sufficient to address the bundling of the Thanks again for everyone's help! |
||
| opts.ToolboxName = "MATLAB Arrow Interface"; | ||
| opts.ToolboxVersion = toolboxVersion; | ||
| opts.AuthorName = "The Apache Software Foundation"; | ||
| opts.AuthorEmail = "dev@arrow.apache.org"; | ||
|
|
||
| % Set the SupportedPlatforms | ||
| opts.SupportedPlatforms.Win64 = true; | ||
| opts.SupportedPlatforms.Maci64 = true; | ||
| opts.SupportedPlatforms.Glnxa64 = true; | ||
| opts.SupportedPlatforms.MatlabOnline = true; | ||
|
|
||
| % Interface is only qualified against R2023a at the moment | ||
| opts.MinimumMatlabRelease = "R2023a"; | ||
| opts.MaximumMatlabRelease = "R2023a"; | ||
|
|
||
| opts.OutputFile = fullfile(outputFolder, compose("matlab-arrow-%s.mltbx", toolboxVersionRaw)); | ||
| disp("Output File: " + opts.OutputFile); | ||
| matlab.addons.toolbox.packageToolbox(opts); | ||
|
|
||
| function appendLicenseText(filename) | ||
| licenseText = [ ... | ||
| newline + "--------------------------------------------------------------------------------" + newline | ||
| "3rdparty dependency mathworks/libmexclass is redistributed as a dynamically" | ||
| "linked shared library in certain binary distributions, like the MATLAB" | ||
| "distribution." + newline | ||
| "Copyright: 2022-2024 The MathWorks, Inc. All rights reserved." | ||
| "Homepage: https://github.com/mathworks/libmexclass" | ||
| "License: 3-clause BSD" ]; | ||
| writelines(licenseText, filename, WriteMode="append"); | ||
| end | ||
|
|
||
| function appendNoticeText(filename) | ||
| noticeText = [ ... | ||
| newline + "---------------------------------------------------------------------------------" + newline | ||
| "This product includes software from The MathWorks, Inc. (Apache 2.0)" | ||
| " * Copyright (C) 2024 The MathWorks, Inc."]; | ||
| writelines(noticeText, filename, WriteMode="append"); | ||
| end | ||
Uh oh!
There was an error while loading. Please reload this page.