From f6df7225b66b29147c82eea98dfc268552e5f3bf Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sun, 21 Sep 2025 16:54:12 +0900 Subject: [PATCH 1/5] chore: Action some old TODOs in github actions --- .../actions/setup-macos-builder/action.yaml | 47 ------------------- .../actions/setup-rust-runtime/action.yaml | 14 ++---- .github/workflows/audit.yml | 4 +- .github/workflows/rust.yml | 27 +++-------- 4 files changed, 13 insertions(+), 79 deletions(-) delete mode 100644 .github/actions/setup-macos-builder/action.yaml diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml deleted file mode 100644 index fffdab160b043..0000000000000 --- a/.github/actions/setup-macos-builder/action.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Prepare Rust Builder for MacOS -description: 'Prepare Rust Build Environment for MacOS' -inputs: - rust-version: - description: 'version of rust to install (e.g. stable)' - required: true - default: 'stable' -runs: - using: "composite" - steps: - - name: Install protobuf compiler - shell: bash - run: | - mkdir -p $HOME/d/protoc - cd $HOME/d/protoc - export PROTO_ZIP="protoc-29.1-osx-x86_64.zip" - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v29.1/$PROTO_ZIP - unzip $PROTO_ZIP - echo "$HOME/d/protoc/bin" >> $GITHUB_PATH - export PATH=$PATH:$HOME/d/protoc/bin - protoc --version - - name: Setup Rust toolchain - shell: bash - run: | - rustup update stable - rustup toolchain install stable - rustup default stable - rustup component add rustfmt - - name: Configure rust runtime env - uses: ./.github/actions/setup-rust-runtime diff --git a/.github/actions/setup-rust-runtime/action.yaml b/.github/actions/setup-rust-runtime/action.yaml index b6fb2c898bf2f..eae82728014f3 100644 --- a/.github/actions/setup-rust-runtime/action.yaml +++ b/.github/actions/setup-rust-runtime/action.yaml @@ -20,10 +20,8 @@ description: 'Setup Rust Runtime Environment' runs: using: "composite" steps: - # https://github.com/apache/datafusion/issues/15535 - # disabled because neither version nor git hash works with apache github policy - #- name: Run sccache-cache - # uses: mozilla-actions/sccache-action@65101d47ea8028ed0c98a1cdea8dd9182e9b5133 # v0.0.8 + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Configure runtime env shell: bash # do not produce debug symbols to keep memory usage down @@ -32,11 +30,9 @@ runs: # # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build # See for more details: https://github.com/rust-lang/rust/issues/119560 - # - # readd the following to the run below once sccache-cache is re-enabled - # echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - # echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV - + diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 321dbd36d0c5d..8ecbcf1945254 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -46,6 +46,4 @@ jobs: with: tool: cargo-audit - name: Run audit check - # Ignored until https://github.com/apache/datafusion/issues/15571 - # ignored py03 warning until arrow 55 upgrade - run: cargo audit --ignore RUSTSEC-2024-0370 --ignore RUSTSEC-2025-0020 + run: cargo audit diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5f09e40f0f60a..d0f1fd47c9db2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -308,10 +308,8 @@ jobs: name: cargo test datafusion-cli (amd64) needs: linux-build-lib runs-on: ubuntu-latest - # should be uncommented once https://github.com/apache/datafusion/pull/16644 is merged - # and cache should be added - # container: - # image: amd64/rust + container: + image: amd64/rust steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: @@ -319,6 +317,11 @@ jobs: fetch-depth: 1 - name: Setup Rust toolchain run: rustup toolchain install stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + with: + save-if: false # set in linux-test + shared-key: "amd-ci" - name: Run tests (excluding doctests) env: RUST_BACKTRACE: 1 @@ -533,22 +536,6 @@ jobs: # export PATH=$PATH:$HOME/d/protoc/bin # cargo test --lib --tests --bins --features avro,json,backtrace - # Commenting out intel mac build as so few users would ever use it - # Details: https://github.com/apache/datafusion/issues/13846 - # macos: - # name: cargo test (macos) - # runs-on: macos-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # fetch-depth: 1 - # - name: Setup Rust toolchain - # uses: ./.github/actions/setup-macos-builder - # - name: Run tests (excluding doctests) - # shell: bash - # run: cargo test run --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace - macos-aarch64: name: cargo test (macos-aarch64) runs-on: macos-14 From d7d69064062c7074277b99a569a093869a7b4566 Mon Sep 17 00:00:00 2001 From: Jeffrey Vo Date: Sun, 21 Sep 2025 18:24:58 +1000 Subject: [PATCH 2/5] Update Cargo.toml --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index bbe21f96954f7..07cce25a86a87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -199,6 +199,7 @@ rpath = false strip = false # Retain debug info for flamegraphs [profile.ci] +debug = false inherits = "dev" incremental = false From 930650ea6bd74270f499bb183f15a93596a229c1 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sun, 21 Sep 2025 22:19:45 +0900 Subject: [PATCH 3/5] testing --- datafusion-cli/tests/cli_integration.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index b92b0790babd8..c327e5acc77bb 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -56,7 +56,10 @@ async fn setup_minio_container() -> ContainerAsync { .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER) .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) .with_mount(Mount::bind_mount( - absolute_data_path.to_str().unwrap(), + absolute_data_path + .to_str() + .unwrap() + .replace("/__w/", "/home/runner/work/"), "/source", )) .start() From 729b7dd3ddaf7a47d76f36344e275c530a67114e Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Wed, 24 Sep 2025 10:36:13 +0900 Subject: [PATCH 4/5] Revert changing cli test runner to use container --- .github/workflows/rust.yml | 2 -- datafusion-cli/tests/cli_integration.rs | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b5404d710e020..ab7eace8ee1d1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -308,8 +308,6 @@ jobs: name: cargo test datafusion-cli (amd64) needs: linux-build-lib runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index c327e5acc77bb..b92b0790babd8 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -56,10 +56,7 @@ async fn setup_minio_container() -> ContainerAsync { .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER) .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) .with_mount(Mount::bind_mount( - absolute_data_path - .to_str() - .unwrap() - .replace("/__w/", "/home/runner/work/"), + absolute_data_path.to_str().unwrap(), "/source", )) .start() From f95c9f559f69fdb1f022802a98b53052c20e551a Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Fri, 26 Sep 2025 11:45:27 +0900 Subject: [PATCH 5/5] Remove sccache --- .github/actions/setup-rust-runtime/action.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/actions/setup-rust-runtime/action.yaml b/.github/actions/setup-rust-runtime/action.yaml index eae82728014f3..e0341de93b83d 100644 --- a/.github/actions/setup-rust-runtime/action.yaml +++ b/.github/actions/setup-rust-runtime/action.yaml @@ -20,8 +20,6 @@ description: 'Setup Rust Runtime Environment' runs: using: "composite" steps: - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 - name: Configure runtime env shell: bash # do not produce debug symbols to keep memory usage down @@ -32,7 +30,4 @@ runs: # See for more details: https://github.com/rust-lang/rust/issues/119560 run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV -