From 26e1f7424b413b1e56c49ec73ac4bf11a6291127 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 5 Jul 2025 17:48:44 +0100 Subject: [PATCH 01/14] Add rust CI cache --- .github/workflows/rust.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ecb25483ce07e..581ff29ef6f94 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -51,6 +51,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: false # set in linux-test + shared-key: "amd-ci" - name: Prepare cargo build run: | # Adding `--locked` here to assert that the `Cargo.lock` file is up to @@ -99,6 +104,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: false # set in linux-test + shared-key: "amd-ci" - name: Check datafusion-substrait (default features) run: cargo check --profile ci --all-targets -p datafusion-substrait # @@ -162,6 +172,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: false # set in linux-test + shared-key: "amd-ci" - name: Check datafusion (default features) run: cargo check --profile ci --all-targets -p datafusion # @@ -256,6 +271,11 @@ jobs: run: rustup toolchain install stable - name: Install Protobuf Compiler run: sudo apt-get install -y protobuf-compiler + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref_name == 'main' }} || ${{ github.ref_name == 'ci-cache' }} + shared-key: "amd-ci" - name: Run tests (excluding doctests and datafusion-cli) env: RUST_BACKTRACE: 1 @@ -329,6 +349,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: false # set in linux-test + shared-key: "amd-ci" - name: Run examples run: | # test datafusion-sql examples From 81d1418d8f274833d4bc3ff1916191f5afbff508 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 5 Jul 2025 18:14:13 +0100 Subject: [PATCH 02/14] comment save if for now --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 581ff29ef6f94..24c0a411b1b5d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -274,7 +274,7 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.ref_name == 'main' }} || ${{ github.ref_name == 'ci-cache' }} +# save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci" - name: Run tests (excluding doctests and datafusion-cli) env: From de4bb0aa13820c750bc1691d03f0815bf978b195 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 21:11:22 +0100 Subject: [PATCH 03/14] Add cache to depscache --- .github/workflows/rust.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 24c0a411b1b5d..577ccd7deb2e7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -379,6 +379,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: false # set in linux-test + shared-key: "amd-ci" - name: Run doctests run: cargo test --profile ci --doc --features avro,json - name: Verify Working Directory Clean From 165b288b6da4e72428324b78763d40a233fe57cf Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 21:29:25 +0100 Subject: [PATCH 04/14] Unify the containers --- .github/workflows/rust.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 577ccd7deb2e7..e01d71d2da60a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -262,20 +262,23 @@ jobs: name: cargo test (amd64) needs: linux-build-lib runs-on: ubuntu-latest + container: + image: amd64/rust steps: - uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 - name: Setup Rust toolchain - run: rustup toolchain install stable - - name: Install Protobuf Compiler - run: sudo apt-get install -y protobuf-compiler + uses: ./.github/actions/setup-builder + with: + rust-version: stable - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: # save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci" + env-vars: "" # by default, - name: Run tests (excluding doctests and datafusion-cli) env: RUST_BACKTRACE: 1 @@ -299,13 +302,22 @@ jobs: name: cargo test datafusion-cli (amd64) needs: linux-build-lib runs-on: ubuntu-latest + container: + image: amd64/rust steps: - uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 - name: Setup Rust toolchain - run: rustup toolchain install stable + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: false # set in linux-test + shared-key: "amd-ci" - name: Setup Minio - S3-compatible storage run: | docker run -d --name minio-container \ From 8017c9213f3175d3fa39fcf7d1b9118aa847a7ce Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 21:31:34 +0100 Subject: [PATCH 05/14] linux-build-lib to use its own cache --- .github/workflows/rust.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e01d71d2da60a..4e7dce70ff11c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -54,8 +54,7 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: - save-if: false # set in linux-test - shared-key: "amd-ci" + shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs - name: Prepare cargo build run: | # Adding `--locked` here to assert that the `Cargo.lock` file is up to From 46484e449f509a02959fb3807b999a1df21c556a Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 21:42:18 +0100 Subject: [PATCH 06/14] rollback datafusion-cli for now (blocked by https://github.com/apache/datafusion/pull/16644) --- .github/workflows/rust.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4e7dce70ff11c..72c396bda9793 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -301,6 +301,8 @@ jobs: name: cargo test datafusion-cli (amd64) needs: linux-build-lib runs-on: ubuntu-latest + # should be uncommented once https://github.com/apache/datafusion/pull/16644 is merged + # and cache should be added container: image: amd64/rust steps: @@ -309,14 +311,7 @@ jobs: submodules: true fetch-depth: 1 - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@v2 - with: - save-if: false # set in linux-test - shared-key: "amd-ci" + run: rustup toolchain install stable - name: Setup Minio - S3-compatible storage run: | docker run -d --name minio-container \ From d5b0eaaf589cc99409d4328ba01f0f0f77ec861d Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 21:49:37 +0100 Subject: [PATCH 07/14] Fix CLI comment --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 72c396bda9793..9abf1a9121f4d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -303,8 +303,8 @@ jobs: runs-on: ubuntu-latest # should be uncommented once https://github.com/apache/datafusion/pull/16644 is merged # and cache should be added - container: - image: amd64/rust + #container: + # image: amd64/rust steps: - uses: actions/checkout@v4 with: From 481641a4e44a0fc7b6fa4377e429713dfdb1b735 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 22:31:13 +0100 Subject: [PATCH 08/14] Load cache for `linux-datafusion-common-features` --- .github/workflows/rust.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9abf1a9121f4d..418bf13475c00 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -78,6 +78,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "amd-ci-check" + save-if: 'false' # set in linux-test - name: Check datafusion-common (default features) run: cargo check --profile ci --all-targets -p datafusion-common # From 30986a4f64b701d5d5e7ec8774e60bbc2332b639 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 22:39:47 +0100 Subject: [PATCH 09/14] Setup more caches --- .github/actions/setup-macos-aarch64-builder/action.yaml | 2 ++ .github/workflows/rust.yml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-macos-aarch64-builder/action.yaml b/.github/actions/setup-macos-aarch64-builder/action.yaml index 288799a284b01..06fddebe9507d 100644 --- a/.github/actions/setup-macos-aarch64-builder/action.yaml +++ b/.github/actions/setup-macos-aarch64-builder/action.yaml @@ -45,5 +45,7 @@ runs: rustup component add rustfmt - name: Setup rust cache uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref_name == 'main' }} - name: Configure rust runtime env uses: ./.github/actions/setup-rust-runtime diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 418bf13475c00..33f7fa5fc4f71 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -363,8 +363,8 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: - save-if: false # set in linux-test - shared-key: "amd-ci" + # save-if: ${{ github.ref_name == 'main' }} + shared-key: "amd-ci-linux-test-example" - name: Run examples run: | # test datafusion-sql examples @@ -393,8 +393,8 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: - save-if: false # set in linux-test - shared-key: "amd-ci" + # save-if: ${{ github.ref_name == 'main' }} + shared-key: "amd-ci-linux-test-doc" - name: Run doctests run: cargo test --profile ci --doc --features avro,json - name: Verify Working Directory Clean From 6488f473320514edbfa11ea75cd4dd2ebbbaa7a6 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 22:42:15 +0100 Subject: [PATCH 10/14] Cache for clippy --- .github/workflows/rust.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 33f7fa5fc4f71..be174983cbb0f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -696,6 +696,11 @@ jobs: rust-version: stable - name: Install Clippy run: rustup component add clippy + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: false # set in linux-test + shared-key: "amd-ci-clippy" - name: Run clippy run: ci/scripts/rust_clippy.sh From 53d54edea3821a4ef0a94cdac3a4f375f6af1c7f Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 7 Jul 2025 22:42:36 +0100 Subject: [PATCH 11/14] comment clippy save --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index be174983cbb0f..1559635a99209 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -699,7 +699,7 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: - save-if: false # set in linux-test +# save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci-clippy" - name: Run clippy run: ci/scripts/rust_clippy.sh From 1a87552b40b192bc44174020f753f4985a222008 Mon Sep 17 00:00:00 2001 From: blaginin Date: Thu, 10 Jul 2025 23:22:47 +0100 Subject: [PATCH 12/14] Remove caches that aren't working --- .github/workflows/rust.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1559635a99209..18d4a4843d004 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -78,11 +78,6 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@v2 - with: - shared-key: "amd-ci-check" - save-if: 'false' # set in linux-test - name: Check datafusion-common (default features) run: cargo check --profile ci --all-targets -p datafusion-common # @@ -280,9 +275,8 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: -# save-if: ${{ github.ref_name == 'main' }} + save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci" - env-vars: "" # by default, - name: Run tests (excluding doctests and datafusion-cli) env: RUST_BACKTRACE: 1 @@ -363,7 +357,7 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: - # save-if: ${{ github.ref_name == 'main' }} + save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci-linux-test-example" - name: Run examples run: | @@ -390,11 +384,6 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@v2 - with: - # save-if: ${{ github.ref_name == 'main' }} - shared-key: "amd-ci-linux-test-doc" - name: Run doctests run: cargo test --profile ci --doc --features avro,json - name: Verify Working Directory Clean From 12c1a1823a24598966fb184ce9f3e66c228a6199 Mon Sep 17 00:00:00 2001 From: blaginin Date: Thu, 10 Jul 2025 23:23:33 +0100 Subject: [PATCH 13/14] Save if --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 18d4a4843d004..56d17318c466e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -688,7 +688,7 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@v2 with: -# save-if: ${{ github.ref_name == 'main' }} + save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci-clippy" - name: Run clippy run: ci/scripts/rust_clippy.sh From 4da7d1cdff1f82ddc8a6caa1bb0cd100250f848f Mon Sep 17 00:00:00 2001 From: blaginin Date: Thu, 10 Jul 2025 23:24:33 +0100 Subject: [PATCH 14/14] Save if --- .github/workflows/rust.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 56d17318c466e..dc8f95b4a05db 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -55,6 +55,7 @@ jobs: uses: Swatinem/rust-cache@v2 with: shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs + save-if: ${{ github.ref_name == 'main' }} - name: Prepare cargo build run: | # Adding `--locked` here to assert that the `Cargo.lock` file is up to @@ -302,7 +303,7 @@ jobs: runs-on: ubuntu-latest # should be uncommented once https://github.com/apache/datafusion/pull/16644 is merged # and cache should be added - #container: + # container: # image: amd64/rust steps: - uses: actions/checkout@v4