From 3e74058db0ee57b7da333ff6073d552f179c1711 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 13 Nov 2025 10:21:31 -0800 Subject: [PATCH 1/7] ci: cache rust dependencies --- .github/workflows/python.yml | 14 +------------- .github/workflows/rust.yml | 23 +---------------------- Cargo.toml | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 35 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 358846f68b4..a7b5d9945df 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -23,9 +23,7 @@ env: # This env var is used by Swatinem/rust-cache@v2 for the cache # key, so we set it to make sure it is always consistent. CARGO_TERM_COLOR: always - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=line-tables-only" + CARGO_PROFILE: ci RUST_BACKTRACE: "1" CI: "true" # Color output for pytest is off by default. @@ -60,8 +58,6 @@ jobs: with: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - cache-targets: false - cache-workspace-crates: true - name: Install linting tools run: | pip install ruff==0.11.2 maturin tensorflow tqdm ray[data] pyright datasets polars[pyarrow,pandas] @@ -115,8 +111,6 @@ jobs: with: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - cache-targets: false - cache-workspace-crates: true - uses: ./.github/workflows/build_linux_wheel - uses: ./.github/workflows/run_tests - name: Upload wheels as artifacts @@ -179,8 +173,6 @@ jobs: with: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - cache-targets: false - cache-workspace-crates: true - uses: ./.github/workflows/build_linux_wheel with: arm-build: "true" @@ -234,8 +226,6 @@ jobs: with: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - cache-targets: false - cache-workspace-crates: true - uses: ./.github/workflows/build_windows_wheel - uses: ./.github/workflows/run_tests aws-integtest: @@ -258,8 +248,6 @@ jobs: with: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - cache-targets: false - cache-workspace-crates: true - uses: ./.github/workflows/build_linux_wheel - name: Install dependencies run: | diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 02d4ef4400a..02c220e415a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,14 +20,11 @@ env: # This env var is used by Swatinem/rust-cache@v2 for the cache # key, so we set it to make sure it is always consistent. CARGO_TERM_COLOR: always - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" RUST_BACKTRACE: "1" # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html # CI builds are faster with incremental disabled. CARGO_INCREMENTAL: "0" - CARGO_BUILD_JOBS: "1" + CARGO_PROFILE: ci jobs: format: @@ -47,9 +44,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 - with: - cache-targets: false - cache-workspace-crates: true - name: Install dependencies run: | sudo apt update @@ -92,9 +86,6 @@ jobs: rustup default ${{ matrix.toolchain }} - uses: rui314/setup-mold@v1 - uses: Swatinem/rust-cache@v2 - with: - cache-targets: false - cache-workspace-crates: true - name: Install dependencies run: | sudo apt update @@ -128,9 +119,6 @@ jobs: rustup default stable - uses: rui314/setup-mold@v1 - uses: Swatinem/rust-cache@v2 - with: - cache-targets: false - cache-workspace-crates: true - name: Install dependencies run: | sudo apt -y -qq update @@ -189,9 +177,6 @@ jobs: run: | rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} - uses: Swatinem/rust-cache@v2 - with: - cache-targets: false - cache-workspace-crates: true - name: Build tests run: | cargo test --locked --features fp16kernels,cli,tensorflow,dynamodb,substrait --no-run @@ -209,9 +194,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 - with: - cache-targets: false - cache-workspace-crates: true - name: Install Protoc v21.12 working-directory: C:\ run: | @@ -244,9 +226,6 @@ jobs: with: submodules: true - uses: Swatinem/rust-cache@v2 - with: - cache-targets: false - cache-workspace-crates: true - name: Install dependencies run: | sudo apt update diff --git a/Cargo.toml b/Cargo.toml index 1e241d03370..61f4ebe90ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -193,6 +193,20 @@ opt-level = 3 debug = true strip = false +[profile.ci] +debug = "line-tables-only" +inherits = "dev" +incremental = false + +# This rule applies to every package except workspace members (dependencies +# such as `arrow` and `tokio`). It disables debug info and related features on +# dependencies so their binaries stay smaller, improving cache reuse. +[profile.ci.package."*"] +debug = false +debug-assertions = false +strip = "debuginfo" +incremental = false + [workspace.lints.clippy] all = { level = "deny", priority = -1 } style = { level = "deny", priority = -1 } From b9c46feb854e19eea67afe75cfbe4fd785b793ff Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 13 Nov 2025 11:44:10 -0800 Subject: [PATCH 2/7] pass down profile --- .github/workflows/python.yml | 16 +++++++++++----- .github/workflows/rust.yml | 24 ++++++++++++------------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a7b5d9945df..294a0120496 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -23,7 +23,6 @@ env: # This env var is used by Swatinem/rust-cache@v2 for the cache # key, so we set it to make sure it is always consistent. CARGO_TERM_COLOR: always - CARGO_PROFILE: ci RUST_BACKTRACE: "1" CI: "true" # Color output for pytest is off by default. @@ -75,14 +74,14 @@ jobs: run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` cargo fmt --all -- --check - cargo clippy --locked --features ${ALL_FEATURES} --tests -- -D warnings + cargo clippy --profile ci --locked --features ${ALL_FEATURES} --tests -- -D warnings - name: Build run: | python -m venv venv source venv/bin/activate pip install torch tqdm --index-url https://download.pytorch.org/whl/cpu pip install maturin - maturin develop --locked --extras tests,ray + maturin develop --profile ci --locked --extras tests,ray - name: Run doctest run: | source venv/bin/activate @@ -112,6 +111,8 @@ jobs: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - uses: ./.github/workflows/build_linux_wheel + with: + args: "--profile ci" - uses: ./.github/workflows/run_tests - name: Upload wheels as artifacts if: ${{ matrix.python-minor-version == '13' }} @@ -177,6 +178,7 @@ jobs: with: arm-build: "true" manylinux: "2_28" + args: "--profile ci" - name: Install dependencies run: | sudo apt update -y -qq @@ -203,9 +205,9 @@ jobs: with: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - cache-targets: false - cache-workspace-crates: true - uses: ./.github/workflows/build_mac_wheel + with: + args: "--profile ci" - uses: ./.github/workflows/run_tests with: skip-torch: "true" @@ -227,6 +229,8 @@ jobs: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - uses: ./.github/workflows/build_windows_wheel + with: + args: "--profile ci" - uses: ./.github/workflows/run_tests aws-integtest: timeout-minutes: 45 @@ -249,6 +253,8 @@ jobs: workspaces: python prefix-key: ${{ env.CACHE_PREFIX }} - uses: ./.github/workflows/build_linux_wheel + with: + args: "--profile ci" - name: Install dependencies run: | pip install ray[data] diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 02c220e415a..2270698a9cb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -53,7 +53,7 @@ jobs: ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` echo "ALL_FEATURES=${ALL_FEATURES}" >> $GITHUB_ENV - name: Clippy - run: cargo clippy --locked --features ${{ env.ALL_FEATURES }} --all-targets -- -D warnings + run: cargo clippy --profile ci --locked --features ${{ env.ALL_FEATURES }} --all-targets -- -D warnings cargo-deny: name: Check Rust dependencies (cargo-deny) @@ -98,7 +98,7 @@ jobs: if: ${{ matrix.toolchain == 'stable' }} run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` - cargo llvm-cov --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES} + cargo llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES} - name: Upload coverage to Codecov if: ${{ matrix.toolchain == 'stable' }} uses: codecov/codecov-action@v4 @@ -126,13 +126,13 @@ jobs: - name: Build tests run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` - cargo test --locked --features ${ALL_FEATURES} --no-run + cargo test --profile ci --locked --features ${ALL_FEATURES} --no-run - name: Start DynamodDB and S3 run: docker compose -f docker-compose.yml up -d --wait - name: Run tests run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` - cargo test --locked --features ${ALL_FEATURES} + cargo test --profile ci --locked --features ${ALL_FEATURES} build-no-lock: runs-on: warp-ubuntu-2404-x64-8x timeout-minutes: 30 @@ -153,7 +153,7 @@ jobs: - name: Build all run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` - cargo build --benches --features ${ALL_FEATURES} --tests + cargo build --profile ci --benches --features ${ALL_FEATURES} --tests mac-build: runs-on: "warp-macos-14-arm64-6x" timeout-minutes: 45 @@ -179,13 +179,13 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build tests run: | - cargo test --locked --features fp16kernels,cli,tensorflow,dynamodb,substrait --no-run + cargo test --profile ci --locked --features fp16kernels,cli,tensorflow,dynamodb,substrait --no-run - name: Run tests run: | - cargo test --features fp16kernels,cli,tensorflow,dynamodb,substrait + cargo test --profile ci --features fp16kernels,cli,tensorflow,dynamodb,substrait - name: Check benchmarks run: | - cargo check --benches --features fp16kernels,cli,tensorflow,dynamodb,substrait + cargo check --profile ci --benches --features fp16kernels,cli,tensorflow,dynamodb,substrait windows-build: runs-on: warp-windows-latest-x64-4x defaults: @@ -204,11 +204,11 @@ jobs: Add-Content $env:GITHUB_PATH "C:\protoc\bin" shell: powershell - name: Build tests - run: cargo test --locked --no-run + run: cargo test --profile ci --locked --no-run - name: Run tests - run: cargo test + run: cargo test --profile ci - name: Check benchmarks - run: cargo check --benches + run: cargo check --profile ci --benches msrv: # Check the minimum supported Rust version @@ -237,4 +237,4 @@ jobs: - name: cargo +${{ matrix.msrv }} check run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` - cargo check --workspace --tests --benches --features ${ALL_FEATURES} + cargo check --profile ci --workspace --tests --benches --features ${ALL_FEATURES} From c5765520603ca3508fb2a2684bc2203a01e7b6f3 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 13 Nov 2025 11:48:03 -0800 Subject: [PATCH 3/7] cleanup rust --- .github/workflows/rust.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2270698a9cb..1c80414dc96 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,10 +21,6 @@ env: # key, so we set it to make sure it is always consistent. CARGO_TERM_COLOR: always RUST_BACKTRACE: "1" - # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html - # CI builds are faster with incremental disabled. - CARGO_INCREMENTAL: "0" - CARGO_PROFILE: ci jobs: format: From 2c853dbb2b2be10d0c6ab5bfd369ae95fe266b45 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 13 Nov 2025 11:50:39 -0800 Subject: [PATCH 4/7] add profile for Python --- python/Cargo.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/Cargo.toml b/python/Cargo.toml index 215e5ea4a97..6bb04b7899d 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -79,3 +79,17 @@ datagen = ["lance-datagen"] fp16kernels = ["lance/fp16kernels"] rest = ["lance-namespace-impls/rest"] rest-adapter = ["lance-namespace-impls/rest-adapter"] + +[profile.ci] +debug = "line-tables-only" +inherits = "dev" +incremental = false + +# This rule applies to every package except workspace members (dependencies +# such as `arrow` and `tokio`). It disables debug info and related features on +# dependencies so their binaries stay smaller, improving cache reuse. +[profile.ci.package."*"] +debug = false +debug-assertions = false +strip = "debuginfo" +incremental = false From 61437d0ce93d3d8cb30057c923f9f4c207af4201 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 13 Nov 2025 12:05:02 -0800 Subject: [PATCH 5/7] also do Java --- .github/workflows/java.yml | 13 ++----------- java/lance-jni/Cargo.toml | 14 ++++++++++++++ java/pom.xml | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index b8bc66961ec..44e57ab2a64 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -14,14 +14,7 @@ env: # This env var is used by Swatinem/rust-cache@v2 for the cache # key, so we set it to make sure it is always consistent. CARGO_TERM_COLOR: always - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" RUST_BACKTRACE: "1" - # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html - # CI builds are faster with incremental disabled. - CARGO_INCREMENTAL: "0" - CARGO_BUILD_JOBS: "1" jobs: rust-clippy-fmt: @@ -47,7 +40,7 @@ jobs: run: cargo fmt --check - name: Rust Clippy working-directory: java/lance-jni - run: cargo clippy --all-targets -- -D warnings + run: cargo clippy --profile ci --all-targets -- -D warnings build-and-test-java: runs-on: ubuntu-24.04 @@ -88,8 +81,6 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: java/lance-jni -> ../target/rust-maven-plugin/lance-jni - cache-targets: false - cache-workspace-crates: true - name: Set up Java ${{ matrix.java-version }} uses: actions/setup-java@v4 with: @@ -105,4 +96,4 @@ jobs: env: LANCE_INTEGRATION_TEST: "1" run: | - mvn install + mvn install -Pci diff --git a/java/lance-jni/Cargo.toml b/java/lance-jni/Cargo.toml index 188c3c23ddb..3c32b2812d4 100644 --- a/java/lance-jni/Cargo.toml +++ b/java/lance-jni/Cargo.toml @@ -42,3 +42,17 @@ prost = "0.13.5" roaring = "0.10.1" prost-types = "0.13.5" chrono = "0.4.41" + +[profile.ci] +debug = "line-tables-only" +inherits = "dev" +incremental = false + +# This rule applies to every package except workspace members (dependencies +# such as `arrow` and `tokio`). It disables debug info and related features on +# dependencies so their binaries stay smaller, improving cache reuse. +[profile.ci.package."*"] +debug = false +debug-assertions = false +strip = "debuginfo" +incremental = false \ No newline at end of file diff --git a/java/pom.xml b/java/pom.xml index c40a661b0e0..ad01011e872 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -403,6 +403,38 @@ 1.8 + + ci + + false + + + + + org.questdb + rust-maven-plugin + + + lance-jni + + + --profile=ci + + + + + lance-jni-test + + + --profile=ci + + + + + + + + jdk11+ From 3b58ee6812b72a93fbdc50b9105f96b7e5dc7538 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 13 Nov 2025 12:06:30 -0800 Subject: [PATCH 6/7] see if we fix the earlier issue --- .github/workflows/rust.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1c80414dc96..4324a0ed7af 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -62,8 +62,7 @@ jobs: command: check linux-build: - # Make our CI happy until https://github.com/lancedb/lance/issues/5218 addressed. - runs-on: warp-ubuntu-2404-x64-8x + runs-on: "ubuntu-24.04" timeout-minutes: 60 strategy: matrix: From f68c5d1c832ea5097890993a7799457fb4f89189 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 13 Nov 2025 12:44:11 -0800 Subject: [PATCH 7/7] remove ci profile from java --- .github/workflows/java.yml | 4 ++-- java/lance-jni/Cargo.toml | 5 ++--- java/pom.xml | 32 -------------------------------- 3 files changed, 4 insertions(+), 37 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 44e57ab2a64..27254d70fe2 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -40,7 +40,7 @@ jobs: run: cargo fmt --check - name: Rust Clippy working-directory: java/lance-jni - run: cargo clippy --profile ci --all-targets -- -D warnings + run: cargo clippy --all-targets -- -D warnings build-and-test-java: runs-on: ubuntu-24.04 @@ -96,4 +96,4 @@ jobs: env: LANCE_INTEGRATION_TEST: "1" run: | - mvn install -Pci + mvn install diff --git a/java/lance-jni/Cargo.toml b/java/lance-jni/Cargo.toml index 3c32b2812d4..2cadf6230af 100644 --- a/java/lance-jni/Cargo.toml +++ b/java/lance-jni/Cargo.toml @@ -43,15 +43,14 @@ roaring = "0.10.1" prost-types = "0.13.5" chrono = "0.4.41" -[profile.ci] +[profile.dev] debug = "line-tables-only" -inherits = "dev" incremental = false # This rule applies to every package except workspace members (dependencies # such as `arrow` and `tokio`). It disables debug info and related features on # dependencies so their binaries stay smaller, improving cache reuse. -[profile.ci.package."*"] +[profile.dev.package."*"] debug = false debug-assertions = false strip = "debuginfo" diff --git a/java/pom.xml b/java/pom.xml index ad01011e872..c40a661b0e0 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -403,38 +403,6 @@ 1.8 - - ci - - false - - - - - org.questdb - rust-maven-plugin - - - lance-jni - - - --profile=ci - - - - - lance-jni-test - - - --profile=ci - - - - - - - - jdk11+