From 14a5d3ee158e4055793d91a4a3654f06a84b4150 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 11 Dec 2025 16:44:42 +0800 Subject: [PATCH 01/12] Add workspace crate dependency graph to doc --- .github/workflows/dependencies.yml | 21 +- Cargo.lock | 4 + Cargo.toml | 1 + datafusion/ci-dummy/Cargo.toml | 34 ++++ datafusion/ci-dummy/src/lib.rs | 33 ++++ docs/scripts/generate_dependency_graph.sh | 135 +++++++++++++ docs/source/_static/data/deps.dot | 116 ++++++++++++ .../architecture/dependency-graph.md | 179 ++++++++++++++++++ docs/source/index.rst | 1 + 9 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 datafusion/ci-dummy/Cargo.toml create mode 100644 datafusion/ci-dummy/src/lib.rs create mode 100755 docs/scripts/generate_dependency_graph.sh create mode 100644 docs/source/_static/data/deps.dot create mode 100644 docs/source/contributor-guide/architecture/dependency-graph.md diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index fef65870b697..560c0e6715df 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -66,4 +66,23 @@ jobs: - name: Install cargo-machete run: cargo install cargo-machete --version ^0.9 --locked - name: Detect unused dependencies - run: cargo machete --with-metadata \ No newline at end of file + run: cargo machete --with-metadata + + dependency-graph: + name: dependency graph up to date + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + submodules: true + fetch-depth: 1 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Install cargo-depgraph + run: cargo install cargo-depgraph --version ^1.10 --locked + - name: Check workspace dependency graph + run: docs/scripts/generate_dependency_graph.sh --check diff --git a/Cargo.lock b/Cargo.lock index 7638249d98ab..0707606e8e33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1970,6 +1970,10 @@ dependencies = [ "object_store", ] +[[package]] +name = "datafusion-ci-dummy" +version = "51.0.0" + [[package]] name = "datafusion-cli" version = "51.0.0" diff --git a/Cargo.toml b/Cargo.toml index 9632d3397d8a..7ddd1d564dc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ members = [ "datafusion-examples/examples/ffi/ffi_module_loader", "test-utils", "benchmarks", + "datafusion/ci-dummy", "datafusion/macros", "datafusion/doc", ] diff --git a/datafusion/ci-dummy/Cargo.toml b/datafusion/ci-dummy/Cargo.toml new file mode 100644 index 000000000000..d5dd0f70a3b8 --- /dev/null +++ b/datafusion/ci-dummy/Cargo.toml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "datafusion-ci-dummy" +description = "Internal dummy crate used to exercise CI checks" +version = { workspace = true } +edition = "2024" +homepage = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +authors = { workspace = true } +rust-version = { workspace = true } +publish = false + +[lints] +workspace = true + +[dependencies] + diff --git a/datafusion/ci-dummy/src/lib.rs b/datafusion/ci-dummy/src/lib.rs new file mode 100644 index 000000000000..dd9291dd77b2 --- /dev/null +++ b/datafusion/ci-dummy/src/lib.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Internal dummy crate to help validate CI wiring. + +/// A tiny helper that makes sure the crate compiles and can be used in tests. +pub const fn ping() -> &'static str { + "pong" +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn responds_with_pong() { + assert_eq!(ping(), "pong"); + } +} diff --git a/docs/scripts/generate_dependency_graph.sh b/docs/scripts/generate_dependency_graph.sh new file mode 100755 index 000000000000..78dafa01ec3e --- /dev/null +++ b/docs/scripts/generate_dependency_graph.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# See `usage()` for details about this script. +# +# The key commands to generate the dependency graph SVG in this script are: +# cargo depgraph ... | dot -Tsvg > deps.svg +# See below for the exact command used. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +OUTPUT_DIR="${REPO_DIR}/docs/source/_static/data" +DOT_OUTPUT="${OUTPUT_DIR}/deps.dot" +SVG_OUTPUT="${OUTPUT_DIR}/deps.svg" + +usage() { + cat <&2 + usage + exit 1 + ;; + esac + shift +done + +if ! command -v cargo >/dev/null 2>&1; then + echo "cargo is required to build the dependency graph." >&2 + exit 1 +fi + +if ! cargo depgraph --help >/dev/null 2>&1; then + echo "cargo-depgraph is required (install with: cargo install cargo-depgraph)." >&2 + exit 1 +fi + +if [[ "${mode}" != "check" ]] && ! command -v dot >/dev/null 2>&1; then + echo "Graphviz 'dot' is required to render the SVG." >&2 + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" + +generate_dot() { + local output_file="$1" + ( + cd "${REPO_DIR}" + cargo depgraph \ + --workspace-only \ + --all-deps \ + --dedup-transitive-deps \ + > "${output_file}" + ) +} + +if [[ "${mode}" == "check" ]]; then + if [[ ! -f "${DOT_OUTPUT}" ]]; then + echo "Expected graph ${DOT_OUTPUT} is missing; regenerate it with $(basename "$0")." >&2 + exit 1 + fi + + temp_dot="$(mktemp)" + cleanup() { rm -f "${temp_dot}"; } + trap cleanup EXIT + + generate_dot "${temp_dot}" + + if ! cmp -s "${temp_dot}" "${DOT_OUTPUT}"; then + echo "Dependency graph is out of date. Re-run $(basename "$0") to refresh ${DOT_OUTPUT} and ${SVG_OUTPUT}." >&2 + exit 1 + fi + + echo "Dependency graph is up to date." + exit 0 +fi + +generate_dot "${DOT_OUTPUT}" + +dot \ + -Grankdir=TB \ + -Gconcentrate=true \ + -Goverlap=false \ + -Tsvg "${DOT_OUTPUT}" \ + > "${SVG_OUTPUT}" + +echo "Wrote dependency graph DOT to ${DOT_OUTPUT}" +echo "Wrote dependency graph SVG to ${SVG_OUTPUT}" diff --git a/docs/source/_static/data/deps.dot b/docs/source/_static/data/deps.dot new file mode 100644 index 000000000000..c95e41a1ebc0 --- /dev/null +++ b/docs/source/_static/data/deps.dot @@ -0,0 +1,116 @@ +digraph { + 0 [ label = "datafusion-common" shape = box] + 1 [ label = "datafusion-common-runtime" shape = box] + 2 [ label = "datafusion-catalog" shape = box] + 3 [ label = "datafusion-datasource" shape = box] + 4 [ label = "datafusion-execution" shape = box] + 5 [ label = "datafusion-expr" shape = box] + 6 [ label = "datafusion-doc" shape = box] + 7 [ label = "datafusion-expr-common" shape = box] + 8 [ label = "datafusion-functions-aggregate-common" shape = box] + 9 [ label = "datafusion-physical-expr-common" shape = box] + 10 [ label = "datafusion-functions-window-common" shape = box] + 11 [ label = "datafusion-physical-expr" shape = box] + 12 [ label = "datafusion-functions" shape = box] + 13 [ label = "datafusion-macros" shape = box, color = green3] + 14 [ label = "datafusion-physical-expr-adapter" shape = box] + 15 [ label = "datafusion-physical-plan" shape = box] + 16 [ label = "datafusion-functions-aggregate" shape = box] + 17 [ label = "datafusion-functions-window" shape = box] + 18 [ label = "datafusion-session" shape = box] + 19 [ label = "datafusion-catalog-listing" shape = box] + 20 [ label = "datafusion-datasource-parquet" shape = box] + 21 [ label = "datafusion-pruning" shape = box] + 22 [ label = "datafusion-functions-nested" shape = box] + 23 [ label = "datafusion-datasource-arrow" shape = box] + 24 [ label = "datafusion-datasource-avro" shape = box] + 25 [ label = "datafusion-datasource-csv" shape = box] + 26 [ label = "datafusion-datasource-json" shape = box] + 27 [ label = "datafusion" shape = box] + 28 [ label = "datafusion-functions-table" shape = box] + 29 [ label = "datafusion-optimizer" shape = box] + 30 [ label = "datafusion-sql" shape = box] + 31 [ label = "datafusion-physical-optimizer" shape = box] + 32 [ label = "test-utils" shape = box] + 33 [ label = "datafusion-ffi" shape = box] + 34 [ label = "datafusion-proto" shape = box] + 35 [ label = "datafusion-proto-common" shape = box] + 36 [ label = "gen" shape = box] + 37 [ label = "gen-common" shape = box] + 38 [ label = "datafusion-spark" shape = box] + 39 [ label = "datafusion-sqllogictest" shape = box] + 40 [ label = "datafusion-substrait" shape = box] + 41 [ label = "datafusion-wasmtest" shape = box] + 42 [ label = "datafusion-cli" shape = box] + 43 [ label = "datafusion-examples" shape = box] + 44 [ label = "ffi_example_table_provider" shape = box] + 45 [ label = "ffi_module_interface" shape = box] + 46 [ label = "ffi_module_loader" shape = box] + 47 [ label = "datafusion-benchmarks" shape = box] + 2 -> 3 [ ] + 3 -> 14 [ ] + 3 -> 18 [ ] + 4 -> 5 [ ] + 5 -> 6 [ ] + 5 -> 8 [ ] + 5 -> 10 [ ] + 7 -> 0 [ ] + 8 -> 9 [ ] + 9 -> 7 [ ] + 10 -> 9 [ ] + 11 -> 12 [ color = blue] + 12 -> 4 [ ] + 12 -> 13 [ color = green3] + 13 -> 6 [ color = green3] + 14 -> 11 [ ] + 15 -> 1 [ ] + 15 -> 16 [ color = blue] + 15 -> 17 [ color = blue] + 16 -> 11 [ ] + 17 -> 11 [ ] + 18 -> 15 [ ] + 19 -> 2 [ ] + 19 -> 20 [ color = blue] + 20 -> 21 [ ] + 21 -> 3 [ ] + 21 -> 22 [ color = blue] + 22 -> 16 [ ] + 23 -> 3 [ ] + 24 -> 3 [ ] + 25 -> 3 [ ] + 26 -> 3 [ ] + 27 -> 19 [ ] + 27 -> 23 [ ] + 27 -> 24 [ style = dotted] + 27 -> 25 [ ] + 27 -> 26 [ ] + 27 -> 28 [ ] + 27 -> 29 [ ] + 27 -> 31 [ ] + 27 -> 31 [ color = blue] + 27 -> 32 [ color = blue] + 28 -> 2 [ ] + 29 -> 30 [ color = blue] + 30 -> 22 [ color = blue] + 30 -> 17 [ color = blue] + 31 -> 21 [ ] + 32 -> 0 [ ] + 33 -> 34 [ ] + 34 -> 27 [ color = blue] + 34 -> 35 [ ] + 35 -> 0 [ ] + 38 -> 2 [ ] + 38 -> 22 [ ] + 39 -> 38 [ ] + 39 -> 40 [ ] + 40 -> 27 [ ] + 40 -> 27 [ color = blue] + 41 -> 27 [ ] + 42 -> 27 [ ] + 43 -> 34 [ color = blue] + 44 -> 45 [ ] + 45 -> 33 [ ] + 46 -> 45 [ ] + 47 -> 34 [ color = blue] +} + diff --git a/docs/source/contributor-guide/architecture/dependency-graph.md b/docs/source/contributor-guide/architecture/dependency-graph.md new file mode 100644 index 000000000000..0631e667d1e5 --- /dev/null +++ b/docs/source/contributor-guide/architecture/dependency-graph.md @@ -0,0 +1,179 @@ + + +# Workspace Dependency Graph + +This page shows the dependency relationships between DataFusion's workspace +crates. This only includes internal dependencies, external crates like `Arrow` are not included + +The dependency graph is auto-genreated through `docs/scripts/generate_dependency_graph.sh` to ensure up-to-date. + +## Dependency Graph for Workspace Crates + + + +```{raw} html +
+
+``` + +```{eval-rst} +.. raw:: html + :file: ../../_static/data/deps.svg +``` + +```{raw} html +
+
+ Interactive SVG (pan, zoom, search) +
+ + +
+ Open SVG ↗ +
+
+ +``` + +### Explanations +- black lines: normal dependency +- blue lines: dev-dependency +- green lines: build-dependency +- dotted lines: optional dependency (could be removed by disabling a cargo feature) + +The dependency graph is generated through `cargo depgraph`, see [cargo-depgraph] for details. + +[cargo-depgraph]: https://github.com/jplatte/cargo-depgraph?tab=readme-ov-file#output-explanation \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index b589c9ce4047..a07f98023b9b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -157,6 +157,7 @@ To get started, see contributor-guide/communication contributor-guide/development_environment contributor-guide/architecture + contributor-guide/architecture/dependency-graph contributor-guide/testing contributor-guide/api-health contributor-guide/howtos From 82f266ea27b408f0f338349a6fd07080e4aae402 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 11 Dec 2025 16:56:33 +0800 Subject: [PATCH 02/12] fix --- datafusion/ci-dummy/Cargo.toml | 1 - dev/rust_lint.sh | 7 +++++++ .../contributor-guide/architecture/dependency-graph.md | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/datafusion/ci-dummy/Cargo.toml b/datafusion/ci-dummy/Cargo.toml index d5dd0f70a3b8..b976da97480c 100644 --- a/datafusion/ci-dummy/Cargo.toml +++ b/datafusion/ci-dummy/Cargo.toml @@ -31,4 +31,3 @@ publish = false workspace = true [dependencies] - diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh index 490720c537cf..b4b1cc128fcc 100755 --- a/dev/rust_lint.sh +++ b/dev/rust_lint.sh @@ -43,9 +43,16 @@ if ! command -v typos &> /dev/null; then cargo install typos-cli --locked fi +# For dependency graph checks +if ! cargo depgraph --help > /dev/null 2>&1; then + echo "Installing cargo-depgraph using cargo" + cargo install cargo-depgraph --version ^1.10 --locked +fi + ci/scripts/rust_fmt.sh ci/scripts/rust_clippy.sh ci/scripts/rust_toml_fmt.sh ci/scripts/rust_docs.sh ci/scripts/license_header.sh ci/scripts/typos_check.sh +docs/scripts/generate_dependency_graph.sh --check diff --git a/docs/source/contributor-guide/architecture/dependency-graph.md b/docs/source/contributor-guide/architecture/dependency-graph.md index 0631e667d1e5..c9a6c9c4fbb1 100644 --- a/docs/source/contributor-guide/architecture/dependency-graph.md +++ b/docs/source/contributor-guide/architecture/dependency-graph.md @@ -22,7 +22,7 @@ This page shows the dependency relationships between DataFusion's workspace crates. This only includes internal dependencies, external crates like `Arrow` are not included -The dependency graph is auto-genreated through `docs/scripts/generate_dependency_graph.sh` to ensure up-to-date. +The dependency graph is auto-generated by `docs/scripts/generate_dependency_graph.sh` to ensure it stays up-to-date. ## Dependency Graph for Workspace Crates From fc18c15de4d36494be31207753211a46b99661c6 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 11 Dec 2025 17:31:03 +0800 Subject: [PATCH 03/12] fix --- .github/workflows/dependencies.yml | 2 +- dev/rust_lint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 560c0e6715df..8674bd46f439 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -83,6 +83,6 @@ jobs: with: rust-version: stable - name: Install cargo-depgraph - run: cargo install cargo-depgraph --version ^1.10 --locked + run: cargo install cargo-depgraph --version ^1.6 --locked - name: Check workspace dependency graph run: docs/scripts/generate_dependency_graph.sh --check diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh index b4b1cc128fcc..5fb76393c2d7 100755 --- a/dev/rust_lint.sh +++ b/dev/rust_lint.sh @@ -46,7 +46,7 @@ fi # For dependency graph checks if ! cargo depgraph --help > /dev/null 2>&1; then echo "Installing cargo-depgraph using cargo" - cargo install cargo-depgraph --version ^1.10 --locked + cargo install cargo-depgraph --version ^1.6 --locked fi ci/scripts/rust_fmt.sh From 4d36a4f165d282d680e6ebdc17d91ab2de791651 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 11 Dec 2025 17:32:59 +0800 Subject: [PATCH 04/12] fix --- .../contributor-guide/architecture/dependency-graph.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/contributor-guide/architecture/dependency-graph.md b/docs/source/contributor-guide/architecture/dependency-graph.md index c9a6c9c4fbb1..a7087ae60ec0 100644 --- a/docs/source/contributor-guide/architecture/dependency-graph.md +++ b/docs/source/contributor-guide/architecture/dependency-graph.md @@ -26,7 +26,7 @@ The dependency graph is auto-generated by `docs/scripts/generate_dependency_grap ## Dependency Graph for Workspace Crates - @@ -169,6 +169,7 @@ The dependency graph is auto-generated by `docs/scripts/generate_dependency_grap ``` ### Explanations + - black lines: normal dependency - blue lines: dev-dependency - green lines: build-dependency @@ -176,4 +177,4 @@ The dependency graph is auto-generated by `docs/scripts/generate_dependency_grap The dependency graph is generated through `cargo depgraph`, see [cargo-depgraph] for details. -[cargo-depgraph]: https://github.com/jplatte/cargo-depgraph?tab=readme-ov-file#output-explanation \ No newline at end of file +[cargo-depgraph]: https://github.com/jplatte/cargo-depgraph?tab=readme-ov-file#output-explanation From 5adc83c1e3d9351a732c97d2008e8aedc127cbba Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 11 Dec 2025 18:00:40 +0800 Subject: [PATCH 05/12] fix --- docs/source/_static/data/deps.svg | 679 ++++++++++++++++++++++++++++++ 1 file changed, 679 insertions(+) create mode 100644 docs/source/_static/data/deps.svg diff --git a/docs/source/_static/data/deps.svg b/docs/source/_static/data/deps.svg new file mode 100644 index 000000000000..9b03e9b53050 --- /dev/null +++ b/docs/source/_static/data/deps.svg @@ -0,0 +1,679 @@ + + + + + + + + + +0 + +datafusion-common + + + +1 + +datafusion-common-runtime + + + +2 + +datafusion-catalog + + + +3 + +datafusion-datasource + + + +2->3 + + + + + +14 + +datafusion-physical-expr-adapter + + + +3->14 + + + + + +18 + +datafusion-session + + + +3->18 + + + + + +4 + +datafusion-execution + + + +5 + +datafusion-expr + + + +4->5 + + + + + +6 + +datafusion-doc + + + +5->6 + + + + + +8 + +datafusion-functions-aggregate-common + + + +5->8 + + + + + +10 + +datafusion-functions-window-common + + + +5->10 + + + + + +7 + +datafusion-expr-common + + + +7->0 + + + + + +9 + +datafusion-physical-expr-common + + + +8->9 + + + + + +9->7 + + + + + +10->9 + + + + + +11 + +datafusion-physical-expr + + + +12 + +datafusion-functions + + + +11->12 + + + + + +12->4 + + + + + +13 + +datafusion-macros + + + +12->13 + + + + + +13->6 + + + + + +14->11 + + + + + +15 + +datafusion-physical-plan + + + +15->1 + + + + + +16 + +datafusion-functions-aggregate + + + +15->16 + + + + + +17 + +datafusion-functions-window + + + +15->17 + + + + + +16->11 + + + + + +17->11 + + + + + +18->15 + + + + + +19 + +datafusion-catalog-listing + + + +19->2 + + + + + +20 + +datafusion-datasource-parquet + + + +19->20 + + + + + +21 + +datafusion-pruning + + + +20->21 + + + + + +21->3 + + + + + +22 + +datafusion-functions-nested + + + +21->22 + + + + + +22->16 + + + + + +23 + +datafusion-datasource-arrow + + + +23->3 + + + + + +24 + +datafusion-datasource-avro + + + +24->3 + + + + + +25 + +datafusion-datasource-csv + + + +25->3 + + + + + +26 + +datafusion-datasource-json + + + +26->3 + + + + + +27 + +datafusion + + + +27->19 + + + + + +27->23 + + + + + +27->24 + + + + + +27->25 + + + + + + +27->26 + + + + + +28 + +datafusion-functions-table + + + +27->28 + + + + + +29 + +datafusion-optimizer + + + +27->29 + + + + + +31 + +datafusion-physical-optimizer + + + +27->31 + + + + + +32 + +test-utils + + + +27->32 + + + + + +28->2 + + + + + +30 + +datafusion-sql + + + +29->30 + + + + + +30->17 + + + + + +30->22 + + + + + +31->21 + + + + + +32->0 + + + + + +33 + +datafusion-ffi + + + +34 + +datafusion-proto + + + +33->34 + + + + + +34->27 + + + + + +35 + +datafusion-proto-common + + + +34->35 + + + + + +35->0 + + + + + +36 + +gen + + + +37 + +gen-common + + + +38 + +datafusion-spark + + + +38->2 + + + + + +38->22 + + + + + +39 + +datafusion-sqllogictest + + + +39->38 + + + + + +40 + +datafusion-substrait + + + +39->40 + + + + + +40->27 + + + + + +41 + +datafusion-wasmtest + + + +41->27 + + + + + +42 + +datafusion-cli + + + +42->27 + + + + + +43 + +datafusion-examples + + + +43->34 + + + + + +44 + +ffi_example_table_provider + + + +45 + +ffi_module_interface + + + +44->45 + + + + + +45->33 + + + + + +46 + +ffi_module_loader + + + +46->45 + + + + + +47 + +datafusion-benchmarks + + + +47->34 + + + + + From 775c79bbecb3ed763a0345de99624b51f36e63f4 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 11 Dec 2025 18:07:23 +0800 Subject: [PATCH 06/12] remove dummy crate (added for CI testing) --- Cargo.lock | 4 ---- Cargo.toml | 1 - datafusion/ci-dummy/Cargo.toml | 33 --------------------------------- datafusion/ci-dummy/src/lib.rs | 33 --------------------------------- 4 files changed, 71 deletions(-) delete mode 100644 datafusion/ci-dummy/Cargo.toml delete mode 100644 datafusion/ci-dummy/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 0707606e8e33..7638249d98ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1970,10 +1970,6 @@ dependencies = [ "object_store", ] -[[package]] -name = "datafusion-ci-dummy" -version = "51.0.0" - [[package]] name = "datafusion-cli" version = "51.0.0" diff --git a/Cargo.toml b/Cargo.toml index 7ddd1d564dc8..9632d3397d8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,6 @@ members = [ "datafusion-examples/examples/ffi/ffi_module_loader", "test-utils", "benchmarks", - "datafusion/ci-dummy", "datafusion/macros", "datafusion/doc", ] diff --git a/datafusion/ci-dummy/Cargo.toml b/datafusion/ci-dummy/Cargo.toml deleted file mode 100644 index b976da97480c..000000000000 --- a/datafusion/ci-dummy/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "datafusion-ci-dummy" -description = "Internal dummy crate used to exercise CI checks" -version = { workspace = true } -edition = "2024" -homepage = { workspace = true } -repository = { workspace = true } -license = { workspace = true } -authors = { workspace = true } -rust-version = { workspace = true } -publish = false - -[lints] -workspace = true - -[dependencies] diff --git a/datafusion/ci-dummy/src/lib.rs b/datafusion/ci-dummy/src/lib.rs deleted file mode 100644 index dd9291dd77b2..000000000000 --- a/datafusion/ci-dummy/src/lib.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Internal dummy crate to help validate CI wiring. - -/// A tiny helper that makes sure the crate compiles and can be used in tests. -pub const fn ping() -> &'static str { - "pong" -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn responds_with_pong() { - assert_eq!(ping(), "pong"); - } -} From a15307af16ddf80f2fd6db3747bf6ae28bd48811 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Mon, 15 Dec 2025 17:14:28 +0800 Subject: [PATCH 07/12] Apply suggestions from code review Co-authored-by: Martin Grigorov --- dev/rust_lint.sh | 2 +- docs/scripts/generate_dependency_graph.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh index d842ae7b4e5a..dd1d1f28440b 100755 --- a/dev/rust_lint.sh +++ b/dev/rust_lint.sh @@ -44,7 +44,7 @@ if ! command -v typos &> /dev/null; then fi # For dependency graph checks -if ! cargo depgraph --help > /dev/null 2>&1; then +if ! command -v cargo-depgraph > /dev/null 2>&1; then echo "Installing cargo-depgraph using cargo" cargo install cargo-depgraph --version ^1.6 --locked fi diff --git a/docs/scripts/generate_dependency_graph.sh b/docs/scripts/generate_dependency_graph.sh index 78dafa01ec3e..16f0661820f0 100755 --- a/docs/scripts/generate_dependency_graph.sh +++ b/docs/scripts/generate_dependency_graph.sh @@ -77,7 +77,7 @@ if ! command -v cargo >/dev/null 2>&1; then exit 1 fi -if ! cargo depgraph --help >/dev/null 2>&1; then +if ! command -v cargo-depgraph > /dev/null 2>&1; then echo "cargo-depgraph is required (install with: cargo install cargo-depgraph)." >&2 exit 1 fi From 307119c9562ecffe1ae7525acfaa4bd57eb4591c Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 16 Dec 2025 10:54:34 +0800 Subject: [PATCH 08/12] review: generate dep graph through CI --- .github/workflows/dependencies.yml | 19 - .github/workflows/docs.yaml | 6 + .github/workflows/docs_pr.yaml | 7 +- docs/.gitignore | 4 + docs/README.md | 5 + docs/build.sh | 9 +- docs/source/_static/data/deps.dot | 116 --- docs/source/_static/data/deps.svg | 679 ------------------ .../architecture/dependency-graph.md | 2 +- 9 files changed, 30 insertions(+), 817 deletions(-) delete mode 100644 docs/source/_static/data/deps.dot delete mode 100644 docs/source/_static/data/deps.svg diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 8674bd46f439..f32eb7d2ddf6 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -67,22 +67,3 @@ jobs: run: cargo install cargo-machete --version ^0.9 --locked - name: Detect unused dependencies run: cargo machete --with-metadata - - dependency-graph: - name: dependency graph up to date - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Install cargo-depgraph - run: cargo install cargo-depgraph --version ^1.6 --locked - - name: Check workspace dependency graph - run: docs/scripts/generate_dependency_graph.sh --check diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 3e2c48643c36..b62055b13b8f 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -51,6 +51,12 @@ jobs: python3 -m venv venv source venv/bin/activate pip install -r docs/requirements.txt + - name: Install dependency graph tooling + run: | + set -x + sudo apt-get update + sudo apt-get install -y graphviz + cargo install cargo-depgraph --version ^1.6 --locked - name: Build docs run: | diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index 81eeb4039ba9..784a33d4c584 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -54,10 +54,15 @@ jobs: python3 -m venv venv source venv/bin/activate pip install -r docs/requirements.txt + - name: Install dependency graph tooling + run: | + set -x + sudo apt-get update + sudo apt-get install -y graphviz + cargo install cargo-depgraph --version ^1.6 --locked - name: Build docs html and check for warnings run: | set -x source venv/bin/activate cd docs ./build.sh # fails on errors - diff --git a/docs/.gitignore b/docs/.gitignore index a3adddc690ab..e73866cc0f35 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -20,3 +20,7 @@ build/ venv/ .python-version __pycache__/ + +# Generated dependency graph artifacts (produced during docs CI) +source/_static/data/deps.dot +source/_static/data/deps.svg diff --git a/docs/README.md b/docs/README.md index c3d87ee8e84a..0340a3b8bf63 100644 --- a/docs/README.md +++ b/docs/README.md @@ -40,6 +40,11 @@ needing to create a virtual environment: uv run --with-requirements requirements.txt bash build.sh ``` +The docs build regenerates the workspace dependency graph via +`docs/scripts/generate_dependency_graph.sh`, so ensure `cargo`, `cargo-depgraph` +(`cargo install cargo-depgraph --version ^1.6 --locked`), and Graphviz `dot` +(`brew install graphviz` or `sudo apt-get install -y graphviz`) are available. + ## Build & Preview Run the provided script to build the HTML pages. diff --git a/docs/build.sh b/docs/build.sh index 9e4a118580ca..e12e3c1a5f20 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -18,7 +18,14 @@ # under the License. # -set -e +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}" + rm -rf build 2> /dev/null +# Keep the workspace dependency graph in sync with the codebase. +scripts/generate_dependency_graph.sh + make html diff --git a/docs/source/_static/data/deps.dot b/docs/source/_static/data/deps.dot deleted file mode 100644 index c95e41a1ebc0..000000000000 --- a/docs/source/_static/data/deps.dot +++ /dev/null @@ -1,116 +0,0 @@ -digraph { - 0 [ label = "datafusion-common" shape = box] - 1 [ label = "datafusion-common-runtime" shape = box] - 2 [ label = "datafusion-catalog" shape = box] - 3 [ label = "datafusion-datasource" shape = box] - 4 [ label = "datafusion-execution" shape = box] - 5 [ label = "datafusion-expr" shape = box] - 6 [ label = "datafusion-doc" shape = box] - 7 [ label = "datafusion-expr-common" shape = box] - 8 [ label = "datafusion-functions-aggregate-common" shape = box] - 9 [ label = "datafusion-physical-expr-common" shape = box] - 10 [ label = "datafusion-functions-window-common" shape = box] - 11 [ label = "datafusion-physical-expr" shape = box] - 12 [ label = "datafusion-functions" shape = box] - 13 [ label = "datafusion-macros" shape = box, color = green3] - 14 [ label = "datafusion-physical-expr-adapter" shape = box] - 15 [ label = "datafusion-physical-plan" shape = box] - 16 [ label = "datafusion-functions-aggregate" shape = box] - 17 [ label = "datafusion-functions-window" shape = box] - 18 [ label = "datafusion-session" shape = box] - 19 [ label = "datafusion-catalog-listing" shape = box] - 20 [ label = "datafusion-datasource-parquet" shape = box] - 21 [ label = "datafusion-pruning" shape = box] - 22 [ label = "datafusion-functions-nested" shape = box] - 23 [ label = "datafusion-datasource-arrow" shape = box] - 24 [ label = "datafusion-datasource-avro" shape = box] - 25 [ label = "datafusion-datasource-csv" shape = box] - 26 [ label = "datafusion-datasource-json" shape = box] - 27 [ label = "datafusion" shape = box] - 28 [ label = "datafusion-functions-table" shape = box] - 29 [ label = "datafusion-optimizer" shape = box] - 30 [ label = "datafusion-sql" shape = box] - 31 [ label = "datafusion-physical-optimizer" shape = box] - 32 [ label = "test-utils" shape = box] - 33 [ label = "datafusion-ffi" shape = box] - 34 [ label = "datafusion-proto" shape = box] - 35 [ label = "datafusion-proto-common" shape = box] - 36 [ label = "gen" shape = box] - 37 [ label = "gen-common" shape = box] - 38 [ label = "datafusion-spark" shape = box] - 39 [ label = "datafusion-sqllogictest" shape = box] - 40 [ label = "datafusion-substrait" shape = box] - 41 [ label = "datafusion-wasmtest" shape = box] - 42 [ label = "datafusion-cli" shape = box] - 43 [ label = "datafusion-examples" shape = box] - 44 [ label = "ffi_example_table_provider" shape = box] - 45 [ label = "ffi_module_interface" shape = box] - 46 [ label = "ffi_module_loader" shape = box] - 47 [ label = "datafusion-benchmarks" shape = box] - 2 -> 3 [ ] - 3 -> 14 [ ] - 3 -> 18 [ ] - 4 -> 5 [ ] - 5 -> 6 [ ] - 5 -> 8 [ ] - 5 -> 10 [ ] - 7 -> 0 [ ] - 8 -> 9 [ ] - 9 -> 7 [ ] - 10 -> 9 [ ] - 11 -> 12 [ color = blue] - 12 -> 4 [ ] - 12 -> 13 [ color = green3] - 13 -> 6 [ color = green3] - 14 -> 11 [ ] - 15 -> 1 [ ] - 15 -> 16 [ color = blue] - 15 -> 17 [ color = blue] - 16 -> 11 [ ] - 17 -> 11 [ ] - 18 -> 15 [ ] - 19 -> 2 [ ] - 19 -> 20 [ color = blue] - 20 -> 21 [ ] - 21 -> 3 [ ] - 21 -> 22 [ color = blue] - 22 -> 16 [ ] - 23 -> 3 [ ] - 24 -> 3 [ ] - 25 -> 3 [ ] - 26 -> 3 [ ] - 27 -> 19 [ ] - 27 -> 23 [ ] - 27 -> 24 [ style = dotted] - 27 -> 25 [ ] - 27 -> 26 [ ] - 27 -> 28 [ ] - 27 -> 29 [ ] - 27 -> 31 [ ] - 27 -> 31 [ color = blue] - 27 -> 32 [ color = blue] - 28 -> 2 [ ] - 29 -> 30 [ color = blue] - 30 -> 22 [ color = blue] - 30 -> 17 [ color = blue] - 31 -> 21 [ ] - 32 -> 0 [ ] - 33 -> 34 [ ] - 34 -> 27 [ color = blue] - 34 -> 35 [ ] - 35 -> 0 [ ] - 38 -> 2 [ ] - 38 -> 22 [ ] - 39 -> 38 [ ] - 39 -> 40 [ ] - 40 -> 27 [ ] - 40 -> 27 [ color = blue] - 41 -> 27 [ ] - 42 -> 27 [ ] - 43 -> 34 [ color = blue] - 44 -> 45 [ ] - 45 -> 33 [ ] - 46 -> 45 [ ] - 47 -> 34 [ color = blue] -} - diff --git a/docs/source/_static/data/deps.svg b/docs/source/_static/data/deps.svg deleted file mode 100644 index 9b03e9b53050..000000000000 --- a/docs/source/_static/data/deps.svg +++ /dev/null @@ -1,679 +0,0 @@ - - - - - - - - - -0 - -datafusion-common - - - -1 - -datafusion-common-runtime - - - -2 - -datafusion-catalog - - - -3 - -datafusion-datasource - - - -2->3 - - - - - -14 - -datafusion-physical-expr-adapter - - - -3->14 - - - - - -18 - -datafusion-session - - - -3->18 - - - - - -4 - -datafusion-execution - - - -5 - -datafusion-expr - - - -4->5 - - - - - -6 - -datafusion-doc - - - -5->6 - - - - - -8 - -datafusion-functions-aggregate-common - - - -5->8 - - - - - -10 - -datafusion-functions-window-common - - - -5->10 - - - - - -7 - -datafusion-expr-common - - - -7->0 - - - - - -9 - -datafusion-physical-expr-common - - - -8->9 - - - - - -9->7 - - - - - -10->9 - - - - - -11 - -datafusion-physical-expr - - - -12 - -datafusion-functions - - - -11->12 - - - - - -12->4 - - - - - -13 - -datafusion-macros - - - -12->13 - - - - - -13->6 - - - - - -14->11 - - - - - -15 - -datafusion-physical-plan - - - -15->1 - - - - - -16 - -datafusion-functions-aggregate - - - -15->16 - - - - - -17 - -datafusion-functions-window - - - -15->17 - - - - - -16->11 - - - - - -17->11 - - - - - -18->15 - - - - - -19 - -datafusion-catalog-listing - - - -19->2 - - - - - -20 - -datafusion-datasource-parquet - - - -19->20 - - - - - -21 - -datafusion-pruning - - - -20->21 - - - - - -21->3 - - - - - -22 - -datafusion-functions-nested - - - -21->22 - - - - - -22->16 - - - - - -23 - -datafusion-datasource-arrow - - - -23->3 - - - - - -24 - -datafusion-datasource-avro - - - -24->3 - - - - - -25 - -datafusion-datasource-csv - - - -25->3 - - - - - -26 - -datafusion-datasource-json - - - -26->3 - - - - - -27 - -datafusion - - - -27->19 - - - - - -27->23 - - - - - -27->24 - - - - - -27->25 - - - - - - -27->26 - - - - - -28 - -datafusion-functions-table - - - -27->28 - - - - - -29 - -datafusion-optimizer - - - -27->29 - - - - - -31 - -datafusion-physical-optimizer - - - -27->31 - - - - - -32 - -test-utils - - - -27->32 - - - - - -28->2 - - - - - -30 - -datafusion-sql - - - -29->30 - - - - - -30->17 - - - - - -30->22 - - - - - -31->21 - - - - - -32->0 - - - - - -33 - -datafusion-ffi - - - -34 - -datafusion-proto - - - -33->34 - - - - - -34->27 - - - - - -35 - -datafusion-proto-common - - - -34->35 - - - - - -35->0 - - - - - -36 - -gen - - - -37 - -gen-common - - - -38 - -datafusion-spark - - - -38->2 - - - - - -38->22 - - - - - -39 - -datafusion-sqllogictest - - - -39->38 - - - - - -40 - -datafusion-substrait - - - -39->40 - - - - - -40->27 - - - - - -41 - -datafusion-wasmtest - - - -41->27 - - - - - -42 - -datafusion-cli - - - -42->27 - - - - - -43 - -datafusion-examples - - - -43->34 - - - - - -44 - -ffi_example_table_provider - - - -45 - -ffi_module_interface - - - -44->45 - - - - - -45->33 - - - - - -46 - -ffi_module_loader - - - -46->45 - - - - - -47 - -datafusion-benchmarks - - - -47->34 - - - - - diff --git a/docs/source/contributor-guide/architecture/dependency-graph.md b/docs/source/contributor-guide/architecture/dependency-graph.md index a7087ae60ec0..dd8ad05b9b93 100644 --- a/docs/source/contributor-guide/architecture/dependency-graph.md +++ b/docs/source/contributor-guide/architecture/dependency-graph.md @@ -22,7 +22,7 @@ This page shows the dependency relationships between DataFusion's workspace crates. This only includes internal dependencies, external crates like `Arrow` are not included -The dependency graph is auto-generated by `docs/scripts/generate_dependency_graph.sh` to ensure it stays up-to-date. +The dependency graph is auto-generated by `docs/scripts/generate_dependency_graph.sh` to ensure it stays up-to-date, and the script now runs automatically as part of `docs/build.sh`. ## Dependency Graph for Workspace Crates From 01f5d5608637b425b2564907e15f7c9a88fc9f4b Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 16 Dec 2025 11:07:37 +0800 Subject: [PATCH 09/12] simplify scripts --- dev/rust_lint.sh | 7 --- docs/scripts/generate_dependency_graph.sh | 72 +++++------------------ 2 files changed, 16 insertions(+), 63 deletions(-) diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh index dd1d1f28440b..21d461184641 100755 --- a/dev/rust_lint.sh +++ b/dev/rust_lint.sh @@ -43,12 +43,6 @@ if ! command -v typos &> /dev/null; then cargo install typos-cli --locked fi -# For dependency graph checks -if ! command -v cargo-depgraph > /dev/null 2>&1; then - echo "Installing cargo-depgraph using cargo" - cargo install cargo-depgraph --version ^1.6 --locked -fi - ci/scripts/rust_fmt.sh ci/scripts/rust_clippy.sh ci/scripts/rust_toml_fmt.sh @@ -56,4 +50,3 @@ ci/scripts/rust_docs.sh ci/scripts/license_header.sh ci/scripts/typos_check.sh ci/scripts/doc_prettier_check.sh -docs/scripts/generate_dependency_graph.sh --check diff --git a/docs/scripts/generate_dependency_graph.sh b/docs/scripts/generate_dependency_graph.sh index 16f0661820f0..266c7c4323fe 100755 --- a/docs/scripts/generate_dependency_graph.sh +++ b/docs/scripts/generate_dependency_graph.sh @@ -28,7 +28,6 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" OUTPUT_DIR="${REPO_DIR}/docs/source/_static/data" -DOT_OUTPUT="${OUTPUT_DIR}/deps.dot" SVG_OUTPUT="${OUTPUT_DIR}/deps.svg" usage() { @@ -37,28 +36,18 @@ Generate the workspace dependency graph SVG for the docs. 'deps.svg' is embedded in the DataFusion docs (Contributor Guide → Architecture → Workspace Dependency Graph). -'deps.dot' is the intermediate DOT used to render deps.svg; CI checks it to ensure the graph is up to date with the codebase. - -Outputs: - DOT: ${DOT_OUTPUT} +Output: SVG: ${SVG_OUTPUT} -Usage: $(basename "$0") [--check] +Usage: $(basename "$0") Options: - --check Validate the checked-in graph without modifying files. -h, --help Show this help message. EOF } -# Default 'generate' mode creates/overrides 'deps.dot' and 'deps.svg', and the 'check' -# mode verify if the existing 'deps.dot' is up-to-date -mode="generate" while [[ $# -gt 0 ]]; do case "$1" in - --check) - mode="check" - ;; -h|--help) usage exit 0 @@ -82,54 +71,25 @@ if ! command -v cargo-depgraph > /dev/null 2>&1; then exit 1 fi -if [[ "${mode}" != "check" ]] && ! command -v dot >/dev/null 2>&1; then +if ! command -v dot >/dev/null 2>&1; then echo "Graphviz 'dot' is required to render the SVG." >&2 exit 1 fi mkdir -p "${OUTPUT_DIR}" -generate_dot() { - local output_file="$1" - ( - cd "${REPO_DIR}" - cargo depgraph \ - --workspace-only \ - --all-deps \ - --dedup-transitive-deps \ - > "${output_file}" - ) -} - -if [[ "${mode}" == "check" ]]; then - if [[ ! -f "${DOT_OUTPUT}" ]]; then - echo "Expected graph ${DOT_OUTPUT} is missing; regenerate it with $(basename "$0")." >&2 - exit 1 - fi - - temp_dot="$(mktemp)" - cleanup() { rm -f "${temp_dot}"; } - trap cleanup EXIT - - generate_dot "${temp_dot}" - - if ! cmp -s "${temp_dot}" "${DOT_OUTPUT}"; then - echo "Dependency graph is out of date. Re-run $(basename "$0") to refresh ${DOT_OUTPUT} and ${SVG_OUTPUT}." >&2 - exit 1 - fi - - echo "Dependency graph is up to date." - exit 0 -fi - -generate_dot "${DOT_OUTPUT}" - -dot \ - -Grankdir=TB \ - -Gconcentrate=true \ - -Goverlap=false \ - -Tsvg "${DOT_OUTPUT}" \ - > "${SVG_OUTPUT}" +( + cd "${REPO_DIR}" + cargo depgraph \ + --workspace-only \ + --all-deps \ + --dedup-transitive-deps \ + | dot \ + -Grankdir=TB \ + -Gconcentrate=true \ + -Goverlap=false \ + -Tsvg \ + > "${SVG_OUTPUT}" +) -echo "Wrote dependency graph DOT to ${DOT_OUTPUT}" echo "Wrote dependency graph SVG to ${SVG_OUTPUT}" From 2276c95647dee4efd672fb5fd622b0b00cbbdf13 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 13 Jan 2026 16:24:30 +0800 Subject: [PATCH 10/12] Update docs/source/contributor-guide/architecture/dependency-graph.md Co-authored-by: Jeffrey Vo --- docs/source/contributor-guide/architecture/dependency-graph.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/architecture/dependency-graph.md b/docs/source/contributor-guide/architecture/dependency-graph.md index dd8ad05b9b93..b55f63d61dbd 100644 --- a/docs/source/contributor-guide/architecture/dependency-graph.md +++ b/docs/source/contributor-guide/architecture/dependency-graph.md @@ -168,7 +168,7 @@ The dependency graph is auto-generated by `docs/scripts/generate_dependency_grap ``` -### Explanations +### Legend - black lines: normal dependency - blue lines: dev-dependency From 6ac0860dce98e6d1063b29925adc821c347e5f80 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 13 Jan 2026 16:38:30 +0800 Subject: [PATCH 11/12] review: ignore gen,gen-common crate; comment no transitive dep --- docs/scripts/generate_dependency_graph.sh | 2 ++ .../source/contributor-guide/architecture/dependency-graph.md | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/scripts/generate_dependency_graph.sh b/docs/scripts/generate_dependency_graph.sh index 266c7c4323fe..7895586acb7d 100755 --- a/docs/scripts/generate_dependency_graph.sh +++ b/docs/scripts/generate_dependency_graph.sh @@ -84,6 +84,8 @@ mkdir -p "${OUTPUT_DIR}" --workspace-only \ --all-deps \ --dedup-transitive-deps \ + # utility crates only used by internal scripts + --exclude gen,gen-common \ | dot \ -Grankdir=TB \ -Gconcentrate=true \ diff --git a/docs/source/contributor-guide/architecture/dependency-graph.md b/docs/source/contributor-guide/architecture/dependency-graph.md index b55f63d61dbd..be3502f48bed 100644 --- a/docs/source/contributor-guide/architecture/dependency-graph.md +++ b/docs/source/contributor-guide/architecture/dependency-graph.md @@ -175,6 +175,6 @@ The dependency graph is auto-generated by `docs/scripts/generate_dependency_grap - green lines: build-dependency - dotted lines: optional dependency (could be removed by disabling a cargo feature) -The dependency graph is generated through `cargo depgraph`, see [cargo-depgraph] for details. +Transitive dependencies are intentionally ignored to keep the graph readable. -[cargo-depgraph]: https://github.com/jplatte/cargo-depgraph?tab=readme-ov-file#output-explanation +The dependency graph is generated through `cargo depgraph` by `docs/scripts/generate_dependency_graph.sh`. From a34853908905d26fe43ac39e5004f448b197c4fb Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 13 Jan 2026 16:46:47 +0800 Subject: [PATCH 12/12] fix --- docs/scripts/generate_dependency_graph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scripts/generate_dependency_graph.sh b/docs/scripts/generate_dependency_graph.sh index 7895586acb7d..771f6f1932c3 100755 --- a/docs/scripts/generate_dependency_graph.sh +++ b/docs/scripts/generate_dependency_graph.sh @@ -80,11 +80,11 @@ mkdir -p "${OUTPUT_DIR}" ( cd "${REPO_DIR}" + # Ignore utility crates only used by internal scripts cargo depgraph \ --workspace-only \ --all-deps \ --dedup-transitive-deps \ - # utility crates only used by internal scripts --exclude gen,gen-common \ | dot \ -Grankdir=TB \