diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77966b3d..dbe43610 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/checkout@v4 - name: Setup Rust uses: leynos/shared-actions/.github/actions/setup-rust@v1.1.0 + - name: Show Ninja version + run: ninja --version - name: Format run: make check-fmt - name: Lint diff --git a/Cargo.lock b/Cargo.lock index 6bc9d9c8..b34da936 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -370,6 +370,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "futures" version = "0.3.31" @@ -475,6 +481,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + [[package]] name = "gherkin" version = "0.14.0" @@ -584,6 +602,18 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" +[[package]] +name = "insta" +version = "1.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "154934ea70c58054b556dd430b99a98c2a7ff5309ac9891597e339b5c28f4371" +dependencies = [ + "console", + "once_cell", + "serde", + "similar", +] + [[package]] name = "inventory" version = "0.3.20" @@ -710,7 +740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] @@ -721,12 +751,15 @@ dependencies = [ "anyhow", "clap", "cucumber", + "insta", + "itertools", "itoa", "rstest", "semver", "serde", "serde_yml", "sha2", + "tempfile", "thiserror", "tokio", ] @@ -850,6 +883,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "regex" version = "1.11.1" @@ -1048,6 +1087,12 @@ dependencies = [ "digest", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "slab" version = "0.4.10" @@ -1121,6 +1166,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "terminal_size" version = "0.4.2" @@ -1260,6 +1318,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -1414,3 +1481,12 @@ name = "windows_x86_64_msvc" version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.1", +] diff --git a/Cargo.toml b/Cargo.toml index d8967ba6..b9c2d888 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ anyhow = "1" thiserror = "1" sha2 = "0.10" itoa = "1" +itertools = "0.12" [lints.clippy] pedantic = { level = "warn", priority = -1 } @@ -57,6 +58,8 @@ float_arithmetic = "deny" rstest = "0.18.0" cucumber = "0.20.0" tokio = { version = "1", features = ["macros", "rt-multi-thread"], default-features = false } +insta = { version = "1", features = ["yaml"] } +tempfile = "3" [[test]] name = "cucumber" diff --git a/docs/netsuke-design.md b/docs/netsuke-design.md index 9820e8e0..7237cce3 100644 --- a/docs/netsuke-design.md +++ b/docs/netsuke-design.md @@ -960,6 +960,49 @@ pub struct BuildEdge { } ``` +```mermaid +classDiagram + class BuildGraph { + +HashMap actions + +HashMap targets + +Vec default_targets + } + class Action { + +Recipe recipe + +Option description + +Option depfile + +Option deps_format + +Option pool + +bool restat + } + class BuildEdge { + +String action_id + +Vec inputs + +Vec explicit_outputs + +Vec implicit_outputs + +Vec order_only_deps + +bool phony + +bool always + } + class Recipe { + <> + Command + Script + Rule + } + class ninja_gen { + +generate(graph: &BuildGraph) String + } + BuildGraph "1" o-- "many" Action : actions + BuildGraph "1" o-- "many" BuildEdge : targets + Action "1" o-- "1" Recipe + BuildEdge "1" --> "1" Action : action_id + ninja_gen ..> BuildGraph : uses + ninja_gen ..> Action : uses + ninja_gen ..> BuildEdge : uses + ninja_gen ..> Recipe : uses +``` + ### 5.3 The Transformation Process: AST to IR The core logic of the validation stage is a function, `ir::from_manifest`, that @@ -1071,6 +1114,13 @@ representation portable. generator reports `IrGenError::MultipleRules` when encountered. - Duplicate output files are rejected. Attempting to define the same output path twice results in `IrGenError::DuplicateOutput`. +- The Ninja generator sorts actions and edges before output and deduplicates + edges based on their full set of explicit outputs. Sorting uses the joined + path strings to keep ordering stable across platforms, ensuring deterministic + `build.ninja` files. Small macros reduce formatting boilerplate when writing + optional key-value pairs or flags, keeping the generator easy to scan. +- Integration tests snapshot the generated Ninja file with `insta` and + execute the Ninja binary to validate structure and no-op behaviour. ## Section 6: Process Management and Secure Execution diff --git a/docs/roadmap.md b/docs/roadmap.md index 8e1dc7b2..dd08fcde 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -55,11 +55,11 @@ compilation pipeline from parsing to execution. - [ ] **Code Generation and Execution:** - - [ ] Implement the Ninja file synthesizer in - [src/ninja_gen.rs](src/ninja_gen.rs) to traverse the BuildGraph IR. + - [x] Implement the Ninja file synthesizer in + [src/ninja_gen.rs](src/ninja_gen.rs) to traverse the BuildGraph IR. *(done)* - - [ ] Write logic to generate Ninja rule statements from ir::Action structs - and build statements from ir::BuildEdge structs. + - [x] Write logic to generate Ninja rule statements from ir::Action structs + and build statements from ir::BuildEdge structs. *(done)* - [ ] Implement the process management logic in `main.rs` to invoke the ninja executable as a subprocess using `std::process::Command`. diff --git a/src/lib.rs b/src/lib.rs index 1999d5e0..adbc03fd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,4 +8,5 @@ pub mod cli; pub mod hasher; pub mod ir; pub mod manifest; +pub mod ninja_gen; pub mod runner; diff --git a/src/ninja_gen.rs b/src/ninja_gen.rs new file mode 100644 index 00000000..04066fc1 --- /dev/null +++ b/src/ninja_gen.rs @@ -0,0 +1,184 @@ +//! Ninja file generator. +//! +//! This module converts a [`crate::ir::BuildGraph`] into the textual +//! representation expected by the Ninja build system. The generator sorts +//! actions and edges to ensure deterministic output for snapshot tests. + +use crate::ast::Recipe; +use crate::ir::{BuildEdge, BuildGraph}; +use itertools::Itertools; +use std::collections::HashSet; +use std::fmt::{self, Display, Formatter, Write}; +use std::path::PathBuf; + +macro_rules! write_kv { + ($f:expr, $key:expr, $opt:expr) => { + if let Some(val) = $opt { + writeln!($f, " {} = {}", $key, val)?; + } + }; +} + +macro_rules! write_flag { + ($f:expr, $key:expr, $cond:expr) => { + if $cond { + writeln!($f, " {} = 1", $key)?; + } + }; +} + +/// Generate a Ninja build file as a string. +/// +/// # Panics +/// +/// Panics if a build edge references an unknown action or if writing to the +/// output string fails (which is unexpected under normal conditions). +#[must_use] +pub fn generate(graph: &BuildGraph) -> String { + let mut out = String::new(); + + let mut actions: Vec<_> = graph.actions.iter().collect(); + actions.sort_by_key(|(id, _)| *id); + for (id, action) in actions { + write!(out, "{}", NamedAction { id, action }).expect("write Ninja rule"); + } + + let mut edges: Vec<_> = graph.targets.values().collect(); + edges.sort_by(|a, b| path_key(&a.explicit_outputs).cmp(&path_key(&b.explicit_outputs))); + let mut seen = HashSet::new(); + for edge in edges { + let key = path_key(&edge.explicit_outputs); + if !seen.insert(key.clone()) { + continue; + } + let action = graph.actions.get(&edge.action_id).expect("action"); + write!( + out, + "{}", + DisplayEdge { + edge, + action_restat: action.restat, + } + ) + .expect("write Ninja edge"); + } + + if !graph.default_targets.is_empty() { + let mut defs = graph.default_targets.clone(); + defs.sort(); + writeln!(out, "default {}", join(&defs)).expect("write defaults"); + } + + out +} + +/// Convert a slice of paths into a space-separated string. +fn join(paths: &[PathBuf]) -> String { + paths.iter().map(|p| p.display()).join(" ") +} + +/// Generate a stable key for a list of paths. +fn path_key(paths: &[PathBuf]) -> String { + let mut parts: Vec<_> = paths.iter().map(|p| p.display().to_string()).collect(); + parts.sort(); + parts.join("\u{0}") +} + +/// Wrapper struct to display a rule with its identifier. +struct NamedAction<'a> { + id: &'a str, + action: &'a crate::ir::Action, +} + +impl Display for NamedAction<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + writeln!(f, "rule {}", self.id)?; + match &self.action.recipe { + Recipe::Command { command } => writeln!(f, " command = {command}")?, + Recipe::Script { script } => { + let escaped = script.replace('\\', "\\\\").replace('"', "\\\""); + writeln!(f, " command = /bin/sh -e -c \"{escaped}\"")?; + } + Recipe::Rule { .. } => unreachable!("rules do not reference other rules"), + } + write_kv!(f, "description", &self.action.description); + write_kv!(f, "depfile", &self.action.depfile); + write_kv!(f, "deps", &self.action.deps_format); + write_kv!(f, "pool", &self.action.pool); + write_flag!(f, "restat", self.action.restat); + writeln!(f) + } +} + +/// Wrapper struct to display a build edge. +struct DisplayEdge<'a> { + edge: &'a BuildEdge, + action_restat: bool, +} + +impl Display for DisplayEdge<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "build {}", join(&self.edge.explicit_outputs))?; + if !self.edge.implicit_outputs.is_empty() { + write!(f, " | {}", join(&self.edge.implicit_outputs))?; + } + let rule = if self.edge.phony { + "phony" + } else { + &self.edge.action_id + }; + write!(f, ": {rule}")?; + if !self.edge.inputs.is_empty() { + write!(f, " {}", join(&self.edge.inputs))?; + } + if !self.edge.order_only_deps.is_empty() { + write!(f, " || {}", join(&self.edge.order_only_deps))?; + } + writeln!(f)?; + write_flag!(f, "restat", self.edge.always && !self.action_restat); + writeln!(f) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::{Action, BuildEdge, BuildGraph}; + use rstest::rstest; + + #[rstest] + fn generate_simple_ninja() { + let action = Action { + recipe: Recipe::Command { + command: "echo hi".into(), + }, + description: None, + depfile: None, + deps_format: None, + pool: None, + restat: false, + }; + let edge = BuildEdge { + action_id: "a".into(), + inputs: vec![PathBuf::from("in")], + explicit_outputs: vec![PathBuf::from("out")], + implicit_outputs: Vec::new(), + order_only_deps: Vec::new(), + phony: false, + always: false, + }; + let mut graph = BuildGraph::default(); + graph.actions.insert("a".into(), action); + graph.targets.insert(PathBuf::from("out"), edge); + graph.default_targets.push(PathBuf::from("out")); + + let ninja = generate(&graph); + let expected = concat!( + "rule a\n", + " command = echo hi\n\n", + "build out: a in\n\n", + "default out\n" + ); + assert_eq!(ninja, expected); + } +} diff --git a/tests/cucumber.rs b/tests/cucumber.rs index 90f57129..56c422d8 100644 --- a/tests/cucumber.rs +++ b/tests/cucumber.rs @@ -7,6 +7,7 @@ pub struct CliWorld { pub manifest: Option, pub manifest_error: Option, pub build_graph: Option, + pub ninja: Option, } mod steps; diff --git a/tests/features/ninja.feature b/tests/features/ninja.feature new file mode 100644 index 00000000..1f98fd0b --- /dev/null +++ b/tests/features/ninja.feature @@ -0,0 +1,12 @@ +Feature: Ninja file generation + + Scenario: Generate build statements + When the manifest file "tests/data/rules.yml" is compiled to IR + And the ninja file is generated + Then the ninja file contains "rule" + And the ninja file contains "build hello.o:" + + Scenario: Phony target rule + When the manifest file "tests/data/phony.yml" is compiled to IR + And the ninja file is generated + Then the ninja file contains "build clean: phony" diff --git a/tests/ninja_gen_tests.rs b/tests/ninja_gen_tests.rs new file mode 100644 index 00000000..e9d7adbe --- /dev/null +++ b/tests/ninja_gen_tests.rs @@ -0,0 +1,120 @@ +//! Unit tests for Ninja file generation. +//! +//! Tests cover various scenarios including phony targets, standard builds +//! with multiple inputs and outputs, complex dependency relationships, and +//! edge cases like empty build graphs. + +use netsuke::ast::Recipe; +use netsuke::ir::{Action, BuildEdge, BuildGraph}; +use netsuke::ninja_gen::generate; +use rstest::rstest; +use std::path::PathBuf; + +#[rstest] +fn generate_phony() { + let action = Action { + recipe: Recipe::Command { + command: "true".into(), + }, + description: None, + depfile: None, + deps_format: None, + pool: None, + restat: false, + }; + let edge = BuildEdge { + action_id: "a".into(), + inputs: vec![PathBuf::from("in")], + explicit_outputs: vec![PathBuf::from("out")], + implicit_outputs: Vec::new(), + order_only_deps: Vec::new(), + phony: true, + always: false, + }; + let mut graph = BuildGraph::default(); + graph.actions.insert("a".into(), action); + graph.targets.insert(PathBuf::from("out"), edge); + + let ninja = generate(&graph); + let expected = concat!( + "rule a\n", + " command = true\n\n", + "build out: phony in\n\n", + ); + assert_eq!(ninja, expected); +} + +#[rstest] +fn generate_standard_build() { + let action = Action { + recipe: Recipe::Command { + command: "cc -c $in -o $out".into(), + }, + description: None, + depfile: None, + deps_format: None, + pool: None, + restat: false, + }; + let edge = BuildEdge { + action_id: "compile".into(), + inputs: vec![PathBuf::from("a.c"), PathBuf::from("b.c")], + explicit_outputs: vec![PathBuf::from("ab.o")], + implicit_outputs: Vec::new(), + order_only_deps: Vec::new(), + phony: false, + always: false, + }; + let mut graph = BuildGraph::default(); + graph.actions.insert("compile".into(), action); + graph.targets.insert(PathBuf::from("ab.o"), edge); + + let ninja = generate(&graph); + let expected = concat!( + "rule compile\n", + " command = cc -c $in -o $out\n\n", + "build ab.o: compile a.c b.c\n\n", + ); + assert_eq!(ninja, expected); +} + +#[rstest] +fn generate_complex_dependencies() { + let action = Action { + recipe: Recipe::Command { + command: "true".into(), + }, + description: None, + depfile: None, + deps_format: None, + pool: None, + restat: false, + }; + let edge = BuildEdge { + action_id: "b".into(), + inputs: vec![PathBuf::from("in")], + explicit_outputs: vec![PathBuf::from("out"), PathBuf::from("log")], + implicit_outputs: vec![PathBuf::from("out.d")], + order_only_deps: vec![PathBuf::from("stamp")], + phony: false, + always: false, + }; + let mut graph = BuildGraph::default(); + graph.actions.insert("b".into(), action); + graph.targets.insert(PathBuf::from("out"), edge); + + let ninja = generate(&graph); + let expected = concat!( + "rule b\n", + " command = true\n\n", + "build out log | out.d: b in || stamp\n\n", + ); + assert_eq!(ninja, expected); +} + +#[rstest] +fn generate_empty_graph() { + let graph = BuildGraph::default(); + let ninja = generate(&graph); + assert!(ninja.is_empty()); +} diff --git a/tests/ninja_snapshot_tests.rs b/tests/ninja_snapshot_tests.rs new file mode 100644 index 00000000..09507325 --- /dev/null +++ b/tests/ninja_snapshot_tests.rs @@ -0,0 +1,81 @@ +//! End-to-end validation of Ninja file generation. +//! +//! These tests generate a Ninja file from a manifest, snapshot the +//! output using `insta`, and validate it with the real `ninja` +//! executable. The manifest uses a simple TOUCH rule so the build is +//! fast and deterministic. + +use insta::{Settings, assert_snapshot}; +use netsuke::{ir::BuildGraph, manifest, ninja_gen}; +use std::{fs, process::Command}; +use tempfile::tempdir; + +fn run_ok(cmd: &mut Command) -> String { + let out = cmd.output().expect("should spawn command"); + assert!( + out.status.success(), + "command failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("stdout utf8") +} + +#[test] +fn touch_manifest_ninja_validation() { + let ninja_check = Command::new("ninja").arg("--version").output(); + if ninja_check.is_err() || !ninja_check.as_ref().expect("spawn ninja").status.success() { + eprintln!("skipping test: ninja must be installed for integration tests"); + return; + } + let manifest_yaml = r#" + netsuke_version: "1.0.0" + rules: + - name: touch + recipe: + kind: command + command: "python3 -c 'import os,sys; open(sys.argv[1],\"a\").close()' $out" + targets: + - name: out/a + sources: in/a + recipe: + kind: rule + rule: touch + "#; + + let manifest = manifest::from_str(manifest_yaml).expect("parse manifest"); + let ir = BuildGraph::from_manifest(&manifest).expect("ir generation"); + let ninja_content = ninja_gen::generate(&ir); + + let mut settings = Settings::new(); + settings.set_snapshot_path(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/snapshots/ninja" + )); + settings.bind(|| { + assert_snapshot!("touch_manifest_ninja", ninja_content); + }); + + let dir = tempdir().expect("tempdir"); + let build_file = dir.path().join("build.ninja"); + fs::write(&build_file, &ninja_content).expect("write ninja"); + fs::create_dir_all(dir.path().join("in")).expect("dir"); + fs::write(dir.path().join("in/a"), "").expect("input"); + + let ninja_cmd = |args: &[&str]| { + let mut cmd = Command::new("ninja"); + cmd.arg("-f").arg(&build_file).args(args); + cmd.current_dir(&dir); + run_ok(&mut cmd) + }; + + let _ = ninja_cmd(&["-t", "rules"]); + let _ = ninja_cmd(&["-t", "targets", "all"]); + let _ = ninja_cmd(&["-t", "query", "out/a"]); + + let _ = ninja_cmd(&["-w", "dupbuild=err", "-d", "stats"]); + let second = ninja_cmd(&["-n", "-d", "explain", "-v"]); + assert!( + second.contains("no work to do"), + "expected no-op second pass, got:\n{second}" + ); +} diff --git a/tests/snapshots/ninja/ninja_snapshot_tests__touch_manifest_ninja.snap b/tests/snapshots/ninja/ninja_snapshot_tests__touch_manifest_ninja.snap new file mode 100644 index 00000000..ca715357 --- /dev/null +++ b/tests/snapshots/ninja/ninja_snapshot_tests__touch_manifest_ninja.snap @@ -0,0 +1,8 @@ +--- +source: tests/ninja_snapshot_tests.rs +expression: ninja_content +--- +rule ca3067639652d0018b982cd2fc8262e3a02f4404f60148b8493de0f656d9b1a2 + command = python3 -c 'import os,sys; open(sys.argv[1],"a").close()' $out + +build out/a: ca3067639652d0018b982cd2fc8262e3a02f4404f60148b8493de0f656d9b1a2 in/a diff --git a/tests/steps/mod.rs b/tests/steps/mod.rs index 8524b752..cb30bcdf 100644 --- a/tests/steps/mod.rs +++ b/tests/steps/mod.rs @@ -1,3 +1,4 @@ mod cli_steps; mod ir_steps; mod manifest_steps; +mod ninja_steps; diff --git a/tests/steps/ninja_steps.rs b/tests/steps/ninja_steps.rs new file mode 100644 index 00000000..71591d79 --- /dev/null +++ b/tests/steps/ninja_steps.rs @@ -0,0 +1,27 @@ +//! Step definitions for Ninja file generation scenarios. + +use crate::CliWorld; +use cucumber::{then, when}; +use netsuke::ninja_gen; + +#[when("the ninja file is generated")] +fn generate_ninja(world: &mut CliWorld) { + let graph = world + .build_graph + .as_ref() + .expect("build graph should be available"); + world.ninja = Some(ninja_gen::generate(graph)); +} + +#[expect( + clippy::needless_pass_by_value, + reason = "Cucumber requires owned String arguments" +)] +#[then(expr = "the ninja file contains {string}")] +fn ninja_contains(world: &mut CliWorld, text: String) { + let ninja = world + .ninja + .as_ref() + .expect("ninja content should be available"); + assert!(ninja.contains(&text)); +}