feat(docs): warn-or-allowlist for non-rivet files in scanned dirs (Task #56)

rivet-cli/src/docs.rs

@@ -325,11 +325,25 @@ sources:                    # Artifact sources
     #   key: value
 
 docs:                       # Documentation directories (for [[ID]] scanning)
-  - docs
+  - docs                    # legacy: just a path
+  - path: arch              # detailed: path + opt-out allowlist
+    exclude:                # silently skip these (still scanned otherwise)
+      - "generated/**"      # `**` matches any subtree
+      - "*.draft.md"        # bare patterns match the file name only
 
 results: results            # Test results directory (JUnit XML, LCOV)
 ```
 
+### Loud-by-default doc scanning
+
+The doc scanner emits a stderr warning for every `.md` file it declines
+(no YAML front-matter, malformed front-matter). This is by design:
+silently-skipped files don't participate in the link graph, so artifact
+IDs in their prose go invisible. The warning includes a hint to add the
+file to `docs[].exclude` if the silence was intentional. A summary line
+at the end of the scan reports `<loaded> loaded, <warned> skipped,
+<excluded> excluded by allowlist`.
+
 ## Available Schemas
 
 | Name           | Types | Description                              |

rivet-cli/src/main.rs

@@ -3653,7 +3653,7 @@ fn cmd_init_agents(cli: &Cli, migrate: bool, force_regen: bool) -> Result<bool>
         config
             .docs
             .iter()
-            .map(|d| format!("`{}`", d))
+            .map(|d| format!("`{}`", d.path()))
             .collect::<Vec<_>>()
             .join(", ")
     };
@@ -6880,9 +6880,7 @@ fn cmd_docs(
 /// Run `rivet docs check` — assert documentation matches reality.
 fn cmd_docs_check(cli: &Cli, format: &str, fix: bool) -> Result<bool> {
     use clap::CommandFactory;
-    use rivet_core::doc_check::{
-        DocCheckContext, apply_fixes, collect_docs, default_invariants, run_all,
-    };
+    use rivet_core::doc_check::{DocCheckContext, apply_fixes, default_invariants, run_all};
     use std::collections::BTreeSet;
 
     validate_format(format, &["text", "json"])?;
@@ -6897,9 +6895,19 @@ fn cmd_docs_check(cli: &Cli, format: &str, fix: bool) -> Result<bool> {
     // silently misses every markdown file outside the top-level `docs/`.
     // Missing or unreadable config degrades to the default `docs/` scan.
     let project_config = rivet_core::load_project_config(&project_root.join("rivet.yaml")).ok();
-    let extra_doc_dirs: Vec<std::path::PathBuf> = project_config
+    let scan_roots: Vec<rivet_core::doc_check::DocScanRoot> = project_config
         .as_ref()
-        .map(|c| c.docs.iter().map(std::path::PathBuf::from).collect())
+        .map(|c| {
+            c.docs
+                .iter()
+                .map(|e| {
+                    rivet_core::doc_check::DocScanRoot::with_exclude(
+                        std::path::PathBuf::from(e.path()),
+                        e.exclude().to_vec(),
+                    )
+                })
+                .collect()
+        })
         .unwrap_or_default();
     let external_namespaces: Vec<String> = project_config
         .as_ref()
@@ -6917,9 +6925,22 @@ fn cmd_docs_check(cli: &Cli, format: &str, fix: bool) -> Result<bool> {
         })
         .unwrap_or_default();
 
-    // 1. Collect docs.
-    let docs = collect_docs(&project_root, &extra_doc_dirs)
-        .with_context(|| format!("scanning docs under {}", project_root.display()))?;
+    // 1. Collect docs (honoring per-root `exclude:` allowlists).
+    let (docs, scan_summary) =
+        rivet_core::doc_check::collect_docs_with_summary(&project_root, &scan_roots)
+            .with_context(|| format!("scanning docs under {}", project_root.display()))?;
+    // Print the per-root scan summary so the user sees how many files
+    // were silently allowlisted under each docs entry.
+    for rs in &scan_summary.roots {
+        if rs.excluded > 0 {
+            eprintln!(
+                "rivet docs check: {} included, {} excluded by allowlist under {}",
+                rs.included,
+                rs.excluded,
+                rs.path.display(),
+            );
+        }
+    }
 
     // 2. Build known-subcommand set from clap metadata (keeps check in sync
     //    with the actual CLI at compile time).
@@ -7270,7 +7291,8 @@ fn cmd_context(cli: &Cli) -> Result<bool> {
             .join(", ")
     ));
     if !config.docs.is_empty() {
-        out.push_str(&format!("- **Docs:** {}\n", config.docs.join(", ")));
+        let names: Vec<&str> = config.docs.iter().map(|e| e.path()).collect();
+        out.push_str(&format!("- **Docs:** {}\n", names.join(", ")));
     }
     if let Some(ref r) = config.results {
         out.push_str(&format!("- **Results:** {r}\n"));
@@ -9612,17 +9634,31 @@ impl ProjectContext {
     }
 
     /// Load project with artifacts, schema, link graph, and documents.
+    ///
+    /// The docs scanner emits one stderr warning per file declined for
+    /// missing or malformed YAML front-matter (see
+    /// [`document::load_documents_with_report`]). Files matching an
+    /// `exclude:` glob in the corresponding `docs:` entry are silently
+    /// allowlisted so generated content can stay in-tree without spam.
     fn load_with_docs(cli: &Cli) -> Result<Self> {
         let mut ctx = Self::load(cli)?;
 
         let mut doc_store = DocumentStore::new();
-        for docs_path in &ctx.config.docs {
-            let dir = cli.project.join(docs_path);
-            let docs = document::load_documents(&dir)
-                .with_context(|| format!("loading docs from '{docs_path}'"))?;
+        let mut total = rivet_core::document::ScanReport::default();
+        for entry in &ctx.config.docs {
+            let dir = cli.project.join(entry.path());
+            let (docs, report) = document::load_documents_with_report(&dir, entry.exclude())
+                .with_context(|| format!("loading docs from '{}'", entry.path()))?;
             for doc in docs {
                 doc_store.insert(doc);
             }
+            total.merge(&report);
+        }
+        if total.warned > 0 || total.excluded > 0 {
+            eprintln!(
+                "rivet docs: {} loaded, {} skipped (warnings above), {} excluded by allowlist",
+                total.loaded, total.warned, total.excluded,
+            );
         }
         ctx.doc_store = Some(doc_store);
         Ok(ctx)
@@ -10923,9 +10959,11 @@ fn cmd_lsp(cli: &Cli) -> Result<bool> {
     // Load documents and results from config
     if config_path.exists() {
         if let Ok(config) = rivet_core::load_project_config(&config_path) {
-            for docs_path in &config.docs {
-                let dir = project_dir.join(docs_path);
-                if let Ok(docs) = rivet_core::document::load_documents(&dir) {
+            for entry in &config.docs {
+                let dir = project_dir.join(entry.path());
+                if let Ok((docs, _report)) =
+                    rivet_core::document::load_documents_with_report(&dir, entry.exclude())
+                {
                     for doc in docs {
                         doc_store.insert(doc);
                     }

rivet-cli/src/serve/mod.rs

@@ -441,13 +441,14 @@ fn load_docs_and_results(
 ) -> Result<(DocumentStore, ResultStore, Vec<PathBuf>)> {
     let mut doc_store = DocumentStore::new();
     let mut doc_dirs = Vec::new();
-    for docs_path in &config.docs {
-        let dir = project_path.join(docs_path);
+    for entry in &config.docs {
+        let dir = project_path.join(entry.path());
         if dir.is_dir() {
             doc_dirs.push(dir.clone());
         }
-        let docs = rivet_core::document::load_documents(&dir)
-            .with_context(|| format!("loading docs from '{docs_path}'"))?;
+        let (docs, _report) =
+            rivet_core::document::load_documents_with_report(&dir, entry.exclude())
+                .with_context(|| format!("loading docs from '{}'", entry.path()))?;
         for doc in docs {
             doc_store.insert(doc);
         }

rivet-cli/tests/docs_scanner_warnings.rs

@@ -0,0 +1,171 @@
+// SAFETY-REVIEW (SCRC Phase 1, DD-058): Integration test / bench code.
+// Tests legitimately use unwrap/expect/panic/assert-indexing patterns
+// because a test failure should panic with a clear stack. Blanket-allow
+// the Phase 1 restriction lints at crate scope; real risk analysis for
+// these lints is carried by production code in rivet-core/src and
+// rivet-cli/src, not by the test harnesses.
+#![allow(
+    clippy::unwrap_used,
+    clippy::expect_used,
+    clippy::indexing_slicing,
+    clippy::arithmetic_side_effects,
+    clippy::as_conversions,
+    clippy::cast_possible_truncation,
+    clippy::cast_sign_loss,
+    clippy::wildcard_enum_match_arm,
+    clippy::match_wildcard_for_single_variants,
+    clippy::panic,
+    clippy::todo,
+    clippy::unimplemented,
+    clippy::dbg_macro,
+    clippy::print_stdout,
+    clippy::print_stderr
+)]
+
+//! End-to-end coverage for the docs-scanner warn-or-allowlist behavior:
+//! `rivet validate` against a project whose `docs/` contains a non-rivet
+//! file must emit a stderr warning, and adding the file to
+//! `docs[].exclude` must silence that warning.
+
+use std::process::Command;
+
+fn rivet_bin() -> std::path::PathBuf {
+    if let Ok(bin) = std::env::var("CARGO_BIN_EXE_rivet") {
+        return std::path::PathBuf::from(bin);
+    }
+    let manifest = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let workspace_root = manifest.parent().expect("workspace root");
+    workspace_root.join("target").join("debug").join("rivet")
+}
+
+/// Write a minimal viable rivet project under `dir` whose `docs/` folder
+/// contains both a well-formed rivet doc and a generated file with no
+/// front-matter. Returns the project root.
+fn fixture_with_generated_doc(dir: &std::path::Path, docs_section: &str) {
+    std::fs::write(
+        dir.join("rivet.yaml"),
+        format!(
+            "project:\n  \
+              name: test\n  \
+              version: \"0.1.0\"\n  \
+              schemas: []\n\
+             sources:\n  \
+              - path: artifacts\n    \
+                format: generic-yaml\n\
+             {docs_section}",
+        ),
+    )
+    .expect("write rivet.yaml");
+
+    let artifacts = dir.join("artifacts");
+    std::fs::create_dir_all(&artifacts).expect("create artifacts/");
+    // Empty artifacts dir — no need for content for the docs scan.
+
+    let docs = dir.join("docs");
+    std::fs::create_dir_all(&docs).expect("create docs/");
+
+    // A real rivet doc — passes the scanner.
+    std::fs::write(
+        docs.join("real.md"),
+        "---\nid: D-1\ntitle: Real\ntype: document\n---\n\nbody\n",
+    )
+    .expect("write real.md");
+
+    // A generated/unrelated file — no front-matter, the scanner declines.
+    std::fs::write(
+        docs.join("generated-report.md"),
+        "# Generated report\n\nNo front-matter here.\n",
+    )
+    .expect("write generated-report.md");
+}
+
+#[test]
+fn rivet_validate_warns_on_unfrontmattered_doc() {
+    let tmp = tempfile::tempdir().expect("tempdir");
+    fixture_with_generated_doc(tmp.path(), "docs:\n  - docs\n");
+
+    let out = Command::new(rivet_bin())
+        .args([
+            "--project",
+            tmp.path().to_str().unwrap(),
+            "validate",
+            "--format",
+            "json",
+        ])
+        .output()
+        .expect("run rivet validate");
+
+    let stderr = String::from_utf8_lossy(&out.stderr);
+    assert!(
+        stderr.contains("rivet doc scanner skipping"),
+        "stderr should warn about the un-frontmattered file. stderr:\n{stderr}",
+    );
+    assert!(
+        stderr.contains("generated-report.md"),
+        "stderr should name the offending file. stderr:\n{stderr}",
+    );
+    assert!(
+        stderr.contains("docs[].exclude"),
+        "stderr should hint at the exclude knob. stderr:\n{stderr}",
+    );
+}
+
+#[test]
+fn rivet_validate_silent_when_file_is_excluded() {
+    let tmp = tempfile::tempdir().expect("tempdir");
+    fixture_with_generated_doc(
+        tmp.path(),
+        "docs:\n  - path: docs\n    exclude:\n      - \"generated-*.md\"\n",
+    );
+
+    let out = Command::new(rivet_bin())
+        .args([
+            "--project",
+            tmp.path().to_str().unwrap(),
+            "validate",
+            "--format",
+            "json",
+        ])
+        .output()
+        .expect("run rivet validate");
+
+    let stderr = String::from_utf8_lossy(&out.stderr);
+    assert!(
+        !stderr.contains("rivet doc scanner skipping"),
+        "stderr must not warn for files that match an exclude glob. stderr:\n{stderr}",
+    );
+    // The summary line should still note the allowlist hit.
+    assert!(
+        stderr.contains("excluded by allowlist"),
+        "stderr should report the allowlist count. stderr:\n{stderr}",
+    );
+}
+
+#[test]
+fn rivet_validate_legacy_string_docs_still_works() {
+    // Pure-legacy syntax: `docs: [docs]`. No exclude knob, but the
+    // warning should still fire — that's the whole point of this PR.
+    let tmp = tempfile::tempdir().expect("tempdir");
+    fixture_with_generated_doc(tmp.path(), "docs: [docs]\n");
+
+    let out = Command::new(rivet_bin())
+        .args([
+            "--project",
+            tmp.path().to_str().unwrap(),
+            "validate",
+            "--format",
+            "json",
+        ])
+        .output()
+        .expect("run rivet validate");
+
+    assert!(
+        !String::from_utf8_lossy(&out.stdout).is_empty(),
+        "validate should still produce JSON on stdout under the legacy schema",
+    );
+    let stderr = String::from_utf8_lossy(&out.stderr);
+    assert!(
+        stderr.contains("rivet doc scanner skipping"),
+        "legacy form must still warn for unfrontmattered files. stderr:\n{stderr}",
+    );
+}