From 24be917c03eb8e6702415266a07cb35aa8946be7 Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Mon, 27 Oct 2025 14:47:37 -0700 Subject: [PATCH 1/2] rustdoc: add mergeable CCI docs to the unstable feature book --- src/doc/rustdoc/src/unstable-features.md | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/doc/rustdoc/src/unstable-features.md b/src/doc/rustdoc/src/unstable-features.md index 04d3c0cd630f5..73986199661fb 100644 --- a/src/doc/rustdoc/src/unstable-features.md +++ b/src/doc/rustdoc/src/unstable-features.md @@ -197,6 +197,37 @@ themselves marked as unstable. To use any of these options, pass `-Z unstable-op the flag in question to Rustdoc on the command-line. To do this from Cargo, you can either use the `RUSTDOCFLAGS` environment variable or the `cargo rustdoc` command. +### `--merge`, `--parts-out-dir`, and `--include-parts-dir` + +These options control how rustdoc handles files that combine data from multiple crates. + +By default, they act like `--merge=shared` is set, and `--parts-out-dir` and `--include-parts-dir` +are turned off. The `--merge=shared` mode causes rustdoc to load the existing data in the out-dir, +combine the new crate data into it, and write the result. This is very easy to use in scripts that +manually invoke rustdoc, but it's also slow, because it performs O(crates) work on +every crate, meaning it performs O(crates2) work. + +```console +$ rustdoc crate1.rs --out-dir=doc +$ cat doc/search.index/crateNames/* +rd_("fcrate1") +$ rustdoc crate2.rs --out-dir=doc +$ cat doc/search.index/crateNames/* +rd_("fcrate1fcrate2") +``` + +To delay shared-data merging until the end of a build, so that you only have to perform O(crates) +work, use `--merge=none` on every crate except the last one, which will use `--merge=finalize`. + +```console +$ rustdoc +nightly crate1.rs --merge=none --parts-out-dir=crate1.d -Zunstable-options +$ cat doc/search.index/crateNames/* +cat: 'doc/search.index/crateNames/*': No such file or directory +$ rustdoc +nightly crate2.rs --merge=finalize --include-parts-dir=crate1.d -Zunstable-options +$ cat doc/search.index/crateNames/* +rd_("fcrate1fcrate2") +``` + ### `--document-hidden-items`: Show items that are `#[doc(hidden)]` From 0786642d61f660e7872a4e2ce8d19b74efa13105 Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Tue, 28 Oct 2025 15:51:03 -0700 Subject: [PATCH 2/2] rustdoc: clean up and further test mergeable CCI This fixes a limitation that prevented me from adding it to Cargo. The rust compiler-docs bundle is built by running multiple `cargo` commands in succession, and I want to support that, so I'm stuck putting the doc parts all in one directory, so that subsequent `cargo` runs can pick up the previous runs' data. It's less clean, but it should still be usable in hermetic build environments if you give every crate its own directory (which you needed to do before, oddly enough). --- src/librustdoc/config.rs | 17 ++++--- src/librustdoc/html/render/write_shared.rs | 32 +++++++++---- .../run-make/rustdoc-merge-directory/dep1.rs | 4 ++ .../run-make/rustdoc-merge-directory/dep2.rs | 4 ++ .../rustdoc-merge-directory/dep_missing.rs | 4 ++ .../run-make/rustdoc-merge-directory/rmake.rs | 46 +++++++++++++++++++ .../rustdoc-merge-no-input-finalize/rmake.rs | 2 +- 7 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 tests/run-make/rustdoc-merge-directory/dep1.rs create mode 100644 tests/run-make/rustdoc-merge-directory/dep2.rs create mode 100644 tests/run-make/rustdoc-merge-directory/dep_missing.rs create mode 100644 tests/run-make/rustdoc-merge-directory/rmake.rs diff --git a/src/librustdoc/config.rs b/src/librustdoc/config.rs index cf0858810f55f..521f3e638a3a2 100644 --- a/src/librustdoc/config.rs +++ b/src/librustdoc/config.rs @@ -975,15 +975,16 @@ fn parse_extern_html_roots( Ok(externs) } -/// Path directly to crate-info file. +/// Path directly to crate-info directory. /// -/// For example, `/home/user/project/target/doc.parts//crate-info`. +/// For example, `/home/user/project/target/doc.parts`. +/// Each crate has its info stored in a file called `CRATENAME.json`. #[derive(Clone, Debug)] pub(crate) struct PathToParts(pub(crate) PathBuf); impl PathToParts { fn from_flag(path: String) -> Result { - let mut path = PathBuf::from(path); + let path = PathBuf::from(path); // check here is for diagnostics if path.exists() && !path.is_dir() { Err(format!( @@ -992,20 +993,22 @@ impl PathToParts { )) } else { // if it doesn't exist, we'll create it. worry about that in write_shared - path.push("crate-info"); Ok(PathToParts(path)) } } } -/// Reports error if --include-parts-dir / crate-info is not a file +/// Reports error if --include-parts-dir is not a directory fn parse_include_parts_dir(m: &getopts::Matches) -> Result, String> { let mut ret = Vec::new(); for p in m.opt_strs("include-parts-dir") { let p = PathToParts::from_flag(p)?; // this is just for diagnostic - if !p.0.is_file() { - return Err(format!("--include-parts-dir expected {} to be a file", p.0.display())); + if !p.0.is_dir() { + return Err(format!( + "--include-parts-dir expected {} to be a directory", + p.0.display() + )); } ret.push(p); } diff --git a/src/librustdoc/html/render/write_shared.rs b/src/librustdoc/html/render/write_shared.rs index 3a1db805d01cf..69282551f041a 100644 --- a/src/librustdoc/html/render/write_shared.rs +++ b/src/librustdoc/html/render/write_shared.rs @@ -14,7 +14,7 @@ //! or contains "invocation-specific". use std::cell::RefCell; -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; use std::fs::File; use std::io::{self, Write as _}; use std::iter::once; @@ -83,9 +83,11 @@ pub(crate) fn write_shared( }; if let Some(parts_out_dir) = &opt.parts_out_dir { - create_parents(&parts_out_dir.0)?; + let mut parts_out_file = parts_out_dir.0.clone(); + parts_out_file.push(&format!("{crate_name}.json")); + create_parents(&parts_out_file)?; try_err!( - fs::write(&parts_out_dir.0, serde_json::to_string(&info).unwrap()), + fs::write(&parts_out_file, serde_json::to_string(&info).unwrap()), &parts_out_dir.0 ); } @@ -237,13 +239,25 @@ impl CrateInfo { pub(crate) fn read_many(parts_paths: &[PathToParts]) -> Result, Error> { parts_paths .iter() - .map(|parts_path| { - let path = &parts_path.0; - let parts = try_err!(fs::read(path), &path); - let parts: CrateInfo = try_err!(serde_json::from_slice(&parts), &path); - Ok::<_, Error>(parts) + .fold(Ok(Vec::new()), |acc, parts_path| { + let mut acc = acc?; + let dir = &parts_path.0; + acc.append(&mut try_err!(std::fs::read_dir(dir), dir.as_path()) + .filter_map(|file| { + let to_crate_info = |file: Result| -> Result, Error> { + let file = try_err!(file, dir.as_path()); + if file.path().extension() != Some(OsStr::new("json")) { + return Ok(None); + } + let parts = try_err!(fs::read(file.path()), file.path()); + let parts: CrateInfo = try_err!(serde_json::from_slice(&parts), file.path()); + Ok(Some(parts)) + }; + to_crate_info(file).transpose() + }) + .collect::, Error>>()?); + Ok(acc) }) - .collect::, Error>>() } } diff --git a/tests/run-make/rustdoc-merge-directory/dep1.rs b/tests/run-make/rustdoc-merge-directory/dep1.rs new file mode 100644 index 0000000000000..5a1238adec0eb --- /dev/null +++ b/tests/run-make/rustdoc-merge-directory/dep1.rs @@ -0,0 +1,4 @@ +//@ hasraw crates.js 'dep1' +//@ hasraw search.index/name/*.js 'Dep1' +//@ has dep1/index.html +pub struct Dep1; diff --git a/tests/run-make/rustdoc-merge-directory/dep2.rs b/tests/run-make/rustdoc-merge-directory/dep2.rs new file mode 100644 index 0000000000000..238ff2e4f9b70 --- /dev/null +++ b/tests/run-make/rustdoc-merge-directory/dep2.rs @@ -0,0 +1,4 @@ +//@ hasraw crates.js 'dep1' +//@ hasraw search.index/name/*.js 'Dep1' +//@ has dep2/index.html +pub struct Dep2; diff --git a/tests/run-make/rustdoc-merge-directory/dep_missing.rs b/tests/run-make/rustdoc-merge-directory/dep_missing.rs new file mode 100644 index 0000000000000..74236aef47ea5 --- /dev/null +++ b/tests/run-make/rustdoc-merge-directory/dep_missing.rs @@ -0,0 +1,4 @@ +//@ !hasraw crates.js 'dep_missing' +//@ !hasraw search.index/name/*.js 'DepMissing' +//@ has dep_missing/index.html +pub struct DepMissing; diff --git a/tests/run-make/rustdoc-merge-directory/rmake.rs b/tests/run-make/rustdoc-merge-directory/rmake.rs new file mode 100644 index 0000000000000..e4695ddad0b48 --- /dev/null +++ b/tests/run-make/rustdoc-merge-directory/rmake.rs @@ -0,0 +1,46 @@ +// Running --merge=finalize without an input crate root should not trigger ICE. +// Issue: https://github.com/rust-lang/rust/issues/146646 + +//@ needs-target-std + +use run_make_support::{htmldocck, path, rustdoc}; + +fn main() { + let out_dir = path("out"); + let merged_dir = path("merged"); + let parts_out_dir = path("parts"); + + rustdoc() + .input("dep1.rs") + .out_dir(&out_dir) + .arg("-Zunstable-options") + .arg(format!("--parts-out-dir={}", parts_out_dir.display())) + .arg("--merge=none") + .run(); + assert!(parts_out_dir.join("dep1.json").exists()); + + rustdoc() + .input("dep2.rs") + .out_dir(&out_dir) + .arg("-Zunstable-options") + .arg(format!("--parts-out-dir={}", parts_out_dir.display())) + .arg("--merge=none") + .run(); + assert!(parts_out_dir.join("dep2.json").exists()); + + // dep_missing is different, because --parts-out-dir is not supplied + rustdoc().input("dep_missing.rs").out_dir(&out_dir).run(); + assert!(parts_out_dir.join("dep2.json").exists()); + + let output = rustdoc() + .arg("-Zunstable-options") + .out_dir(&out_dir) + .arg(format!("--include-parts-dir={}", parts_out_dir.display())) + .arg("--merge=finalize") + .run(); + output.assert_stderr_not_contains("error: the compiler unexpectedly panicked. this is a bug."); + + htmldocck().arg(&out_dir).arg("dep1.rs").run(); + htmldocck().arg(&out_dir).arg("dep2.rs").run(); + htmldocck().arg(out_dir).arg("dep_missing.rs").run(); +} diff --git a/tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs b/tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs index 0b1e1948d5fcc..dadfe61235ac0 100644 --- a/tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs +++ b/tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs @@ -16,7 +16,7 @@ fn main() { .arg(format!("--parts-out-dir={}", parts_out_dir.display())) .arg("--merge=none") .run(); - assert!(parts_out_dir.join("crate-info").exists()); + assert!(parts_out_dir.join("sierra.json").exists()); let output = rustdoc() .arg("-Zunstable-options")