diff --git a/scripts/rmc.py b/scripts/rmc.py index 4c6cf27bfba0..19c0228041f9 100644 --- a/scripts/rmc.py +++ b/scripts/rmc.py @@ -142,6 +142,8 @@ def compile_single_rust_file(input_filename, output_filename, verbose=False, deb "-Z", f"symbol-mangling-version={mangler}", "-Z", f"symbol_table_passes={' '.join(symbol_table_passes)}", f"--cfg={RMC_CFG}", "-o", output_filename, input_filename] + if "RUSTFLAGS" in os.environ: + build_cmd += os.environ["RUSTFLAGS"].split(" ") build_env = os.environ if debug: add_rmc_rustc_debug_to_env(build_env) diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 417310e9a59b..8a35c142f3aa 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -2394,12 +2394,12 @@ impl<'test> TestCx<'test> { /// Adds rmc scripts directory to the `PATH` environment variable. fn add_rmc_dir_to_path(&self, command: &mut Command) { - // If the PATH enviornment variable is already defined, + // If the PATH environment variable is already defined, if let Some((key, val)) = env::vars().find(|(key, _)| key == "PATH") { // Add the RMC scripts directory to the PATH. command.env(key, format!("{}:{}", self.config.rmc_dir_path.to_str().unwrap(), val)); } else { - // Otherwise, insert PATH as a new enviornment variable and set its value to the RMC scripts directory. + // Otherwise, insert PATH as a new environment variable and set its value to the RMC scripts directory. command.env( String::from("PATH"), String::from(self.config.rmc_dir_path.to_str().unwrap()), @@ -2411,7 +2411,10 @@ impl<'test> TestCx<'test> { /// error message is printed to stdout if the check result is not expected. fn check(&self) { let mut rustc = Command::new("rmc-rustc"); - rustc.args(["-Z", "no-codegen"]).arg(&self.testpaths.file); + rustc + .args(self.props.compile_flags.clone()) + .args(["-Z", "no-codegen"]) + .arg(&self.testpaths.file); self.add_rmc_dir_to_path(&mut rustc); let proc_res = self.compose_and_run_compiler(rustc, None); if self.props.rmc_panic_step == Some(RMCFailStep::Check) { @@ -2431,6 +2434,7 @@ impl<'test> TestCx<'test> { fn codegen(&self) { let mut rustc = Command::new("rmc-rustc"); rustc + .args(self.props.compile_flags.clone()) .args(["-Z", "codegen-backend=gotoc", "--cfg=rmc", "--out-dir"]) .arg(self.output_base_dir()) .arg(&self.testpaths.file); @@ -2461,6 +2465,13 @@ impl<'test> TestCx<'test> { // 2. It may pass some options that do not make sense for RMC // So we create our own command to execute RMC and pass it to self.compose_and_run_compiler(...) directly. let mut rmc = Command::new("rmc"); + // We cannot pass rustc flags directly to RMC. Instead, we add them + // to the current environment through the `RUSTFLAGS` environment + // variable. RMC recognizes the variable and adds those flags to its + // internal call to rustc. + if !self.props.compile_flags.is_empty() { + rmc.env("RUSTFLAGS", self.props.compile_flags.join(" ")); + } // Pass the test path along with RMC and CBMC flags parsed from comments at the top of the test file. rmc.args(&self.props.rmc_flags) .arg("--input") diff --git a/src/tools/dashboard/src/reference.rs b/src/tools/dashboard/src/reference.rs index f38077eb4db2..ead13f6c3b03 100644 --- a/src/tools/dashboard/src/reference.rs +++ b/src/tools/dashboard/src/reference.rs @@ -8,7 +8,10 @@ use crate::dashboard; use pulldown_cmark::{Parser, Tag}; use std::{ collections::HashMap, - env, fs, + env, + fmt::{Debug, Formatter, Result}, + fs::{self, File}, + hash::Hash, io::{BufRead, BufReader}, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -19,6 +22,7 @@ use std::{ /// code to corresponding directories where the extracted rust code should /// reside. fn parse_hierarchy(summary_path: &Path) -> HashMap { + let summary_dir = summary_path.parent().unwrap().to_path_buf(); let start = "# The Rust Reference\n\n[Introduction](introduction.md)"; let summary = fs::read_to_string(summary_path).unwrap(); assert!(summary.starts_with(start), "Error: The start of the summary file changed."); @@ -26,10 +30,10 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap { let n = Parser::new(start).count(); let parser = Parser::new(&summary).skip(n); // Set "ref" as the root of the hierarchical path. - let mut hierarchy = PathBuf::from("ref"); + let mut hierarchy: PathBuf = ["src", "test", "ref"].iter().collect(); let mut map = HashMap::new(); // Introduction is a especial case, so handle it separately. - map.insert(PathBuf::from("introduction.md"), hierarchy.join("Introduction")); + map.insert(summary_dir.join("introduction.md"), hierarchy.join("Introduction")); for event in parser { match event { pulldown_cmark::Event::End(Tag::Item) => { @@ -42,7 +46,9 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap { // contain the title of the current chapter/section. So, we wait // for the end of the link tag before adding the path and // hierarchy of the current chapter/section to the map. - map.insert(path.split('/').collect(), hierarchy.clone()); + let mut full_path = summary_dir.clone(); + full_path.extend(path.split('/')); + map.insert(full_path, hierarchy.clone()); } pulldown_cmark::Event::Text(text) => { // Add the current chapter/section title to the hierarchy. @@ -54,87 +60,131 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap { map } -/// Extracts examples from the given relative `paths` in the `book_dir` and -/// saves them in `gen_dir`. -fn extract_examples(paths: Vec<&PathBuf>, book_dir: &Path, gen_dir: &Path) { - for path in paths { - let mut cmd = Command::new("rustdoc"); - cmd.args([ - "+nightly", - "--test", - "-Z", - "unstable-options", - book_dir.join(path).to_str().unwrap(), - "--test-builder", - &["src", "tools", "dashboard", "print.sh"] - .iter() - .collect::() - .to_str() - .unwrap(), - "--persist-doctests", - gen_dir.to_str().unwrap(), - "--no-run", - ]); - cmd.stdout(Stdio::null()); - cmd.spawn().unwrap().wait().unwrap(); +/// The data structure represents the "full" path to examples in the Rust books. +#[derive(PartialEq, Eq, Hash)] +struct Example { + /// Path to the markdown file containing the example. + path: PathBuf, + /// Line number of the code block introducing the example. + line: usize, +} + +impl Example { + /// Creates a new [`Example`] instance representing "full" path to the + /// Rust example. + fn new(path: PathBuf, line: usize) -> Example { + Example { path, line } } } -/// Copies the extracted rust code in `from_dir` to `src/test` following the -/// hierarchy specified by `map`. -fn organize_examples(map: &HashMap, book_dir: &Path, from_dir: &Path) { - // The names of the extracted examples generated by `rustdoc` have the - // format `__` where occurrences of '/', '-', and - // '.' in are replaced by '_'. This transformation is not injective, - // so we cannot map those names back to the original markdown file path. - // Instead, we apply the same transformation on the keys of `map` in the for - // loop below and lookup in those modified keys. - let mut modified_map = HashMap::new(); - for (path, hierarchy) in map.iter() { - modified_map.insert( - book_dir.join(path).to_str().unwrap().replace(&['\\', '/', '-', '.'][..], "_"), - hierarchy.clone(), - ); +impl Debug for Example { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + f.write_fmt(format_args!("{}:{}", self.path.to_str().unwrap(), self.line)) } - for dir in from_dir.read_dir().unwrap() { - let dir = dir.unwrap().path(); +} + +/// Extracts examples from the markdown files specified by each key in the given +/// `map` and saves them in the directory specified by the corresponding value. +/// Returns a mapping from the original location of **_each_** example to the +/// path it was extracted to. +fn extract_examples(par_map: HashMap) -> HashMap { + let mut full_map = HashMap::new(); + for (par_from, par_to) in par_map { + let pairs = extract(&par_from, &par_to); + for (key, val) in pairs { + full_map.insert(key, val); + } + } + full_map +} + +/// Extracts examples from the markdown files specified by `par_from` and saves +/// them in the directory specified by `par_to`. Returns a mapping from the +/// original location of **_each_** example to the path it was extracted to. +fn extract(par_from: &Path, par_to: &Path) -> Vec<(Example, PathBuf)> { + let build_dir = &env::var("BUILD_DIR").unwrap(); + let triple = &env::var("TRIPLE").unwrap(); + // Create a temporary directory to save the files generated by `rustdoc`. + let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect(); + fs::create_dir_all(&gen_dir).unwrap(); + let mut cmd = Command::new("rustdoc"); + cmd.args([ + "+nightly", + "--test", + "-Z", + "unstable-options", + par_from.to_str().unwrap(), + "--test-builder", + &["src", "tools", "dashboard", "print.sh"].iter().collect::().to_str().unwrap(), + "--persist-doctests", + gen_dir.to_str().unwrap(), + "--no-run", + ]); + cmd.stdout(Stdio::null()); + cmd.spawn().unwrap().wait().unwrap(); + // Mapping from path and line number of rust example to where it was extracted to. + let mut pairs = Vec::new(); + + for dir in gen_dir.read_dir().unwrap() { // Some directories do not contain tests because the markdown file - // instructs `rustdoc` to ignore those tests. - if let Some(example) = dir.read_dir().unwrap().next() { - let example = example.unwrap().path(); - copy(&example, &modified_map); + // instructs `rustdoc` to "ignore" those tests. + let dir = dir.unwrap().path(); + if let Some(from) = dir.read_dir().unwrap().next() { + // The path to each example extracted by `rustdoc` has the form: + // = `/__/rust_out` + // where occurrences of '/', '-', and '.' in are replaced + // by '_'. We copy the file in this path to a new path of the form: + // = `/.rs` + // We omit because all tests have the same number, 0. + let from = from.unwrap().path(); + let path_line_test = dir.file_name().unwrap().to_str().unwrap(); + let splits: Vec<_> = path_line_test.rsplitn(3, '_').collect(); + let line: usize = splits[1].parse().unwrap(); + let to = par_to.join(format!("{}.rs", line)); + fs::create_dir_all(par_to).unwrap(); + fs::copy(&from, &to).unwrap(); + pairs.push((Example::new(par_from.to_path_buf(), line), to)); } } + // Delete the temporary directory. + fs::remove_dir_all(gen_dir).unwrap(); + pairs } -/// Copy the file specified by `from` to the corresponding location specified by -/// `map`. -fn copy(from: &Path, map: &HashMap) { - // The path specified by `from` has the form: - // `build//dashboard/ref/__/rust_out` - // We copy the file in this path to a new path of the form: - // `src/test//.rs - // where `map[] == `. We omit because all tests have - // the same number, 0. - // Extract `__`. - let key_line_test = from.parent().unwrap().file_name().unwrap().to_str().unwrap(); - // Extract and from `key_line_test` to get and - // construct destination path. - let splits: Vec<_> = key_line_test.rsplitn(3, '_').collect(); - let key = splits[2]; - let line = splits[1]; - let val = &map[key]; - let name = &format!("{}.rs", line); - let to = Path::new("src").join("test").join(val).join(name); - fs::create_dir_all(to.parent().unwrap()).unwrap(); - fs::copy(&from, &to).unwrap(); +/// Prepends the text in `path` with the given `text`. +fn prepend_text(path: &Path, text: &str) { + let code = fs::read_to_string(&path).unwrap(); + let code = format!("{}\n{}", text, code); + fs::write(&path, code).unwrap(); } -/// Pre-processes the tests in the specified `paths` before running them with -/// `compiletest`. -fn preprocess_examples(_paths: Vec<&PathBuf>) { - // For now, we will only pre-process the tests that cause infinite loops. - // TODO: properly implement this step (see issue #324). +/// Pre-processes the examples in `map` before running them with `compiletest`. +fn preprocess_examples(map: &HashMap) { + // Copy compiler configurations specified in the original markdown code + // block. + for (from, to) in map.iter() { + let file = File::open(&from.path).unwrap(); + // Skip to the first line of the example code block. + // Line numbers in files start with 1 but `nth(...)` starts with 0. + // Subtract 1 to account for the difference. + let line = BufReader::new(file).lines().nth(from.line - 1).unwrap().unwrap(); + if line.contains("edition2015") { + prepend_text(to, "// compile-flags: --edition 2015"); + } else { + prepend_text(to, "// compile-flags: --edition 2018"); + } + // Most examples with `compile_fail` configuration fail because of + // check errors. + if line.contains("compile_fail") { + prepend_text(to, "// rmc-check-fail"); + } + // RMC should catch run-time errors. + if line.contains("should_panic") { + prepend_text(to, "// rmc-verify-fail"); + } + } + // For now, we will only manually pre-process the tests that cause infinite loops. + // TODO: Add support for manually adding options and assertions (see issue #324). let loop_tests: [PathBuf; 4] = [ ["src", "test", "ref", "Appendices", "Glossary", "263.rs"].iter().collect(), ["src", "test", "ref", "Linkage", "190.rs"].iter().collect(), @@ -193,7 +243,6 @@ fn run_examples(suite: &str, log_path: &Path) { ]); cmd.env_clear().envs(filtered_env); cmd.stdout(Stdio::null()); - cmd.spawn().unwrap().wait().unwrap(); } @@ -258,20 +307,17 @@ fn display_dashboard(dashboard: dashboard::Tree) { /// displays their results in a terminal dashboard. pub fn display_reference_dashboard() { let summary_path: PathBuf = ["src", "doc", "reference", "src", "SUMMARY.md"].iter().collect(); - let ref_dir: PathBuf = ["src", "doc", "reference", "src"].iter().collect(); let build_dir = &env::var("BUILD_DIR").unwrap(); let triple = &env::var("TRIPLE").unwrap(); - let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect(); let log_path: PathBuf = [build_dir, triple, "dashboard", "ref.log"].iter().collect(); // Parse the chapter/section hierarchy from the table of contents in The // Rust Reference. let map = parse_hierarchy(&summary_path); - // Extract examples from The Rust Reference. - extract_examples(map.keys().collect(), &ref_dir, &gen_dir); - // Reorganize those examples following the The Rust Reference hierarchy. - organize_examples(&map, &ref_dir, &gen_dir); + // Extract examples from The Rust Reference, organize them following the + // partial hierarchy in map, and return the full hierarchy map. + let map = extract_examples(map); // Pre-process the examples before running them through `compiletest`. - preprocess_examples(map.values().collect()); + preprocess_examples(&map); // Run `compiletest` on the reference examples. run_examples("ref", &log_path); // Parse `compiletest` log file.