src/tools/lint-docs/src/lib.rs

use std::error::Error;
use std::fmt::Write;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;

use rustc_literal_escaper::unescape_str;
use walkdir::WalkDir;

mod groups;

/// List of lints which have been renamed.
///
/// These will get redirects in the output to the new name. The
/// format is `(level, [(old_name, new_name), ...])`.
///
/// Note: This hard-coded list is a temporary hack. The intent is in the
/// future to have `rustc` expose this information in some way (like a `-Z`
/// flag spitting out JSON). Also, this does not yet support changing the
/// level of the lint, which will be more difficult to support, since rustc
/// currently does not track that historical information.
static RENAMES: &[(Level, &[(&str, &str)])] = &[
    (
        Level::Allow,
        &[
            ("single-use-lifetime", "single-use-lifetimes"),
            ("elided-lifetime-in-path", "elided-lifetimes-in-paths"),
            ("async-idents", "keyword-idents"),
            ("disjoint-capture-migration", "rust-2021-incompatible-closure-captures"),
            ("keyword-idents", "keyword-idents-2018"),
            ("or-patterns-back-compat", "rust-2021-incompatible-or-patterns"),
        ],
    ),
    (
        Level::Warn,
        &[
            ("bare-trait-object", "bare-trait-objects"),
            ("unstable-name-collision", "unstable-name-collisions"),
            ("unused-doc-comment", "unused-doc-comments"),
            ("redundant-semicolon", "redundant-semicolons"),
            ("overlapping-patterns", "overlapping-range-endpoints"),
            ("non-fmt-panic", "non-fmt-panics"),
            ("unused-tuple-struct-fields", "dead-code"),
            ("static-mut-ref", "static-mut-refs"),
        ],
    ),
    (Level::Deny, &[("exceeding-bitshifts", "arithmetic-overflow")]),
];

pub struct LintExtractor<'a> {
    /// Path to the `src` directory, where it will scan for `.rs` files to
    /// find lint declarations.
    pub src_path: &'a Path,
    /// Path where to save the output.
    pub out_path: &'a Path,
    /// Path to the `rustc` executable.
    pub rustc_path: &'a Path,
    /// The target arch to build the docs for.
    pub rustc_target: &'a str,
    /// The target linker overriding `rustc`'s default
    pub rustc_linker: Option<&'a str>,
    /// Stage of the compiler that builds the docs (the stage of `rustc_path`).
    pub build_rustc_stage: u32,
    /// Verbose output.
    pub verbose: bool,
    /// Validate the style and the code example.
    pub validate: bool,
}

struct Lint {
    name: String,
    doc: Vec<String>,
    level: Level,
    path: PathBuf,
    lineno: usize,
}

impl Lint {
    fn doc_contains(&self, text: &str) -> bool {
        self.doc.iter().any(|line| line.contains(text))
    }

    fn is_ignored(&self) -> bool {
        let blocks: Vec<_> = self.doc.iter().filter(|line| line.starts_with("```rust")).collect();
        !blocks.is_empty() && blocks.iter().all(|line| line.contains(",ignore"))
    }

    /// Checks the doc style of the lint.
    fn check_style(&self) -> Result<(), Box<dyn Error>> {
        for &expected in &["### Example", "### Explanation", "{{produces}}"] {
            if expected == "{{produces}}" && self.is_ignored() {
                if self.doc_contains("{{produces}}") {
                    return Err(
                        "the lint example has `ignore`, but also contains the {{produces}} marker\n\
                        \n\
                        The documentation generator cannot generate the example output when the \
                        example is ignored.\n\
                        Manually include the sample output below the example. For example:\n\
                        \n\
                        /// ```rust,ignore (needs command line option)\n\
                        /// #[cfg(widnows)]\n\
                        /// fn foo() {{}}\n\
                        /// ```\n\
                        ///\n\
                        /// This will produce:\n\
                        /// \n\
                        /// ```text\n\
                        /// warning: unknown condition name used\n\
                        ///  --> lint_example.rs:1:7\n\
                        ///   |\n\
                        /// 1 | #[cfg(widnows)]\n\
                        ///   |       ^^^^^^^\n\
                        ///   |\n\
                        ///   = note: `#[warn(unexpected_cfgs)]` on by default\n\
                        /// ```\n\
                        \n\
                        Replacing the output with the text of the example you \
                        compiled manually yourself.\n\
                        "
                    .into());
                }
                continue;
            }
            if !self.doc_contains(expected) {
                return Err(format!("lint docs should contain the line `{}`", expected).into());
            }
        }
        if let Some(first) = self.doc.first() {
            if !first.starts_with(&format!("The `{}` lint", self.name)) {
                return Err(format!(
                    "lint docs should start with the text \"The `{}` lint\" to introduce the lint",
                    self.name
                )
                .into());
            }
        }
        Ok(())
    }
}

#[derive(Clone, Copy, PartialEq)]
enum Level {
    Allow,
    Warn,
    Deny,
}

impl Level {
    fn doc_filename(&self) -> &str {
        match self {
            Level::Allow => "allowed-by-default.md",
            Level::Warn => "warn-by-default.md",
            Level::Deny => "deny-by-default.md",
        }
    }
}

impl<'a> LintExtractor<'a> {
    /// Collects all lints, and writes the markdown documentation at the given directory.
    pub fn extract_lint_docs(&self) -> Result<(), Box<dyn Error>> {
        let mut lints = self.gather_lints()?;
        for lint in &mut lints {
            self.generate_output_example(lint).map_err(|e| {
                format!(
                    "failed to test example in lint docs for `{}` in {}:{}: {}",
                    lint.name,
                    lint.path.display(),
                    lint.lineno,
                    e
                )
            })?;
        }
        add_renamed_lints(&mut lints);
        self.save_lints_markdown(&lints)?;
        self.generate_group_docs(&lints)?;
        Ok(())
    }

    /// Collects all lints from all files in the given directory.
    fn gather_lints(&self) -> Result<Vec<Lint>, Box<dyn Error>> {
        let mut lints = Vec::new();
        for entry in WalkDir::new(self.src_path).into_iter().filter_map(|e| e.ok()) {
            if !entry.path().extension().map_or(false, |ext| ext == "rs") {
                continue;
            }
            lints.extend(self.lints_from_file(entry.path())?);
        }
        if lints.is_empty() {
            return Err("no lints were found!".into());
        }
        Ok(lints)
    }

    /// Collects all lints from the given file.
    fn lints_from_file(&self, path: &Path) -> Result<Vec<Lint>, Box<dyn Error>> {
        let mut lints = Vec::new();
        let contents = fs::read_to_string(path)
            .map_err(|e| format!("could not read {}: {}", path.display(), e))?;
        let mut lines = contents.lines().enumerate();
        'outer: loop {
            // Find a lint declaration.
            let lint_start = loop {
                match lines.next() {
                    Some((lineno, line)) => {
                        if line.trim().starts_with("declare_lint!") {
                            break lineno + 1;
                        }
                    }
                    None => return Ok(lints),
                }
            };
            // Read the lint.
            let mut doc_lines = Vec::new();
            let (doc, name) = loop {
                match lines.next() {
                    Some((lineno, line)) => {
                        let line = line.trim();
                        if let Some(text) = line.strip_prefix("/// ") {
                            doc_lines.push(text.to_string());
                        } else if let Some(text) = line.strip_prefix("#[doc = \"") {
                            let buf = parse_doc_string(text);
                            doc_lines.push(buf);
                        } else if line == "///" {
                            doc_lines.push("".to_string());
                        } else if line.starts_with("// ") {
                            // Ignore comments.
                            continue;
                        } else if line.starts_with("#[allow") {
                            // Ignore allow of lints (useful for
                            // invalid_rust_codeblocks).
                            continue;
                        } else if let Some(text) =
                            line.strip_prefix("#[cfg_attr(not(bootstrap), doc = \"")
                        {
                            if self.build_rustc_stage >= 1 {
                                let buf = parse_doc_string(text);
                                doc_lines.push(buf);
                            }
                        } else if let Some(text) =
                            line.strip_prefix("#[cfg_attr(bootstrap, doc = \"")
                        {
                            if self.build_rustc_stage == 0 {
                                let buf = parse_doc_string(text);
                                doc_lines.push(buf);
                            }
                        } else {
                            let name = lint_name(line).map_err(|e| {
                                format!(
                                    "could not determine lint name in {}:{}: {}, line was `{}`",
                                    path.display(),
                                    lineno,
                                    e,
                                    line
                                )
                            })?;
                            if doc_lines.is_empty() {
                                if self.validate {
                                    return Err(format!(
                                        "did not find doc lines for lint `{}` in {}",
                                        name,
                                        path.display()
                                    )
                                    .into());
                                } else {
                                    eprintln!(
                                        "warning: lint `{}` in {} does not define any doc lines, \
                                         these are required for the lint documentation",
                                        name,
                                        path.display()
                                    );
                                    continue 'outer;
                                }
                            }
                            break (doc_lines, name);
                        }
                    }
                    None => {
                        return Err(format!(
                            "unexpected EOF for lint definition at {}:{}",
                            path.display(),
                            lint_start
                        )
                        .into());
                    }
                }
            };
            // These lints are specifically undocumented. This should be reserved
            // for internal rustc-lints only.
            if name == "deprecated_in_future" {
                continue;
            }
            // Read the level.
            let level = loop {
                match lines.next() {
                    // Ignore comments.
                    Some((_, line)) if line.trim().starts_with("// ") => {}
                    Some((lineno, line)) => match line.trim() {
                        "Allow," => break Level::Allow,
                        "Warn," => break Level::Warn,
                        "Deny," => break Level::Deny,
                        _ => {
                            return Err(format!(
                                "unexpected lint level `{}` in {}:{}",
                                line,
                                path.display(),
                                lineno
                            )
                            .into());
                        }
                    },
                    None => {
                        return Err(format!(
                            "expected lint level in {}:{}, got EOF",
                            path.display(),
                            lint_start
                        )
                        .into());
                    }
                }
            };
            // The rest of the lint definition is ignored.
            assert!(!doc.is_empty());
            lints.push(Lint { name, doc, level, path: PathBuf::from(path), lineno: lint_start });
        }
    }

    /// Mutates the lint definition to replace the `{{produces}}` marker with the
    /// actual output from the compiler.
    fn generate_output_example(&self, lint: &mut Lint) -> Result<(), Box<dyn Error>> {
        // Explicit list of lints that are allowed to not have an example. Please
        // try to avoid adding to this list.
        if matches!(
            lint.name.as_str(),
            "unused_features" // broken lint
        ) {
            return Ok(());
        }
        if lint.doc_contains("[rustdoc book]") && !lint.doc_contains("{{produces}}") {
            // Rustdoc lints are documented in the rustdoc book, don't check these.
            return Ok(());
        }
        if self.validate {
            lint.check_style()?;
        }
        // Unfortunately some lints have extra requirements that this simple test
        // setup can't handle (like extern crates). An alternative is to use a
        // separate test suite, and use an include mechanism such as mdbook's
        // `{{#rustdoc_include}}`.
        if !lint.is_ignored() {
            if let Err(e) = self.replace_produces(lint) {
                if self.validate {
                    return Err(e);
                }
                eprintln!(
                    "warning: the code example in lint `{}` in {} failed to \
                     generate the expected output: {}",
                    lint.name,
                    lint.path.display(),
                    e
                );
            }
        }
        Ok(())
    }

    /// Mutates the lint docs to replace the `{{produces}}` marker with the actual
    /// output from the compiler.
    fn replace_produces(&self, lint: &mut Lint) -> Result<(), Box<dyn Error>> {
        let mut lines = lint.doc.iter_mut();
        loop {
            // Find start of example.
            let options = loop {
                match lines.next() {
                    Some(line) if line.starts_with("```rust") => {
                        break line[7..].split(',').collect::<Vec<_>>();
                    }
                    Some(line) if line.contains("{{produces}}") => {
                        return Err("lint marker {{{{produces}}}} found, \
                            but expected to immediately follow a rust code block"
                            .into());
                    }
                    Some(_) => {}
                    None => return Ok(()),
                }
            };
            // Find the end of example.
            let mut example = Vec::new();
            loop {
                match lines.next() {
                    Some(line) if line == "```" => break,
                    Some(line) => example.push(line),
                    None => {
                        return Err(format!(
                            "did not find end of example triple ticks ```, docs were:\n{:?}",
                            lint.doc
                        )
                        .into());
                    }
                }
            }
            // Find the {{produces}} line.
            loop {
                match lines.next() {
                    Some(line) if line.is_empty() => {}
                    Some(line) if line == "{{produces}}" => {
                        let output = self.generate_lint_output(&lint.name, &example, &options)?;
                        line.replace_range(
                            ..,
                            &format!(
                                "This will produce:\n\
                                \n\
                                ```text\n\
                                {}\
                                ```",
                                output
                            ),
                        );
                        break;
                    }
                    // No {{produces}} after example, find next example.
                    Some(_line) => break,
                    None => return Ok(()),
                }
            }
        }
    }

    /// Runs the compiler against the example, and extracts the output.
    fn generate_lint_output(
        &self,
        name: &str,
        example: &[&mut String],
        options: &[&str],
    ) -> Result<String, Box<dyn Error>> {
        if self.verbose {
            eprintln!("compiling lint {}", name);
        }
        let tempdir = tempfile::TempDir::new()?;
        let tempfile = tempdir.path().join("lint_example.rs");
        let mut source = String::new();
        let needs_main = !example.iter().any(|line| line.contains("fn main"));
        // Remove `# ` prefix for hidden lines.
        let unhidden = example.iter().map(|line| line.strip_prefix("# ").unwrap_or(line));
        let mut lines = unhidden.peekable();
        while let Some(line) = lines.peek() {
            if line.starts_with("#!") {
                source.push_str(line);
                source.push('\n');
                lines.next();
            } else {
                break;
            }
        }
        if needs_main {
            source.push_str("fn main() {\n");
        }
        for line in lines {
            source.push_str(line);
            source.push('\n')
        }
        if needs_main {
            source.push_str("}\n");
        }
        fs::write(&tempfile, source)
            .map_err(|e| format!("failed to write {}: {}", tempfile.display(), e))?;
        let mut cmd = Command::new(self.rustc_path);
        let edition = options
            .iter()
            .filter_map(|opt| opt.strip_prefix("edition"))
            .next()
            // defaults to latest edition
            .unwrap_or("2024");
        cmd.arg(format!("--edition={edition}"));
        // Just in case this is an unstable edition.
        cmd.arg("-Zunstable-options");
        cmd.arg("--error-format=json");
        cmd.arg("--target").arg(self.rustc_target);
        if let Some(target_linker) = self.rustc_linker {
            cmd.arg(format!("-Clinker={target_linker}"));
        }
        if options.contains(&"test") {
            cmd.arg("--test");
        }
        cmd.arg("lint_example.rs");
        cmd.current_dir(tempdir.path());
        if self.verbose {
            eprintln!("running: {cmd:?}");
        }
        let output = cmd.output().map_err(|e| format!("failed to run command {:?}\n{}", cmd, e))?;
        let stderr = std::str::from_utf8(&output.stderr).unwrap();
        let msgs = stderr
            .lines()
            .filter(|line| line.starts_with('{'))
            .map(serde_json::from_str)
            .collect::<Result<Vec<serde_json::Value>, _>>()?;

        // First try to find the messages with the `code` field set to our lint.
        let matches: Vec<_> = msgs
            .iter()
            .filter(|msg| matches!(&msg["code"]["code"], serde_json::Value::String(s) if s==name))
            .map(|msg| msg["rendered"].as_str().expect("rendered field should exist").to_string())
            .collect();
        if !matches.is_empty() {
            return Ok(matches.join("\n"));
        }

        // Try to detect if an unstable lint forgot to enable a `#![feature(..)]`.
        // Specifically exclude `test_unstable_lint` which exercises this on purpose.
        if name != "test_unstable_lint"
            && msgs.iter().any(|msg| {
                matches!(&msg["code"]["code"], serde_json::Value::String(s) if s=="unknown_lints")
                    && matches!(&msg["message"], serde_json::Value::String(s) if s.contains(name))
            })
        {
            let rendered: Vec<&str> =
                msgs.iter().filter_map(|msg| msg["rendered"].as_str()).collect();
            let non_json: Vec<&str> =
                stderr.lines().filter(|line| !line.starts_with('{')).collect();
            return Err(format!(
                "did not find lint `{}` in output of example (got unknown_lints)\n\
                Is the lint possibly misspelled, or does it need a `#![feature(...)]`?\n\
                Output was:\n\
                {}\n{}",
                name,
                rendered.join("\n"),
                non_json.join("\n"),
            )
            .into());
        }

        // Some lints override their code to something else (E0566).
        // Try to find something that looks like it could be our lint.
        let matches: Vec<_> = msgs
            .iter()
            .filter(
                |msg| matches!(&msg["rendered"], serde_json::Value::String(s) if s.contains(name)),
            )
            .map(|msg| msg["rendered"].as_str().expect("rendered field should exist").to_string())
            .collect();
        if !matches.is_empty() {
            return Ok(matches.join("\n"));
        }

        // Otherwise, give a descriptive error.
        let rendered: Vec<&str> = msgs.iter().filter_map(|msg| msg["rendered"].as_str()).collect();
        let non_json: Vec<&str> = stderr.lines().filter(|line| !line.starts_with('{')).collect();
        Err(format!(
            "did not find lint `{}` in output of example, got:\n{}\n{}",
            name,
            non_json.join("\n"),
            rendered.join("\n")
        )
        .into())
    }

    /// Saves the mdbook lint chapters at the given path.
    fn save_lints_markdown(&self, lints: &[Lint]) -> Result<(), Box<dyn Error>> {
        self.save_level(lints, Level::Allow, ALLOWED_MD)?;
        self.save_level(lints, Level::Warn, WARN_MD)?;
        self.save_level(lints, Level::Deny, DENY_MD)?;
        Ok(())
    }

    fn save_level(&self, lints: &[Lint], level: Level, header: &str) -> Result<(), Box<dyn Error>> {
        let mut result = String::new();
        result.push_str(header);
        let mut these_lints: Vec<_> = lints.iter().filter(|lint| lint.level == level).collect();
        these_lints.sort_unstable_by_key(|lint| &lint.name);
        for lint in &these_lints {
            writeln!(result, "* [`{}`](#{})", lint.name, lint.name.replace('_', "-")).unwrap();
        }
        result.push('\n');
        for lint in &these_lints {
            write!(result, "## {}\n\n", lint.name.replace('_', "-")).unwrap();
            for line in &lint.doc {
                result.push_str(line);
                result.push('\n');
            }
            result.push('\n');
        }
        add_rename_redirect(level, &mut result);
        let out_path = self.out_path.join("listing").join(level.doc_filename());
        // Delete the output because bootstrap uses hard links in its copies.
        let _ = fs::remove_file(&out_path);
        fs::write(&out_path, result)
            .map_err(|e| format!("could not write to {}: {}", out_path.display(), e))?;
        Ok(())
    }
}

/// Parses a doc string that follows `#[doc = "`.
fn parse_doc_string(text: &str) -> String {
    let escaped = text.strip_suffix("]").unwrap_or(text);
    let escaped = escaped.strip_suffix(")").unwrap_or(escaped).strip_suffix("\"");
    let Some(escaped) = escaped else {
        panic!("Cannot extract docstring content from {text}");
    };
    let mut buf = String::new();
    unescape_str(escaped, |_, res| match res {
        Ok(c) => buf.push(c),
        Err(err) => {
            assert!(!err.is_fatal(), "failed to unescape string literal")
        }
    });
    buf
}

/// Adds `Lint`s that have been renamed.
fn add_renamed_lints(lints: &mut Vec<Lint>) {
    for (level, names) in RENAMES {
        for (from, to) in *names {
            lints.push(Lint {
                name: from.to_string(),
                doc: vec![format!("The lint `{from}` has been renamed to [`{to}`](#{to}).")],
                level: *level,
                path: PathBuf::new(),
                lineno: 0,
            });
        }
    }
}

// This uses DOMContentLoaded instead of running immediately because for some
// reason on Firefox (124 of this writing) doesn't update the `target` CSS
// selector if only the hash changes.
static RENAME_START: &str = "
<script>
document.addEventListener(\"DOMContentLoaded\", (event) => {
    var fragments = {
";

static RENAME_END: &str = "\
    };
    var target = fragments[window.location.hash];
    if (target) {
        var url = window.location.toString();
        var base = url.substring(0, url.lastIndexOf('/'));
        window.location.replace(base + \"/\" + target);
    }
});
</script>
";

/// Adds the javascript redirection code to the given markdown output.
fn add_rename_redirect(level: Level, output: &mut String) {
    for (rename_level, names) in RENAMES {
        if *rename_level == level {
            let filename = level.doc_filename().replace(".md", ".html");
            output.push_str(RENAME_START);
            for (from, to) in *names {
                writeln!(output, "        \"#{from}\": \"{filename}#{to}\",").unwrap();
            }
            output.push_str(RENAME_END);
        }
    }
}

/// Extracts the lint name (removing the visibility modifier, and checking validity).
fn lint_name(line: &str) -> Result<String, &'static str> {
    // Skip over any potential `pub` visibility.
    match line.trim().split(' ').next_back() {
        Some(name) => {
            if !name.ends_with(',') {
                return Err("lint name should end with comma");
            }
            let name = &name[..name.len() - 1];
            if !name.chars().all(|ch| ch.is_uppercase() || ch.is_ascii_digit() || ch == '_')
                || name.is_empty()
            {
                return Err("lint name did not have expected format");
            }
            Ok(name.to_lowercase().to_string())
        }
        None => Err("could not find lint name"),
    }
}

static ALLOWED_MD: &str = r#"# Allowed-by-default Lints

These lints are all set to the 'allow' level by default. As such, they won't show up
unless you set them to a higher lint level with a flag or attribute.

"#;

static WARN_MD: &str = r#"# Warn-by-default Lints

These lints are all set to the 'warn' level by default.

"#;

static DENY_MD: &str = r#"# Deny-by-default Lints

These lints are all set to the 'deny' level by default.

"#;