Rollup merge of rust-lang#58626 - QuietMisdreavus:doc-coverage, r=GuillaumeGomez

rustdoc: add option to calculate "documentation coverage"

This PR adds a new flag to rustdoc, `--show-coverage`. When passed, this flag will make rustdoc count the number of items in a crate with documentation instead of generating docs. This count will be output as a table of each file in the crate, like this (when run on my crate `egg-mode`):

```
+-------------------------------------+------------+------------+------------+
| File                                | Documented |      Total | Percentage |
+-------------------------------------+------------+------------+------------+
| src/auth.rs                         |         16 |         16 |     100.0% |
| src/common/mod.rs                   |          1 |          1 |     100.0% |
| src/common/response.rs              |          9 |          9 |     100.0% |
| src/cursor.rs                       |         24 |         24 |     100.0% |
| src/direct/fun.rs                   |          6 |          6 |     100.0% |
| src/direct/mod.rs                   |         41 |         41 |     100.0% |
| src/entities.rs                     |         50 |         50 |     100.0% |
| src/error.rs                        |         27 |         27 |     100.0% |
| src/lib.rs                          |          1 |          1 |     100.0% |
| src/list/fun.rs                     |         19 |         19 |     100.0% |
| src/list/mod.rs                     |         22 |         22 |     100.0% |
| src/media/mod.rs                    |         27 |         27 |     100.0% |
| src/place/fun.rs                    |          8 |          8 |     100.0% |
| src/place/mod.rs                    |         35 |         35 |     100.0% |
| src/search.rs                       |         26 |         26 |     100.0% |
| src/service.rs                      |         74 |         74 |     100.0% |
| src/stream/mod.rs                   |         49 |         49 |     100.0% |
| src/tweet/fun.rs                    |         15 |         15 |     100.0% |
| src/tweet/mod.rs                    |         73 |         73 |     100.0% |
| src/user/fun.rs                     |         24 |         24 |     100.0% |
| src/user/mod.rs                     |         87 |         87 |     100.0% |
+-------------------------------------+------------+------------+------------+
| Total                               |        634 |        634 |     100.0% |
+-------------------------------------+------------+------------+------------+
```

Trait implementations are not counted because by default they "inherit" the docs from the trait, even though an impl can override those docs. Similarly, inherent impl blocks are not counted at all, because for the majority of cases such docs are not useful. (The usual pattern for inherent impl blocks is to throw all the methods on a type into a single impl block. Any docs you would put on that block would be better served on the type itself.)

In addition, `--show-coverage` can be combined with `--document-private-items` to get the coverage counts for everything in the crate, not just public items.

The coverage calculation is implemented as a late pass and two new sets of passes which strip out most of the work that rustdoc otherwise does when generating docs. The is because after the new pass is executed, rustdoc immediately closes instead of going on to generate documentation.

Many examples of coverage calculations have been included as `rustdoc-ui` tests.

r? @rust-lang/rustdoc

Author: Centril

src/doc/rustdoc/src/unstable-features.md

@@ -53,7 +53,7 @@ For example, in the following code:
 ```rust
 /// Does the thing.
 pub fn do_the_thing(_: SomeType) {
-	println!("Let's do the thing!");
+    println!("Let's do the thing!");
 }
 
 /// Token you use to [`do_the_thing`].
@@ -66,15 +66,15 @@ target out also works:
 
 ```rust
 pub mod some_module {
-	/// Token you use to do the thing.
-	pub struct SomeStruct;
+    /// Token you use to do the thing.
+    pub struct SomeStruct;
 }
 
 /// Does the thing. Requires one [`SomeStruct`] for the thing to work.
 ///
 /// [`SomeStruct`]: some_module::SomeStruct
 pub fn do_the_thing(_: some_module::SomeStruct) {
-	println!("Let's do the thing!");
+    println!("Let's do the thing!");
 }
 ```
 
@@ -428,3 +428,30 @@ $ rustdoc src/lib.rs --test -Z unstable-options --persist-doctests target/rustdo
 This flag allows you to keep doctest executables around after they're compiled or run.
 Usually, rustdoc will immediately discard a compiled doctest after it's been tested, but
 with this option, you can keep those binaries around for farther testing.
+
+### `--show-coverage`: calculate the percentage of items with documentation
+
+Using this flag looks like this:
+
+```bash
+$ rustdoc src/lib.rs -Z unstable-options --show-coverage
+```
+
+If you want to determine how many items in your crate are documented, pass this flag to rustdoc.
+When it receives this flag, it will count the public items in your crate that have documentation,
+and print out the counts and a percentage instead of generating docs.
+
+Some methodology notes about what rustdoc counts in this metric:
+
+* Rustdoc will only count items from your crate (i.e. items re-exported from other crates don't
+  count).
+* Docs written directly onto inherent impl blocks are not counted, even though their doc comments
+  are displayed, because the common pattern in Rust code is to write all inherent methods into the
+  same impl block.
+* Items in a trait implementation are not counted, as those impls will inherit any docs from the
+  trait itself.
+* By default, only public items are counted. To count private items as well, pass
+  `--document-private-items` at the same time.
+
+Public items that are not documented can be seen with the built-in `missing_docs` lint. Private
+items that are not documented can be seen with Clippy's `missing_docs_in_private_items` lint.

src/librustdoc/config.rs

@@ -85,6 +85,9 @@ pub struct Options {
     /// Whether to display warnings during doc generation or while gathering doctests. By default,
     /// all non-rustdoc-specific lints are allowed when generating docs.
     pub display_warnings: bool,
+    /// Whether to run the `calculate-doc-coverage` pass, which counts the number of public items
+    /// with and without documentation.
+    pub show_coverage: bool,
 
     // Options that alter generated documentation pages
 
@@ -128,6 +131,7 @@ impl fmt::Debug for Options {
             .field("default_passes", &self.default_passes)
             .field("manual_passes", &self.manual_passes)
             .field("display_warnings", &self.display_warnings)
+            .field("show_coverage", &self.show_coverage)
             .field("crate_version", &self.crate_version)
             .field("render_options", &self.render_options)
             .finish()
@@ -224,6 +228,18 @@ impl Options {
             for &name in passes::DEFAULT_PRIVATE_PASSES {
                 println!("{:>20}", name);
             }
+
+            if nightly_options::is_nightly_build() {
+                println!("\nPasses run with `--show-coverage`:");
+                for &name in passes::DEFAULT_COVERAGE_PASSES {
+                    println!("{:>20}", name);
+                }
+                println!("\nPasses run with `--show-coverage --document-private-items`:");
+                for &name in passes::PRIVATE_COVERAGE_PASSES {
+                    println!("{:>20}", name);
+                }
+            }
+
             return Err(0);
         }
 
@@ -413,9 +429,16 @@ impl Options {
             }
         });
 
+        let show_coverage = matches.opt_present("show-coverage");
+        let document_private = matches.opt_present("document-private-items");
+
         let default_passes = if matches.opt_present("no-defaults") {
             passes::DefaultPassOption::None
-        } else if matches.opt_present("document-private-items") {
+        } else if show_coverage && document_private {
+            passes::DefaultPassOption::PrivateCoverage
+        } else if show_coverage {
+            passes::DefaultPassOption::Coverage
+        } else if document_private {
             passes::DefaultPassOption::Private
         } else {
             passes::DefaultPassOption::Default
@@ -463,6 +486,7 @@ impl Options {
             default_passes,
             manual_passes,
             display_warnings,
+            show_coverage,
             crate_version,
             persist_doctests,
             render_options: RenderOptions {

src/librustdoc/core.rs

@@ -617,10 +617,13 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
 
             info!("Executing passes");
 
-            for pass in &passes {
-                match passes::find_pass(pass).map(|p| p.pass) {
-                    Some(pass) => krate = pass(krate, &ctxt),
-                    None => error!("unknown pass {}, skipping", *pass),
+            for pass_name in &passes {
+                match passes::find_pass(pass_name).map(|p| p.pass) {
+                    Some(pass) => {
+                        debug!("running pass {}", pass_name);
+                        krate = pass(krate, &ctxt);
+                    }
+                    None => error!("unknown pass {}, skipping", *pass_name),
                 }
             }
 

src/librustdoc/html/item_type.rs

@@ -15,7 +15,7 @@ use crate::clean;
 /// module headings. If you are adding to this enum and want to ensure that the sidebar also prints
 /// a heading, edit the listing in `html/render.rs`, function `sidebar_module`. This uses an
 /// ordering based on a helper function inside `item_module`, in the same file.
-#[derive(Copy, PartialEq, Clone, Debug)]
+#[derive(Copy, PartialEq, Eq, Clone, Debug, PartialOrd, Ord)]
 pub enum ItemType {
     Module          = 0,
     ExternCrate     = 1,

src/librustdoc/lib.rs

@@ -348,6 +348,11 @@ fn opts() -> Vec<RustcOptGroup> {
                       "generate-redirect-pages",
                       "Generate extra pages to support legacy URLs and tool links")
         }),
+        unstable("show-coverage", |o| {
+            o.optflag("",
+                      "show-coverage",
+                      "calculate percentage of public items with documentation")
+        }),
     ]
 }
 
@@ -392,7 +397,14 @@ fn main_args(args: &[String]) -> isize {
     let diag_opts = (options.error_format,
                      options.debugging_options.treat_err_as_bug,
                      options.debugging_options.ui_testing);
+    let show_coverage = options.show_coverage;
     rust_input(options, move |out| {
+        if show_coverage {
+            // if we ran coverage, bail early, we don't need to also generate docs at this point
+            // (also we didn't load in any of the useful passes)
+            return rustc_driver::EXIT_SUCCESS;
+        }
+
         let Output { krate, passes, renderinfo, renderopts } = out;
         info!("going to format");
         let (error_format, treat_err_as_bug, ui_testing) = diag_opts;

src/librustdoc/passes/calculate_doc_coverage.rs

@@ -0,0 +1,166 @@
+use crate::clean;
+use crate::core::DocContext;
+use crate::fold::{self, DocFolder};
+use crate::passes::Pass;
+
+use syntax::attr;
+use syntax_pos::FileName;
+
+use std::collections::BTreeMap;
+use std::ops;
+
+pub const CALCULATE_DOC_COVERAGE: Pass = Pass {
+    name: "calculate-doc-coverage",
+    pass: calculate_doc_coverage,
+    description: "counts the number of items with and without documentation",
+};
+
+fn calculate_doc_coverage(krate: clean::Crate, _: &DocContext<'_, '_, '_>) -> clean::Crate {
+    let mut calc = CoverageCalculator::default();
+    let krate = calc.fold_crate(krate);
+
+    calc.print_results();
+
+    krate
+}
+
+#[derive(Default, Copy, Clone)]
+struct ItemCount {
+    total: u64,
+    with_docs: u64,
+}
+
+impl ItemCount {
+    fn count_item(&mut self, has_docs: bool) {
+        self.total += 1;
+
+        if has_docs {
+            self.with_docs += 1;
+        }
+    }
+
+    fn percentage(&self) -> Option<f64> {
+        if self.total > 0 {
+            Some((self.with_docs as f64 * 100.0) / self.total as f64)
+        } else {
+            None
+        }
+    }
+}
+
+impl ops::Sub for ItemCount {
+    type Output = Self;
+
+    fn sub(self, rhs: Self) -> Self {
+        ItemCount {
+            total: self.total - rhs.total,
+            with_docs: self.with_docs - rhs.with_docs,
+        }
+    }
+}
+
+impl ops::AddAssign for ItemCount {
+    fn add_assign(&mut self, rhs: Self) {
+        self.total += rhs.total;
+        self.with_docs += rhs.with_docs;
+    }
+}
+
+#[derive(Default)]
+struct CoverageCalculator {
+    items: BTreeMap<FileName, ItemCount>,
+}
+
+impl CoverageCalculator {
+    fn print_results(&self) {
+        let mut total = ItemCount::default();
+
+        fn print_table_line() {
+            println!("+-{0:->35}-+-{0:->10}-+-{0:->10}-+-{0:->10}-+", "");
+        }
+
+        fn print_table_record(name: &str, count: ItemCount, percentage: f64) {
+            println!("| {:<35} | {:>10} | {:>10} | {:>9.1}% |",
+                     name, count.with_docs, count.total, percentage);
+        }
+
+        print_table_line();
+        println!("| {:<35} | {:>10} | {:>10} | {:>10} |",
+                 "File", "Documented", "Total", "Percentage");
+        print_table_line();
+
+        for (file, &count) in &self.items {
+            if let Some(percentage) = count.percentage() {
+                let mut name = file.to_string();
+                // if a filename is too long, shorten it so we don't blow out the table
+                // FIXME(misdreavus): this needs to count graphemes, and probably also track
+                // double-wide characters...
+                if name.len() > 35 {
+                    name = "...".to_string() + &name[name.len()-32..];
+                }
+
+                print_table_record(&name, count, percentage);
+
+                total += count;
+            }
+        }
+
+        print_table_line();
+        print_table_record("Total", total, total.percentage().unwrap_or(0.0));
+        print_table_line();
+    }
+}
+
+impl fold::DocFolder for CoverageCalculator {
+    fn fold_item(&mut self, i: clean::Item) -> Option<clean::Item> {
+        let has_docs = !i.attrs.doc_strings.is_empty();
+
+        match i.inner {
+            _ if !i.def_id.is_local() => {
+                // non-local items are skipped because they can be out of the users control,
+                // especially in the case of trait impls, which rustdoc eagerly inlines
+                return Some(i);
+            }
+            clean::StrippedItem(..) => {
+                // don't count items in stripped modules
+                return Some(i);
+            }
+            clean::ImportItem(..) | clean::ExternCrateItem(..) => {
+                // docs on `use` and `extern crate` statements are not displayed, so they're not
+                // worth counting
+                return Some(i);
+            }
+            clean::ImplItem(ref impl_)
+                if attr::contains_name(&i.attrs.other_attrs, "automatically_derived")
+                    || impl_.synthetic || impl_.blanket_impl.is_some() =>
+            {
+                // built-in derives get the `#[automatically_derived]` attribute, and
+                // synthetic/blanket impls are made up by rustdoc and can't be documented
+                // FIXME(misdreavus): need to also find items that came out of a derive macro
+                return Some(i);
+            }
+            clean::ImplItem(ref impl_) => {
+                if let Some(ref tr) = impl_.trait_ {
+                    debug!("impl {:#} for {:#} in {}", tr, impl_.for_, i.source.filename);
+
+                    // don't count trait impls, the missing-docs lint doesn't so we shouldn't
+                    // either
+                    return Some(i);
+                } else {
+                    // inherent impls *can* be documented, and those docs show up, but in most
+                    // cases it doesn't make sense, as all methods on a type are in one single
+                    // impl block
+                    debug!("impl {:#} in {}", impl_.for_, i.source.filename);
+                }
+            }
+            _ => {
+                debug!("counting {} {:?} in {}", i.type_(), i.name, i.source.filename);
+                self.items.entry(i.source.filename.clone())
+                          .or_default()
+                          .count_item(has_docs);
+            }
+        }
+
+        self.fold_item_recur(i)
+    }
+}

src/librustdoc/passes/mod.rs

@@ -45,6 +45,9 @@ pub use self::collect_trait_impls::COLLECT_TRAIT_IMPLS;
 mod check_code_block_syntax;
 pub use self::check_code_block_syntax::CHECK_CODE_BLOCK_SYNTAX;
 
+mod calculate_doc_coverage;
+pub use self::calculate_doc_coverage::CALCULATE_DOC_COVERAGE;
+
 /// A single pass over the cleaned documentation.
 ///
 /// Runs in the compiler context, so it has access to types and traits and the like.
@@ -67,6 +70,7 @@ pub const PASSES: &'static [Pass] = &[
     COLLECT_INTRA_DOC_LINKS,
     CHECK_CODE_BLOCK_SYNTAX,
     COLLECT_TRAIT_IMPLS,
+    CALCULATE_DOC_COVERAGE,
 ];
 
 /// The list of passes run by default.
@@ -94,12 +98,29 @@ pub const DEFAULT_PRIVATE_PASSES: &[&str] = &[
     "propagate-doc-cfg",
 ];
 
+/// The list of default passes run when `--doc-coverage` is passed to rustdoc.
+pub const DEFAULT_COVERAGE_PASSES: &'static [&'static str] = &[
+    "collect-trait-impls",
+    "strip-hidden",
+    "strip-private",
+    "calculate-doc-coverage",
+];
+
+/// The list of default passes run when `--doc-coverage --document-private-items` is passed to
+/// rustdoc.
+pub const PRIVATE_COVERAGE_PASSES: &'static [&'static str] = &[
+    "collect-trait-impls",
+    "calculate-doc-coverage",
+];
+
 /// A shorthand way to refer to which set of passes to use, based on the presence of
 /// `--no-defaults` or `--document-private-items`.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum DefaultPassOption {
     Default,
     Private,
+    Coverage,
+    PrivateCoverage,
     None,
 }
 
@@ -108,6 +129,8 @@ pub fn defaults(default_set: DefaultPassOption) -> &'static [&'static str] {
     match default_set {
         DefaultPassOption::Default => DEFAULT_PASSES,
         DefaultPassOption::Private => DEFAULT_PRIVATE_PASSES,
+        DefaultPassOption::Coverage => DEFAULT_COVERAGE_PASSES,
+        DefaultPassOption::PrivateCoverage => PRIVATE_COVERAGE_PASSES,
         DefaultPassOption::None => &[],
     }
 }