<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::{
    borrow::Cow,
    collections::HashSet,
    fmt::Display,
    io::{self, Read},
    path::PathBuf,
    process,
};

use anyhow::Error;
use clap::{crate_authors, crate_version, Arg, ArgMatches, Command};
use itertools::Itertools;
use mdbook::{book::Book, preprocess::CmdPreprocessor};
use proc_macro2::{Span, TokenStream};
use rayon::prelude::*;
use syn::{Expr, __private::ToTokens, spanned::Spanned};

static LONG_ABOUT: &str =
    "This is an mdbook preprocessor tailored for autocxx code examples. Autocxx
code examples don't fit well 'mdbook test' or even alternatives such as
'skeptic' or 'doc_comment' for these reasons:

a) A single code example consists of both Rust and C++ code. They must be
   linked into a separate executable, i.e. we must make one executable per
   doc test.
b) The code examples must be presented/formatted nicely with suitable
   separate blocks for the Rust and C++ code.
c) mdbook test is not good at handling doctests which have dependencies.

This preprocessor will find code snippets like this:
```rust,autocxx
autocxx_integration_tests::doctest(
\" /* any C++ implementation code */\",
\" /* C++ header code */\",
{
/* complete Rust code including 'main' */
)
```

and will build and run them, while emitting better formatted markdown blocks
for subsequent preprocessors and renderers.
";

static RUST_MDBOOK_SINGLE_TEST: &str = "RUST_MDBOOK_SINGLE_TEST";

fn main() {
    let matches = Command::new("autocxx-mdbook-preprocessor")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Expands and tests code examples in the autocxx book.")
        .long_about(LONG_ABOUT)
        .subcommand(
            Command::new("supports")
                .arg(Arg::new("renderer").required(true))
                .about("Whether a given renderer is supported by this preprocessor"),
        )
        .arg(
            Arg::new("skip_tests")
                .short('s')
                .help("Skip running doctests"),
        )
        .arg(
            Arg::new("manifest_dir")
            .long("manifest-dir")
            .help("Path to directory containing outermost autocxx Cargo.toml; necessary for trybuild to build test code successfully")
            .default_value_os(calculate_cargo_dir().as_os_str())
        )
        .get_matches();
    if let Some(supports_matches) = matches.subcommand_matches("supports") {
        // Only do our preprocessing and testing for the html renderer, not linkcheck.
        if supports_matches.value_of("renderer") == Some("html") {
            process::exit(0);
        } else {
            process::exit(1);
        }
    }
    preprocess(&matches).unwrap();
}

fn calculate_cargo_dir() -> PathBuf {
    let mut path = std::env::current_exe().unwrap();
    for _ in 0..3 {
        path = path.parent().map(|p| p.to_path_buf()).unwrap_or(path);
    }
    path.join("integration-tests")
}

fn preprocess(args: &ArgMatches) -> Result<(), Error> {
    let (_, mut book) = CmdPreprocessor::parse_input(io::stdin())?;

    env_logger::builder().init();
    let mut test_cases = Vec::new();

    Book::for_each_mut(&mut book, |sec| {
        if let mdbook::BookItem::Chapter(chapter) = sec {
            let filename = chapter
                .path
                .as_ref()
                .map(|pb| pb.to_string_lossy())
                .unwrap_or_default()
                .to_string();
            chapter.content = substitute_chapter(&chapter.content, &filename, &mut test_cases);
        }
    });

    // Now run any test cases we accumulated.
    if !args.is_present("skip_tests") {
        let stdout_gag = gag::BufferRedirect::stdout().unwrap();
        let num_tests = test_cases.len();
        let fails: Vec<_> = test_cases
            .into_par_iter()
            .enumerate()
            .filter_map(|(counter, case)| {
                if let Ok(test) = std::env::var(RUST_MDBOOK_SINGLE_TEST) {
                    let desired_id: usize = test.parse().unwrap();
                    if desired_id != (counter + 1) {
                        return None;
                    }
                }
                eprintln!(
                    "Running doctest {}/{} at {}",
                    counter + 1,
                    num_tests,
                    &case.location
                );
                let err = autocxx_integration_tests::doctest(
                    &case.cpp,
                    &case.hdr,
                    case.rs,
                    args.value_of_os("manifest_dir").unwrap(),
                );
                let desc = match err {
                    Ok(_) => "passed".to_string(),
                    Err(ref err) => format!("failed: {:?}", err),
                };
                eprintln!(
                    "Doctest {}/{} at {} {}.",
                    counter + 1,
                    num_tests,
                    &case.location,
                    desc
                );
                if err.is_err() {
                    Some(TestId {
                        location: case.location,
                        test_id: counter + 1,
                    })
                } else {
                    None
                }
            })
            .collect();
        let mut stdout_str = String::new();
        stdout_gag
            .into_inner()
            .read_to_string(&mut stdout_str)
            .unwrap();
        if !stdout_str.is_empty() {
            eprintln!("Stdout from tests:\n{}", stdout_str);
        }<commit_msg>Fix compatibility with clap 3.2.<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::{
    borrow::Cow,
    collections::HashSet,
    ffi::OsString,
    fmt::Display,
    io::{self, Read},
    path::PathBuf,
    process,
};

use anyhow::Error;
use clap::{crate_authors, crate_version, Arg, ArgMatches, Command};
use itertools::Itertools;
use mdbook::{book::Book, preprocess::CmdPreprocessor};
use proc_macro2::{Span, TokenStream};
use rayon::prelude::*;
use syn::{Expr, __private::ToTokens, spanned::Spanned};

static LONG_ABOUT: &str =
    "This is an mdbook preprocessor tailored for autocxx code examples. Autocxx
code examples don't fit well 'mdbook test' or even alternatives such as
'skeptic' or 'doc_comment' for these reasons:

a) A single code example consists of both Rust and C++ code. They must be
   linked into a separate executable, i.e. we must make one executable per
   doc test.
b) The code examples must be presented/formatted nicely with suitable
   separate blocks for the Rust and C++ code.
c) mdbook test is not good at handling doctests which have dependencies.

This preprocessor will find code snippets like this:
```rust,autocxx
autocxx_integration_tests::doctest(
\" /* any C++ implementation code */\",
\" /* C++ header code */\",
{
/* complete Rust code including 'main' */
)
```

and will build and run them, while emitting better formatted markdown blocks
for subsequent preprocessors and renderers.
";

static RUST_MDBOOK_SINGLE_TEST: &str = "RUST_MDBOOK_SINGLE_TEST";

fn main() {
    let matches = Command::new("autocxx-mdbook-preprocessor")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Expands and tests code examples in the autocxx book.")
        .long_about(LONG_ABOUT)
        .subcommand(
            Command::new("supports")
                .arg(Arg::new("renderer").required(true))
                .about("Whether a given renderer is supported by this preprocessor"),
        )
        .arg(
            Arg::new("skip_tests")
                .short('s')
                .help("Skip running doctests"),
        )
        .arg(
            Arg::new("manifest_dir")
            .long("manifest-dir")
            .help("Path to directory containing outermost autocxx Cargo.toml; necessary for trybuild to build test code successfully")
            .default_value_os(calculate_cargo_dir().as_os_str())
        )
        .get_matches();
    if let Some(supports_matches) = matches.subcommand_matches("supports") {
        // Only do our preprocessing and testing for the html renderer, not linkcheck.
        if supports_matches.value_of("renderer") == Some("html") {
            process::exit(0);
        } else {
            process::exit(1);
        }
    }
    preprocess(&matches).unwrap();
}

fn calculate_cargo_dir() -> PathBuf {
    let mut path = std::env::current_exe().unwrap();
    for _ in 0..3 {
        path = path.parent().map(|p| p.to_path_buf()).unwrap_or(path);
    }
    path.join("integration-tests")
}

fn preprocess(args: &ArgMatches) -> Result<(), Error> {
    let (_, mut book) = CmdPreprocessor::parse_input(io::stdin())?;

    env_logger::builder().init();
    let mut test_cases = Vec::new();

    Book::for_each_mut(&mut book, |sec| {
        if let mdbook::BookItem::Chapter(chapter) = sec {
            let filename = chapter
                .path
                .as_ref()
                .map(|pb| pb.to_string_lossy())
                .unwrap_or_default()
                .to_string();
            chapter.content = substitute_chapter(&chapter.content, &filename, &mut test_cases);
        }
    });

    // Now run any test cases we accumulated.
    if !args.is_present("skip_tests") {
        let stdout_gag = gag::BufferRedirect::stdout().unwrap();
        let num_tests = test_cases.len();
        let fails: Vec<_> = test_cases
            .into_par_iter()
            .enumerate()
            .filter_map(|(counter, case)| {
                if let Ok(test) = std::env::var(RUST_MDBOOK_SINGLE_TEST) {
                    let desired_id: usize = test.parse().unwrap();
                    if desired_id != (counter + 1) {
                        return None;
                    }
                }
                eprintln!(
                    "Running doctest {}/{} at {}",
                    counter + 1,
                    num_tests,
                    &case.location
                );
                let err = autocxx_integration_tests::doctest(
                    &case.cpp,
                    &case.hdr,
                    case.rs,
                    &OsString::from(args.value_of("manifest_dir").unwrap()),
                );
                let desc = match err {
                    Ok(_) => "passed".to_string(),
                    Err(ref err) => format!("failed: {:?}", err),
                };
                eprintln!(
                    "Doctest {}/{} at {} {}.",
                    counter + 1,
                    num_tests,
                    &case.location,
                    desc
                );
                if err.is_err() {
                    Some(TestId {
                        location: case.location,
                        test_id: counter + 1,
                    })
                } else {
                    None
                }
            })
            .collect();
        let mut stdout_str = String::new();
        stdout_gag
            .into_inner()
            .read_to_string(&mut stdout_str)
            .unwrap();
        if !stdout_str.is_empty() {
            eprintln!("Stdout from tests:\n{}", stdout_str);
        }
{'fraction_non_alphanumeric': 0.12226184411614875, 'fraction_numerical': 0.003056546102903719, 'mean_word_length': 1.8855952964233218, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5699420', 'n_tokens_mistral': 3417, 'n_tokens_neox': 3282, 'n_words': 859}