<commit_before>extern crate bindgen;
extern crate clap;
extern crate diff;
#[cfg(feature = "logging")]
extern crate env_logger;
extern crate shlex;

use bindgen::{clang_version, Builder};
use std::env;
use std::fs;
use std::io::{self, BufRead, BufReader, Error, ErrorKind, Read, Write};
use std::path::PathBuf;
use std::process;
use std::sync::{Once, ONCE_INIT};

#[path = "../src/options.rs"]
mod options;
use options::builder_from_flags;

/// The environment variable that determines if test expectations are overwritten.
static OVERWRITE_ENV_VAR: &str = "BINDGEN_OVERWRITE_EXPECTED";

// Run `rustfmt` on the given source string and return a tuple of the formatted
// bindings, and rustfmt's stderr.
fn rustfmt(source: String) -> (String, String) {
    static CHECK_RUSTFMT: Once = ONCE_INIT;

    CHECK_RUSTFMT.call_once(|| {
        let have_working_rustfmt = process::Command::new("rustup")
            .args(&["run", "nightly", "rustfmt", "--version"])
            .stdout(process::Stdio::null())
            .stderr(process::Stdio::null())
            .status()
            .ok()
            .map_or(false, |status| status.success());

        if !have_working_rustfmt {
            panic!(
                "
The latest `rustfmt` is required to run the `bindgen` test suite. Install
`rustfmt` with:

    $ rustup update nightly
    $ rustup run nightly cargo install -f rustfmt-nightly
"
            );
        }
    });

    let mut child = process::Command::new("rustup")
        .args(&[
            "run",
            "nightly",
            "rustfmt",
            "--config-path",
            concat!(env!("CARGO_MANIFEST_DIR"), "/tests/rustfmt.toml"),
        ])
        .stdin(process::Stdio::piped())
        .stdout(process::Stdio::piped())
        .stderr(process::Stdio::piped())
        .spawn()
        .expect("should spawn `rustup run nightly rustfmt`");

    let mut stdin = child.stdin.take().unwrap();
    let mut stdout = child.stdout.take().unwrap();
    let mut stderr = child.stderr.take().unwrap();

    // Write to stdin in a new thread, so that we can read from stdout on this
    // thread. This keeps the child from blocking on writing to its stdout which
    // might block us from writing to its stdin.
    let stdin_handle = ::std::thread::spawn(move || stdin.write_all(source.as_bytes()));

    // Read stderr on a new thread for similar reasons.
    let stderr_handle = ::std::thread::spawn(move || {
        let mut output = vec![];
        io::copy(&mut stderr, &mut output).map(|_| String::from_utf8_lossy(&output).to_string())
    });

    let mut output = vec![];
    io::copy(&mut stdout, &mut output).expect("Should copy stdout into vec OK");

    // Ignore actual rustfmt status because it is often non-zero for trivial
    // things.
    let _ = child.wait().expect("should wait on rustfmt child OK");

    stdin_handle
        .join()
        .expect("writer thread should not have panicked")
        .expect("should have written to child rustfmt's stdin OK");

    let bindings = String::from_utf8(output).expect("rustfmt should only emit valid utf-8");

    let stderr = stderr_handle
        .join()
        .expect("stderr reader thread should not have panicked")
        .expect("should have read child rustfmt's stderr OK");

    (bindings, stderr)
}

fn compare_generated_header(header: &PathBuf, builder: Builder) -> Result<(), Error> {
    let file_name = header.file_name().ok_or(Error::new(
        ErrorKind::Other,
        "compare_generated_header expects a file",
    ))?;

    let mut expectation = PathBuf::from(header);
    expectation.pop();
    expectation.pop();
    expectation.push("expectations");
    expectation.push("tests");

    let mut looked_at = vec![];
    let mut expectation_file;

    // Try more specific expectations first.
    {
        let mut expectation = expectation.clone();

        if cfg!(feature = "testing_only_libclang_4") {
            expectation.push("libclang-4");
        } else if cfg!(feature = "testing_only_libclang_5") {
            expectation.push("libclang-5");
        } else if cfg!(feature = "testing_only_libclang_3_9") {
            expectation.push("libclang-3.9");
        } else if cfg!(feature = "testing_only_libclang_3_8") {
            expectation.push("libclang-3.8");
        } else {
            match clang_version().parsed {
                None => {}
                Some(version) => {
                    let (maj, min) = version;
                    let version_str = if maj >= 5 {
                        "5".to_owned()
                    } else if maj == 4 {
                        "4".to_owned()
                    } else {
                        format!("{}.{}", maj, min)
                    };
                    expectation.push(format!("libclang-{}", version_str));
                }
            }
        }<commit_msg>tests: Look at expectations/tests/libclang-9 expectations.<commit_after>extern crate bindgen;
extern crate clap;
extern crate diff;
#[cfg(feature = "logging")]
extern crate env_logger;
extern crate shlex;

use bindgen::{clang_version, Builder};
use std::env;
use std::fs;
use std::io::{self, BufRead, BufReader, Error, ErrorKind, Read, Write};
use std::path::PathBuf;
use std::process;
use std::sync::{Once, ONCE_INIT};

#[path = "../src/options.rs"]
mod options;
use options::builder_from_flags;

/// The environment variable that determines if test expectations are overwritten.
static OVERWRITE_ENV_VAR: &str = "BINDGEN_OVERWRITE_EXPECTED";

// Run `rustfmt` on the given source string and return a tuple of the formatted
// bindings, and rustfmt's stderr.
fn rustfmt(source: String) -> (String, String) {
    static CHECK_RUSTFMT: Once = ONCE_INIT;

    CHECK_RUSTFMT.call_once(|| {
        let have_working_rustfmt = process::Command::new("rustup")
            .args(&["run", "nightly", "rustfmt", "--version"])
            .stdout(process::Stdio::null())
            .stderr(process::Stdio::null())
            .status()
            .ok()
            .map_or(false, |status| status.success());

        if !have_working_rustfmt {
            panic!(
                "
The latest `rustfmt` is required to run the `bindgen` test suite. Install
`rustfmt` with:

    $ rustup update nightly
    $ rustup run nightly cargo install -f rustfmt-nightly
"
            );
        }
    });

    let mut child = process::Command::new("rustup")
        .args(&[
            "run",
            "nightly",
            "rustfmt",
            "--config-path",
            concat!(env!("CARGO_MANIFEST_DIR"), "/tests/rustfmt.toml"),
        ])
        .stdin(process::Stdio::piped())
        .stdout(process::Stdio::piped())
        .stderr(process::Stdio::piped())
        .spawn()
        .expect("should spawn `rustup run nightly rustfmt`");

    let mut stdin = child.stdin.take().unwrap();
    let mut stdout = child.stdout.take().unwrap();
    let mut stderr = child.stderr.take().unwrap();

    // Write to stdin in a new thread, so that we can read from stdout on this
    // thread. This keeps the child from blocking on writing to its stdout which
    // might block us from writing to its stdin.
    let stdin_handle = ::std::thread::spawn(move || stdin.write_all(source.as_bytes()));

    // Read stderr on a new thread for similar reasons.
    let stderr_handle = ::std::thread::spawn(move || {
        let mut output = vec![];
        io::copy(&mut stderr, &mut output).map(|_| String::from_utf8_lossy(&output).to_string())
    });

    let mut output = vec![];
    io::copy(&mut stdout, &mut output).expect("Should copy stdout into vec OK");

    // Ignore actual rustfmt status because it is often non-zero for trivial
    // things.
    let _ = child.wait().expect("should wait on rustfmt child OK");

    stdin_handle
        .join()
        .expect("writer thread should not have panicked")
        .expect("should have written to child rustfmt's stdin OK");

    let bindings = String::from_utf8(output).expect("rustfmt should only emit valid utf-8");

    let stderr = stderr_handle
        .join()
        .expect("stderr reader thread should not have panicked")
        .expect("should have read child rustfmt's stderr OK");

    (bindings, stderr)
}

fn compare_generated_header(header: &PathBuf, builder: Builder) -> Result<(), Error> {
    let file_name = header.file_name().ok_or(Error::new(
        ErrorKind::Other,
        "compare_generated_header expects a file",
    ))?;

    let mut expectation = PathBuf::from(header);
    expectation.pop();
    expectation.pop();
    expectation.push("expectations");
    expectation.push("tests");

    let mut looked_at = vec![];
    let mut expectation_file;

    // Try more specific expectations first.
    {
        let mut expectation = expectation.clone();

        if cfg!(feature = "testing_only_libclang_4") {
            expectation.push("libclang-4");
        } else if cfg!(feature = "testing_only_libclang_5") {
            expectation.push("libclang-5");
        } else if cfg!(feature = "testing_only_libclang_3_9") {
            expectation.push("libclang-3.9");
        } else if cfg!(feature = "testing_only_libclang_3_8") {
            expectation.push("libclang-3.8");
        } else {
            match clang_version().parsed {
                None => {}
                Some(version) => {
                    let (maj, min) = version;
                    let version_str = if maj >= 9 {
                        "9".to_owned()
                    } else if maj >= 5 {
                        "5".to_owned()
                    } else if maj == 4 {
                        "4".to_owned()
                    } else {
                        format!("{}.{}", maj, min)
                    };
                    expectation.push(format!("libclang-{}", version_str));
                }
            }
        }
{'fraction_non_alphanumeric': 0.13518725261341724, 'fraction_numerical': 0.004161169187049629, 'mean_word_length': 2.351700680272109, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5581541', 'n_tokens_mistral': 2921, 'n_tokens_neox': 2860, 'n_words': 775}