<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use syn;

use bindgen::cargo::{Cargo, PackageRef};
use bindgen::error::Error;
use bindgen::ir::{AnnotationSet, Cfg, Constant, Documentation, Enum, Function, GenericParams};
use bindgen::ir::{ItemMap, OpaqueItem, Static, Struct, Typedef, Union};
use bindgen::utilities::{SynAbiHelpers, SynItemHelpers};

const STD_CRATES: &'static [&'static str] = &[
    "std",
    "std_unicode",
    "alloc",
    "collections",
    "core",
    "proc_macro",
];

type ParseResult = Result<Parse, Error>;

/// Parses a single rust source file, not following `mod` or `extern crate`.
pub fn parse_src(src_file: &Path) -> ParseResult {
    let mod_name = src_file.file_stem().unwrap().to_str().unwrap();

    let mut context = Parser {
        binding_crate_name: mod_name.to_owned(),
        lib: None,
        parse_deps: true,
        include: None,
        exclude: Vec::new(),
        expand: Vec::new(),
        expand_all_features: true,
        expand_default_features: true,
        expand_features: None,
        parsed_crates: HashSet::new(),
        cache_src: HashMap::new(),
        cache_expanded_crate: HashMap::new(),
        cfg_stack: Vec::new(),
        out: Parse::new(),
    };

    let pkg_ref = PackageRef {
        name: mod_name.to_owned(),
        version: "0.0.0".to_owned(),
    };

    context.parse_mod(&pkg_ref, src_file)?;
    Ok(context.out)
}

/// Recursively parses a rust library starting at the root crate's directory.
///
/// Inside a crate, `mod` and `extern crate` declarations are followed
/// and parsed. To find an external crate, the parser uses the `cargo metadata`
/// command to find the location of dependencies.
pub(crate) fn parse_lib(
    lib: Cargo,
    parse_deps: bool,
    include: &Option<Vec<String>>,
    exclude: &Vec<String>,
    expand: &Vec<String>,
    expand_all_features: bool,
    expand_default_features: bool,
    expand_features: &Option<Vec<String>>,
) -> ParseResult {
    let mut context = Parser {
        binding_crate_name: lib.binding_crate_name().to_owned(),
        lib: Some(lib),
        parse_deps: parse_deps,
        include: include.clone(),
        exclude: exclude.clone(),
        expand: expand.clone(),
        expand_all_features,
        expand_default_features,
        expand_features: expand_features.clone(),
        parsed_crates: HashSet::new(),
        cache_src: HashMap::new(),
        cache_expanded_crate: HashMap::new(),
        cfg_stack: Vec::new(),
        out: Parse::new(),
    };

    let binding_crate = context.lib.as_ref().unwrap().binding_crate_ref();
    context.parse_crate(&binding_crate)?;
    Ok(context.out)
}

#[derive(Debug, Clone)]
struct Parser {
    binding_crate_name: String,
    lib: Option<Cargo>,
    parse_deps: bool,

    include: Option<Vec<String>>,
    exclude: Vec<String>,
    expand: Vec<String>,
    expand_all_features: bool,
    expand_default_features: bool,
    expand_features: Option<Vec<String>>,

    parsed_crates: HashSet<String>,
    cache_src: HashMap<PathBuf, Vec<syn::Item>>,
    cache_expanded_crate: HashMap<String, Vec<syn::Item>>,

    cfg_stack: Vec<Cfg>,

    out: Parse,
}

impl Parser {
    fn should_parse_dependency(&self, pkg_name: &String) -> bool {
        if self.parsed_crates.contains(pkg_name) {
            return false;
        }

        if !self.parse_deps {
            return false;
        }

        // Skip any whitelist or blacklist for expand
        if self.expand.contains(&pkg_name) {
            return true;
        }

        // If we have a whitelist, check it
        if let Some(ref include) = self.include {
            if !include.contains(&pkg_name) {
                return false;
            }
        }

        // Check the blacklist
        return !STD_CRATES.contains(&pkg_name.as_ref()) && !self.exclude.contains(&pkg_name);
    }

    fn parse_crate(&mut self, pkg: &PackageRef) -> Result<(), Error> {
        assert!(self.lib.is_some());
        self.parsed_crates.insert(pkg.name.clone());

        // Check if we should use cargo expand for this crate
        if self.expand.contains(&pkg.name) {
            return self.parse_expand_crate(pkg);
        }

        // Otherwise do our normal parse
        let crate_src = self.lib.as_ref().unwrap().find_crate_src(pkg);

        match crate_src {
            Some(crate_src) => self.parse_mod(pkg, crate_src.as_path()),
            None => {
                // This should be an error, but is common enough to just elicit a warning
                warn!(
                    "Parsing crate `{}`: can't find lib.rs with `cargo metadata`.",
                    pkg.name
                );
                Ok(())
            }
        }
    }

    fn parse_expand_crate(&mut self, pkg: &PackageRef) -> Result<(), Error> {
        assert!(self.lib.is_some());

        let mod_parsed = {
            if !self.cache_expanded_crate.contains_key(&pkg.name) {
                let s = self
                    .lib
                    .as_ref()
                    .unwrap()
                    .expand_crate(
                        pkg,
                        self.expand_all_features,
                        self.expand_default_features,
                        &self.expand_features,
                    )
                    .map_err(|x| Error::CargoExpand(pkg.name.clone(), x))?;
                let i = syn::parse_file(&s).map_err(|x| Error::ParseSyntaxError {
                    crate_name: pkg.name.clone(),
                    src_path: "".to_owned(),
                    error: x,
                })?;
                self.cache_expanded_crate.insert(pkg.name.clone(), i.items);
            }

            self.cache_expanded_crate.get(&pkg.name).unwrap().clone()
        };

        self.process_expanded_mod(pkg, &mod_parsed)
    }

    fn process_expanded_mod(&mut self, pkg: &PackageRef, items: &[syn::Item]) -> Result<(), Error> {
        self.out.load_syn_crate_mod(
            &self.binding_crate_name,
            &pkg.name,
            &Cfg::join(&self.cfg_stack),
            items,
        );
<commit_msg>Rustfmt<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use syn;

use bindgen::cargo::{Cargo, PackageRef};
use bindgen::error::Error;
use bindgen::ir::{AnnotationSet, Cfg, Constant, Documentation, Enum, Function, GenericParams};
use bindgen::ir::{ItemMap, OpaqueItem, Static, Struct, Typedef, Union};
use bindgen::utilities::{SynAbiHelpers, SynItemHelpers};

const STD_CRATES: &'static [&'static str] = &[
    "std",
    "std_unicode",
    "alloc",
    "collections",
    "core",
    "proc_macro",
];

type ParseResult = Result<Parse, Error>;

/// Parses a single rust source file, not following `mod` or `extern crate`.
pub fn parse_src(src_file: &Path) -> ParseResult {
    let mod_name = src_file.file_stem().unwrap().to_str().unwrap();

    let mut context = Parser {
        binding_crate_name: mod_name.to_owned(),
        lib: None,
        parse_deps: true,
        include: None,
        exclude: Vec::new(),
        expand: Vec::new(),
        expand_all_features: true,
        expand_default_features: true,
        expand_features: None,
        parsed_crates: HashSet::new(),
        cache_src: HashMap::new(),
        cache_expanded_crate: HashMap::new(),
        cfg_stack: Vec::new(),
        out: Parse::new(),
    };

    let pkg_ref = PackageRef {
        name: mod_name.to_owned(),
        version: "0.0.0".to_owned(),
    };

    context.parse_mod(&pkg_ref, src_file)?;
    Ok(context.out)
}

/// Recursively parses a rust library starting at the root crate's directory.
///
/// Inside a crate, `mod` and `extern crate` declarations are followed
/// and parsed. To find an external crate, the parser uses the `cargo metadata`
/// command to find the location of dependencies.
pub(crate) fn parse_lib(
    lib: Cargo,
    parse_deps: bool,
    include: &Option<Vec<String>>,
    exclude: &Vec<String>,
    expand: &Vec<String>,
    expand_all_features: bool,
    expand_default_features: bool,
    expand_features: &Option<Vec<String>>,
) -> ParseResult {
    let mut context = Parser {
        binding_crate_name: lib.binding_crate_name().to_owned(),
        lib: Some(lib),
        parse_deps: parse_deps,
        include: include.clone(),
        exclude: exclude.clone(),
        expand: expand.clone(),
        expand_all_features,
        expand_default_features,
        expand_features: expand_features.clone(),
        parsed_crates: HashSet::new(),
        cache_src: HashMap::new(),
        cache_expanded_crate: HashMap::new(),
        cfg_stack: Vec::new(),
        out: Parse::new(),
    };

    let binding_crate = context.lib.as_ref().unwrap().binding_crate_ref();
    context.parse_crate(&binding_crate)?;
    Ok(context.out)
}

#[derive(Debug, Clone)]
struct Parser {
    binding_crate_name: String,
    lib: Option<Cargo>,
    parse_deps: bool,

    include: Option<Vec<String>>,
    exclude: Vec<String>,
    expand: Vec<String>,
    expand_all_features: bool,
    expand_default_features: bool,
    expand_features: Option<Vec<String>>,

    parsed_crates: HashSet<String>,
    cache_src: HashMap<PathBuf, Vec<syn::Item>>,
    cache_expanded_crate: HashMap<String, Vec<syn::Item>>,

    cfg_stack: Vec<Cfg>,

    out: Parse,
}

impl Parser {
    fn should_parse_dependency(&self, pkg_name: &String) -> bool {
        if self.parsed_crates.contains(pkg_name) {
            return false;
        }

        if !self.parse_deps {
            return false;
        }

        // Skip any whitelist or blacklist for expand
        if self.expand.contains(&pkg_name) {
            return true;
        }

        // If we have a whitelist, check it
        if let Some(ref include) = self.include {
            if !include.contains(&pkg_name) {
                return false;
            }
        }

        // Check the blacklist
        return !STD_CRATES.contains(&pkg_name.as_ref()) && !self.exclude.contains(&pkg_name);
    }

    fn parse_crate(&mut self, pkg: &PackageRef) -> Result<(), Error> {
        assert!(self.lib.is_some());
        self.parsed_crates.insert(pkg.name.clone());

        // Check if we should use cargo expand for this crate
        if self.expand.contains(&pkg.name) {
            return self.parse_expand_crate(pkg);
        }

        // Otherwise do our normal parse
        let crate_src = self.lib.as_ref().unwrap().find_crate_src(pkg);

        match crate_src {
            Some(crate_src) => self.parse_mod(pkg, crate_src.as_path()),
            None => {
                // This should be an error, but is common enough to just elicit a warning
                warn!(
                    "Parsing crate `{}`: can't find lib.rs with `cargo metadata`.",
                    pkg.name
                );
                Ok(())
            }
        }
    }

    fn parse_expand_crate(&mut self, pkg: &PackageRef) -> Result<(), Error> {
        assert!(self.lib.is_some());

        let mod_parsed = {
            if !self.cache_expanded_crate.contains_key(&pkg.name) {
                let s = self
                    .lib
                    .as_ref()
                    .unwrap()
                    .expand_crate(
                        pkg,
                        self.expand_all_features,
                        self.expand_default_features,
                        &self.expand_features,
                    ).map_err(|x| Error::CargoExpand(pkg.name.clone(), x))?;
                let i = syn::parse_file(&s).map_err(|x| Error::ParseSyntaxError {
                    crate_name: pkg.name.clone(),
                    src_path: "".to_owned(),
                    error: x,
                })?;
                self.cache_expanded_crate.insert(pkg.name.clone(), i.items);
            }

            self.cache_expanded_crate.get(&pkg.name).unwrap().clone()
        };

        self.process_expanded_mod(pkg, &mod_parsed)
    }

    fn process_expanded_mod(&mut self, pkg: &PackageRef, items: &[syn::Item]) -> Result<(), Error> {
        self.out.load_syn_crate_mod(
            &self.binding_crate_name,
            &pkg.name,
            &Cfg::join(&self.cfg_stack),
            items,
        );

{'fraction_non_alphanumeric': 0.12906767035207403, 'fraction_numerical': 0.0010977809142946758, 'mean_word_length': 2.210976837865055, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3180249', 'n_tokens_mistral': 3919, 'n_tokens_neox': 3865, 'n_words': 884}