<commit_before>//! Fully type-check project and print various stats, like the number of type
//! errors.

use std::{
    path::Path,
    time::{SystemTime, UNIX_EPOCH},
};

use hir::{
    db::{AstDatabase, DefDatabase, HirDatabase},
    original_range, AssocItem, Crate, HasSource, HirDisplay, ModuleDef,
};
use hir_def::FunctionId;
use hir_ty::{Ty, TypeWalk};
use itertools::Itertools;
use oorandom::Rand32;
use ra_db::{
    salsa::{self, ParallelDatabase},
    SourceDatabaseExt,
};
use ra_syntax::AstNode;
use rayon::prelude::*;
use rustc_hash::FxHashSet;
use stdx::format_to;

use crate::{
    cli::{
        load_cargo::load_cargo, progress_report::ProgressReport, report_metric, Result, Verbosity,
    },
    print_memory_usage,
};
use ra_prof::StopWatch;

/// Need to wrap Snapshot to provide `Clone` impl for `map_with`
struct Snap<DB>(DB);
impl<DB: ParallelDatabase> Clone for Snap<salsa::Snapshot<DB>> {
    fn clone(&self) -> Snap<salsa::Snapshot<DB>> {
        Snap(self.0.snapshot())
    }
}

pub fn analysis_stats(
    verbosity: Verbosity,
    memory_usage: bool,
    path: &Path,
    only: Option<&str>,
    with_deps: bool,
    randomize: bool,
    parallel: bool,
    load_output_dirs: bool,
    with_proc_macro: bool,
) -> Result<()> {
    let mut rng = {
        let seed = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64;
        Rand32::new(seed)
    };

    let mut db_load_sw = StopWatch::start().memory(memory_usage);
    let (host, vfs) = load_cargo(path, load_output_dirs, with_proc_macro)?;
    let db = host.raw_database();
    eprintln!("Database loaded {}", db_load_sw.elapsed());

    let mut analysis_sw = StopWatch::start().memory(memory_usage);
    let mut num_crates = 0;
    let mut visited_modules = FxHashSet::default();
    let mut visit_queue = Vec::new();

    let mut krates = Crate::all(db);
    if randomize {
        shuffle(&mut rng, &mut krates);
    }
    for krate in krates {
        let module = krate.root_module(db).expect("crate without root module");
        let file_id = module.definition_source(db).file_id;
        let file_id = file_id.original_file(db);
        let source_root = db.file_source_root(file_id);
        let source_root = db.source_root(source_root);
        if !source_root.is_library || with_deps {
            num_crates += 1;
            visit_queue.push(module);
        }
    }

    if randomize {
        shuffle(&mut rng, &mut visit_queue);
    }

    eprintln!("Crates in this dir: {}", num_crates);
    let mut num_decls = 0;
    let mut funcs = Vec::new();
    while let Some(module) = visit_queue.pop() {
        if visited_modules.insert(module) {
            visit_queue.extend(module.children(db));

            for decl in module.declarations(db) {
                num_decls += 1;
                if let ModuleDef::Function(f) = decl {
                    funcs.push(f);
                }
            }

            for impl_def in module.impl_defs(db) {
                for item in impl_def.items(db) {
                    num_decls += 1;
                    if let AssocItem::Function(f) = item {
                        funcs.push(f);
                    }
                }
            }
        }
    }
    eprintln!("Total modules found: {}", visited_modules.len());
    eprintln!("Total declarations: {}", num_decls);
    eprintln!("Total functions: {}", funcs.len());
    eprintln!("Item Collection: {}", analysis_sw.elapsed());

    if randomize {
        shuffle(&mut rng, &mut funcs);
    }

    let mut bar = match verbosity {
        Verbosity::Quiet | Verbosity::Spammy => ProgressReport::hidden(),
        _ => ProgressReport::new(funcs.len() as u64),
    };<commit_msg>Do not show progress with parallel analysis<commit_after>//! Fully type-check project and print various stats, like the number of type
//! errors.

use std::{
    path::Path,
    time::{SystemTime, UNIX_EPOCH},
};

use hir::{
    db::{AstDatabase, DefDatabase, HirDatabase},
    original_range, AssocItem, Crate, HasSource, HirDisplay, ModuleDef,
};
use hir_def::FunctionId;
use hir_ty::{Ty, TypeWalk};
use itertools::Itertools;
use oorandom::Rand32;
use ra_db::{
    salsa::{self, ParallelDatabase},
    SourceDatabaseExt,
};
use ra_syntax::AstNode;
use rayon::prelude::*;
use rustc_hash::FxHashSet;
use stdx::format_to;

use crate::{
    cli::{
        load_cargo::load_cargo, progress_report::ProgressReport, report_metric, Result, Verbosity,
    },
    print_memory_usage,
};
use ra_prof::StopWatch;

/// Need to wrap Snapshot to provide `Clone` impl for `map_with`
struct Snap<DB>(DB);
impl<DB: ParallelDatabase> Clone for Snap<salsa::Snapshot<DB>> {
    fn clone(&self) -> Snap<salsa::Snapshot<DB>> {
        Snap(self.0.snapshot())
    }
}

pub fn analysis_stats(
    verbosity: Verbosity,
    memory_usage: bool,
    path: &Path,
    only: Option<&str>,
    with_deps: bool,
    randomize: bool,
    parallel: bool,
    load_output_dirs: bool,
    with_proc_macro: bool,
) -> Result<()> {
    let mut rng = {
        let seed = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64;
        Rand32::new(seed)
    };

    let mut db_load_sw = StopWatch::start().memory(memory_usage);
    let (host, vfs) = load_cargo(path, load_output_dirs, with_proc_macro)?;
    let db = host.raw_database();
    eprintln!("Database loaded {}", db_load_sw.elapsed());

    let mut analysis_sw = StopWatch::start().memory(memory_usage);
    let mut num_crates = 0;
    let mut visited_modules = FxHashSet::default();
    let mut visit_queue = Vec::new();

    let mut krates = Crate::all(db);
    if randomize {
        shuffle(&mut rng, &mut krates);
    }
    for krate in krates {
        let module = krate.root_module(db).expect("crate without root module");
        let file_id = module.definition_source(db).file_id;
        let file_id = file_id.original_file(db);
        let source_root = db.file_source_root(file_id);
        let source_root = db.source_root(source_root);
        if !source_root.is_library || with_deps {
            num_crates += 1;
            visit_queue.push(module);
        }
    }

    if randomize {
        shuffle(&mut rng, &mut visit_queue);
    }

    eprintln!("Crates in this dir: {}", num_crates);
    let mut num_decls = 0;
    let mut funcs = Vec::new();
    while let Some(module) = visit_queue.pop() {
        if visited_modules.insert(module) {
            visit_queue.extend(module.children(db));

            for decl in module.declarations(db) {
                num_decls += 1;
                if let ModuleDef::Function(f) = decl {
                    funcs.push(f);
                }
            }

            for impl_def in module.impl_defs(db) {
                for item in impl_def.items(db) {
                    num_decls += 1;
                    if let AssocItem::Function(f) = item {
                        funcs.push(f);
                    }
                }
            }
        }
    }
    eprintln!("Total modules found: {}", visited_modules.len());
    eprintln!("Total declarations: {}", num_decls);
    eprintln!("Total functions: {}", funcs.len());
    eprintln!("Item Collection: {}", analysis_sw.elapsed());

    if randomize {
        shuffle(&mut rng, &mut funcs);
    }

    let mut bar = match verbosity {
        Verbosity::Quiet | Verbosity::Spammy => ProgressReport::hidden(),
        _ if parallel => ProgressReport::hidden(),
        _ => ProgressReport::new(funcs.len() as u64),
    };
{'fraction_non_alphanumeric': 0.13346693386773548, 'fraction_numerical': 0.0037408149632598532, 'mean_word_length': 2.662426614481409, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3090', 'n_tokens_mistral': 2466, 'n_tokens_neox': 2455, 'n_words': 543}