<commit_before>extern crate log;
extern crate env_logger;
extern crate time;

use env_logger::LogBuilder;
use log::{LogRecord, LogLevelFilter};
use std::env;
use std::sync::{Once, ONCE_INIT};

static INIT_LOG: Once = ONCE_INIT;
// use in application
pub fn init() {
    init_filter(LogLevelFilter::Info);
}

// use in unit case
pub fn silent() {
    init_filter(LogLevelFilter::Off);
}
fn init_filter(filter: LogLevelFilter) {
    INIT_LOG.call_once(|| {
        let format = |record: &LogRecord| {
            let now = time::now();
            format!("{} - {} - {}", time::strftime("%Y%m%d %H:%M:%S", &now).unwrap(), record.level(), record.args())
        };

        let mut builder = LogBuilder::new();
        builder.format(format).filter(None, filter);

        if env::var("RUST_LOG").is_ok() {
            builder.parse(&env::var("RUST_LOG").unwrap());<commit_msg>log to stdout<commit_after>extern crate log;
extern crate env_logger;
extern crate time;

use env_logger::{LogBuilder, LogTarget};
use log::{LogRecord, LogLevelFilter};
use std::env;
use std::sync::{Once, ONCE_INIT};

static INIT_LOG: Once = ONCE_INIT;
// use in application
pub fn init() {
    init_filter(LogLevelFilter::Info);
}

// use in unit case
pub fn silent() {
    init_filter(LogLevelFilter::Off);
}
fn init_filter(filter: LogLevelFilter) {
    INIT_LOG.call_once(|| {
        let format = |record: &LogRecord| {
            let now = time::now();
            format!("{} - {} - {}", time::strftime("%Y%m%d %H:%M:%S", &now).unwrap(), record.level(), record.args())
        };

        let mut builder = LogBuilder::new();
        // log to stdout
        builder.target(LogTarget::Stdout);
        builder.format(format).filter(None, filter);

        if env::var("RUST_LOG").is_ok() {
            builder.parse(&env::var("RUST_LOG").unwrap());
{'fraction_non_alphanumeric': 0.17331875341716785, 'fraction_numerical': 0.0, 'mean_word_length': 3.485294117647059, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4796111', 'n_tokens_mistral': 598, 'n_tokens_neox': 596, 'n_words': 134}