<commit_before>#![feature(test)]

extern crate test;

use std::env;
use std::fs::File;
use std::io::Read;

fn get_file_content(arg: String) -> std::io::Result<String> {
    let mut content = String::new();
    let mut file = try!(File::open(arg));
    try!(file.read_to_string(&mut content));
    Ok(content)
}

mod tokenizer;
use tokenizer::tokenize;

fn main() {
    for argument in env::args().skip(1) {
        let content = &get_file_content(argument).unwrap();
        println!("Started tokenization!");
        let tokens = tokenize(content);
        println!("Tokenized!");
        // for token in &tokens {
        //     println!("{:?}", token);
        // }
        println!("{:#?}", tokens.len());
    }
}<commit_msg>Remove unnecessary bits in main.<commit_after>#![feature(test)]

extern crate test;

use std::env;
use std::fs::File;
use std::io::Read;

fn get_file_content(arg: String) -> std::io::Result<String> {
    let mut content = String::new();
    let mut file = try!(File::open(arg));
    try!(file.read_to_string(&mut content));
    Ok(content)
}

mod tokenizer;
use tokenizer::tokenize;

fn main() {
    for argument in env::args().skip(1) {
        let content = &get_file_content(argument).unwrap();
        let tokens = tokenize(content);
        println!("Token amount: {:#?}", tokens.len());
    }
}
{'fraction_non_alphanumeric': 0.18510158013544017, 'fraction_numerical': 0.0015048908954100827, 'mean_word_length': 3.5238095238095237, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6663192', 'n_tokens_mistral': 438, 'n_tokens_neox': 434, 'n_words': 100}