<commit_before>#![deny(warnings)]
#![feature(phase, slicing_syntax)]

#[phase(plugin, link)] extern crate log;
extern crate criterion;
extern crate serialize;

use criterion::Criterion;
use std::io::{USER_DIR, Command, File, fs};
use std::str;

use executable::Executable;
use problem::Problem;
use solution::Solution;

mod compiler;
mod executable;
mod interpreter;
mod language;
mod problem;
mod solution;

fn lscpu() {
    match Command::new("lscpu").output() {
        Err(_) => panic!("Couldn't spawn `lscpu`"),
        Ok(po) => match str::from_utf8(po.output[]) {
            None => panic!("Couldn't parse the output of `lscpu`"),
            Some(output) => {
                let s = output.lines().filter(|line| {
                    !line.starts_with("CPU MHz:")
                }).collect::<Vec<_>>().connect("\n");

                match File::create(&Path::new("lscpu")).write_str(s.as_slice()) {
                    Err(_) => panic!("Couln't write to the cpu file"),
                    Ok(_) => {},
                }
            }
        },
    }
}

fn main() {
    let languages = language::all();

    lscpu();

    for problem in fs::readdir(&Path::new("problems")).unwrap().into_iter().filter_map(|dir| {
        Problem::new(dir)
    }) {
        let mut processed = 0u;

        for solution in languages.iter().filter_map(|language| {
            Solution::new(language, &problem)
        }) {<commit_msg>fix(main): `from_utf8` now returns `Result` instead of `Option`<commit_after>#![deny(warnings)]
#![feature(phase, slicing_syntax)]

#[phase(plugin, link)] extern crate log;
extern crate criterion;
extern crate serialize;

use criterion::Criterion;
use std::io::{USER_DIR, Command, File, fs};
use std::str;

use executable::Executable;
use problem::Problem;
use solution::Solution;

mod compiler;
mod executable;
mod interpreter;
mod language;
mod problem;
mod solution;

fn lscpu() {
    match Command::new("lscpu").output() {
        Err(_) => panic!("Couldn't spawn `lscpu`"),
        Ok(po) => match str::from_utf8(po.output[]) {
            Err(_) => panic!("Couldn't parse the output of `lscpu`"),
            Ok(output) => {
                let s = output.lines().filter(|line| {
                    !line.starts_with("CPU MHz:")
                }).collect::<Vec<_>>().connect("\n");

                match File::create(&Path::new("lscpu")).write_str(s.as_slice()) {
                    Err(_) => panic!("Couln't write to the cpu file"),
                    Ok(_) => {},
                }
            }
        },
    }
}

fn main() {
    let languages = language::all();

    lscpu();

    for problem in fs::readdir(&Path::new("problems")).unwrap().into_iter().filter_map(|dir| {
        Problem::new(dir)
    }) {
        let mut processed = 0u;

        for solution in languages.iter().filter_map(|language| {
            Solution::new(language, &problem)
        }) {
{'fraction_non_alphanumeric': 0.17297111416781294, 'fraction_numerical': 0.0017193947730398899, 'mean_word_length': 2.591358024691358, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1396145', 'n_tokens_mistral': 932, 'n_tokens_neox': 883, 'n_words': 201}