<commit_before>use std::io;
use std::io::File;

mod lexer;
mod parser;
mod token;
mod view_writer;

fn get_file_contents() -> Result<~[u8], io::IoError> {
    let path = from_str::<Path>("src/compiler/index.rs.html").unwrap();

    File::open(&path).and_then(|file| {
        let mut file = file;
        file.read_to_end()
    })
}

fn main() {
    let contents = match get_file_contents() {
        Ok(contents) => std::str::from_utf8_owned(contents).expect("Non-utf8 source file"),
        Err(e) => fail!(e)
    };

    let mut lexer = lexer::Lexer::new(contents);
    let mut parser = parser::Parser::new(&mut lexer);
    parser.parse();

    println!("Lines: {}\nLast column: {}", parser.lexer.line, parser.lexer.column);

    //debug!("{}", parser.sections);

    view_writer::write_view(&Path::new("src/compiler/index.rs"), parser.sections.as_slice());
}
<commit_msg>Update for rust changes<commit_after>use std::io;
use std::io::File;

mod lexer;
mod parser;
mod token;
mod view_writer;

fn get_file_contents() -> Result<~str, io::IoError> {
    let path = from_str::<Path>("src/compiler/index.rs.html").unwrap();

    File::open(&path).and_then(|file| {
        let mut file = file;
        file.read_to_str()
    })
}

fn main() {
    let contents = match get_file_contents() {
        Ok(contents) => contents,
        Err(e) => fail!(e)
    };

    let mut lexer = lexer::Lexer::new(contents);
    let mut parser = parser::Parser::new(&mut lexer);
    parser.parse();

    println!("Lines: {}\nLast column: {}", parser.lexer.line, parser.lexer.column);

    //debug!("{}", parser.sections);

    view_writer::write_view(&Path::new("src/compiler/index.rs"), parser.sections.as_slice());
}

{'fraction_non_alphanumeric': 0.19047619047619047, 'fraction_numerical': 0.001763668430335097, 'mean_word_length': 3.701657458563536, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '990885', 'n_tokens_mistral': 598, 'n_tokens_neox': 587, 'n_words': 113}