<commit_before>use std::io;
use std::io::prelude::*;
//use std::fs::File;

extern crate time;

fn main() {

    // <134>2017-04-24T20:38:26Z cache-iad2645 fastly-logs-5-east[367774]: 192.168.3.11 "-" "GET
    // /npm/public/registry/i/is-utf8/_attachments/doc.json" 200 "npm/4.2.0 node/v7.8.0 linux x64"
    // "install" "26f82920d62b24e9" "HIT" "(null)" "cache-iad2645-IAD" "0" "1016" "1466" "251"

    // only quoted values have spaces.
    let mut value = String::new();
    let mut values = Vec::new();
    let mut state = "out";

    for byte in io::stdin().bytes() {
        let c = byte.unwrap() as char;

        if c == '\n' {
            // if i have a pending value add it to values
            if value.len() > 0 {
                values.push(value)
            }

            format(&values);

            value = String::new();
            values = Vec::new();
            continue;
        }

        if state == "out" {
            if c == '"' {
                state = "quoted";
                // skip quote
                continue;
            } else if c != ' ' {
                state = "value";
            }
        }

        if state == "value" {
            if c == ' ' {
                values.push(value);
                value = String::new();
                state = "out";
            } else if c != '"' {
                value.push(c);
            }
        }

        if state == "quoted" {
            if c == '"' {
                values.push(value);
                value = String::new();
                state = "out";
            } else {
                value.push(c);
            }
        }
    }
}


fn format(values: &Vec<String>) {
    let len = values.len();

    if len == 0 {
        return;
    }

    let date = parse_date(values[0].clone());
    let unixtime = logtime_to_unixtime(&date);

    // invalid time. wont count.
    if unixtime == 0 {
        return;
    }

    let pop = substr(&values[1], 6, 3);

    // if i have any values emit them.

    let service = path_to_service(&values[5]);

    let status = &values[6];

    let mut offset = 0;
    if len == 18 {
        offset = 1;
    }

    let egress_bytes = &values[15 + offset];
    let ingress_bytes = &values[16 + offset];

    for v in values {
        print!("{}, ", v);
    }

    println!("[{}]", len);

    println!("format {} {} {} {} {} {} {}",
             date,
             unixtime,
             pop,
             service,
             status,
             egress_bytes,
             ingress_bytes);
}

fn parse_date(dateref: String) -> String {
    let mut date = dateref.clone();
    let offset_option = date.find('>');
<commit_msg>adding minute<commit_after>use std::io;
use std::io::prelude::*;
//use std::fs::File;

extern crate time;

fn main() {

    // <134>2017-04-24T20:38:26Z cache-iad2645 fastly-logs-5-east[367774]: 192.168.3.11 "-" "GET
    // /npm/public/registry/i/is-utf8/_attachments/doc.json" 200 "npm/4.2.0 node/v7.8.0 linux x64"
    // "install" "26f82920d62b24e9" "HIT" "(null)" "cache-iad2645-IAD" "0" "1016" "1466" "251"

    // only quoted values have spaces.
    let mut value = String::new();
    let mut values = Vec::new();
    let mut state = "out";

    for byte in io::stdin().bytes() {
        let c = byte.unwrap() as char;

        if c == '\n' {
            // if i have a pending value add it to values
            if value.len() > 0 {
                values.push(value)
            }

            format(&values);

            value = String::new();
            values = Vec::new();
            continue;
        }

        if state == "out" {
            if c == '"' {
                state = "quoted";
                // skip quote
                continue;
            } else if c != ' ' {
                state = "value";
            }
        }

        if state == "value" {
            if c == ' ' {
                values.push(value);
                value = String::new();
                state = "out";
            } else if c != '"' {
                value.push(c);
            }
        }

        if state == "quoted" {
            if c == '"' {
                values.push(value);
                value = String::new();
                state = "out";
            } else {
                value.push(c);
            }
        }
    }
}


fn format(values: &Vec<String>) {
    let len = values.len();

    if len == 0 {
        return;
    }

    let date = parse_date(values[0].clone());
    let unixtime = logtime_to_unixtime(&date);
    let minute = unixtime - (unixtime % 60);

    // invalid time. wont count.
    if unixtime == 0 {
        return;
    }

    let pop = substr(&values[1], 6, 3);

    // if i have any values emit them.

    let service = path_to_service(&values[5]);

    let status = &values[6];

    let mut offset = 0;
    if len == 18 {
        offset = 1;
    }

    let egress_bytes = &values[15 + offset];
    let ingress_bytes = &values[16 + offset];

    for v in values {
        print!("{}, ", v);
    }

    println!("[{}]", len);

    println!("format {} {} {} {} {} {} {} {}",
             date,
             minute,
             unixtime,
             pop,
             service,
             status,
             egress_bytes,
             ingress_bytes);
}

fn parse_date(dateref: String) -> String {
    let mut date = dateref.clone();
    let offset_option = date.find('>');

{'fraction_non_alphanumeric': 0.1552011867235305, 'fraction_numerical': 0.03523085481179306, 'mean_word_length': 1.3762114537444934, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581896', 'n_tokens_mistral': 1870, 'n_tokens_neox': 1682, 'n_words': 421}