<commit_before>use std::io;
use std::io::prelude::*;
use std::net::TcpStream;
use std::fs::File;
use std::env;

extern crate chrono;
use chrono::*;

extern crate stats;
extern crate clap;

mod args;
mod http_status;
mod log_parser;
use log_parser::parse_line;
use log_parser::log_events::*;
mod request_response_matcher;
use request_response_matcher::*;
mod filter;
mod analyzer;
mod render;
use render::*;

// http://stackoverflow.com/a/27590832/376138
macro_rules! println_stderr(
    ($($arg:tt)*) => { {
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
);

fn main() {
    let args = args::parse_args(env::args()).unwrap();

    let input: Box<io::Read> = match args.filename.as_ref() {
        "-" => Box::new(io::stdin()),
        _ => Box::new(File::open(&args.filename).unwrap()),
    };

    let (times, first_request) = extract_times(input, &args.conditions);

    let result = analyzer::analyze(&times);

    match result {
        Some(result) => {
            if args.graphite_server.is_some() {
                let mut stream = TcpStream::connect((args.graphite_server.unwrap().as_ref(),
                                                 args.graphite_port.unwrap()))
                    .expect("Could not connect to the Graphite server");
                let timezone = first_request.unwrap().time.timezone();

                let mut renderer = render::GraphiteRenderer::new(
                    UTC::now().with_timezone(&timezone),
                    args.graphite_prefix,
                    &mut stream,
                );
                renderer.render(result);
            } else {
                let mut renderer = render::TerminalRenderer::new();
                renderer.render(result);
            }
        }
        None => println_stderr!("No matching log lines in file."),
    }
}

fn extract_times(input: Box<io::Read>,
                 conditions: &filter::FilterConditions)
                 -> (Vec<i64>, Option<Request>) {
    let reader = io::BufReader::new(input);
    let lines = reader.lines();

    let mut requests: Vec<Request> = Vec::new();
    let mut responses: Vec<Response> = Vec::new();
    let mut times: Vec<i64> = Vec::new();

    // We need to store 1 Request in order to determine the timezone later
    let mut first_request: Option<Request> = None;

    for line in lines {
        let line_value = &line.unwrap();

        let parsed_line = parse_line(line_value);

        match parsed_line {
            Ok(event) => {
                match event {
                    LogEvent::Request(request) => {
                        if first_request.is_none() {
                            first_request = Some(request.clone());
                        }
                        requests.push(request)
                    }
                    LogEvent::Response(response) => responses.push(response),
                }

                let pairs = extract_matching_request_response_pairs(&mut requests, &mut responses);

                let mut new_times: Vec<i64> = pairs.iter()
                    .filter(|pair| filter::matches_filter(&pair, conditions))
                    .map(|pair| pair.response.response_time.num_milliseconds())
                    .collect();

                times.append(&mut new_times);
            }
            Err(err) => println_stderr!("{}", err),
        }
    }
    (times, first_request)
}

#[test]
fn test_extract_times() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let (times, first_request) = extract_times(Box::new(File::open("src/test/simple-1.log")
                                                   .unwrap()),
                                               &conditions);

    assert_eq!(times, vec![7, 10]);
    assert_eq!(first_request.unwrap().id, 1);
}

#[test]
fn test_extract_times_ignore_broken_lines() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let (times, first_request) = extract_times(Box::new(File::open("src/test/broken.log")
                                                   .unwrap()),
                                               &conditions);

    assert_eq!(times, vec![7]);
    assert_eq!(first_request.unwrap().id, 1);
}
<commit_msg>Cleaner Renderer creation<commit_after>use std::io;
use std::io::prelude::*;
use std::net::TcpStream;
use std::fs::File;
use std::env;

extern crate chrono;
use chrono::*;

extern crate stats;
extern crate clap;

mod args;
mod http_status;
mod log_parser;
use log_parser::parse_line;
use log_parser::log_events::*;
mod request_response_matcher;
use request_response_matcher::*;
mod filter;
mod analyzer;
mod render;
use render::*;

// http://stackoverflow.com/a/27590832/376138
macro_rules! println_stderr(
    ($($arg:tt)*) => { {
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
);

fn main() {
    let args = args::parse_args(env::args()).unwrap();

    let input: Box<io::Read> = match args.filename.as_ref() {
        "-" => Box::new(io::stdin()),
        _ => Box::new(File::open(&args.filename).unwrap()),
    };

    let (times, first_request) = extract_times(input, &args.conditions);

    let result = analyzer::analyze(&times);

    let mut stream;
    let mut renderer: Box<Renderer>;

    if args.graphite_server.is_some() {
        stream = TcpStream::connect((args.graphite_server.unwrap().as_ref(),
                                             args.graphite_port.unwrap()))
                .expect("Could not connect to the Graphite server");

        renderer = Box::new(GraphiteRenderer::new(
                    UTC::now().with_timezone(&first_request.unwrap().time.timezone()),
                    args.graphite_prefix,
                    &mut stream,
                ));
    } else {
        renderer = Box::new(TerminalRenderer::new());
    }

    match result {
        Some(result) => {
            renderer.render(result);
        }
        None => println_stderr!("No matching log lines in file."),
    }
}

fn extract_times(input: Box<io::Read>,
                 conditions: &filter::FilterConditions)
                 -> (Vec<i64>, Option<Request>) {
    let reader = io::BufReader::new(input);
    let lines = reader.lines();

    let mut requests: Vec<Request> = Vec::new();
    let mut responses: Vec<Response> = Vec::new();
    let mut times: Vec<i64> = Vec::new();

    // We need to store 1 Request in order to determine the timezone later
    let mut first_request: Option<Request> = None;

    for line in lines {
        let line_value = &line.unwrap();

        let parsed_line = parse_line(line_value);

        match parsed_line {
            Ok(event) => {
                match event {
                    LogEvent::Request(request) => {
                        if first_request.is_none() {
                            first_request = Some(request.clone());
                        }
                        requests.push(request)
                    }
                    LogEvent::Response(response) => responses.push(response),
                }

                let pairs = extract_matching_request_response_pairs(&mut requests, &mut responses);

                let mut new_times: Vec<i64> = pairs.iter()
                    .filter(|pair| filter::matches_filter(&pair, conditions))
                    .map(|pair| pair.response.response_time.num_milliseconds())
                    .collect();

                times.append(&mut new_times);
            }
            Err(err) => println_stderr!("{}", err),
        }
    }
    (times, first_request)
}

#[test]
fn test_extract_times() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let (times, first_request) = extract_times(Box::new(File::open("src/test/simple-1.log")
                                                   .unwrap()),
                                               &conditions);

    assert_eq!(times, vec![7, 10]);
    assert_eq!(first_request.unwrap().id, 1);
}

#[test]
fn test_extract_times_ignore_broken_lines() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let (times, first_request) = extract_times(Box::new(File::open("src/test/broken.log")
                                                   .unwrap()),
                                               &conditions);

    assert_eq!(times, vec![7]);
    assert_eq!(first_request.unwrap().id, 1);
}

{'fraction_non_alphanumeric': 0.1356531660011409, 'fraction_numerical': 0.006389047347404449, 'mean_word_length': 1.9735413839891451, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '445354', 'n_tokens_mistral': 2531, 'n_tokens_neox': 2419, 'n_words': 511}