<commit_before>extern crate protobuf;
extern crate rand;
extern crate time;

use std::io::File;
use std::io::MemWriter;
use std::io::BufReader;
use std::default::Default;

use rand::Rng;
use rand::StdRng;
use rand::SeedableRng;

use protobuf::Message;
use perftest_data::PerftestData;

mod perftest_data;

fn measure_ns<R>(f: || -> R) -> (u64, R) {
    let start = time::precise_time_ns();
    let r = f();
    (time::precise_time_ns() - start, r)
}

fn measure_and_print<R>(title: &str, iter: u64, f: || -> R) -> R {
    let (ns, r) = measure_ns(f);
    let ns_per_iter = ns / iter;
    println!("{}: {} ns per iter", title, ns_per_iter);
    r
}

fn test<M : Message>(name: &str, data: &[M]) {
    let mut rng: StdRng = SeedableRng::from_seed(&[10, 20, 30, 40]);
    let mut random_data: Vec<M> = Vec::new();

    let mut total_size = 0;
    while total_size < 1000000 {
        let ref item = data[rng.gen_range(0, data.len())];
        random_data.push(item.clone());
        total_size += item.serialized_size();
    }

    let mut writer = MemWriter::new();
    measure_and_print(format!("{}: write", name), random_data.len() as u64, || {
        for m in random_data.iter() {
            m.write_length_delimited_to_writer(&mut writer);
        }
    });

    let buf = writer.unwrap();

    let read_data = measure_and_print(format!("{}: read", name), random_data.len() as u64, || {
        let mut r = Vec::new();
        let mut reader = BufReader::new(buf.as_slice());
        let mut coded_input_stream = protobuf::CodedInputStream::new(&mut reader);
        while !coded_input_stream.eof() {
            r.push(protobuf::parse_length_delimited_from::<M>(&mut coded_input_stream));
        }
        r
    });

    assert_eq!(random_data, read_data);

    let merged = measure_and_print(format!("{}: read reuse", name), random_data.len() as u64, || {
        let mut reader = BufReader::new(buf.as_slice());
        let mut coded_input_stream = protobuf::CodedInputStream::new(&mut reader);
        let mut msg: M = Default::default();
        let mut count = 0;
        while !coded_input_stream.eof() {
            msg.clear();
            coded_input_stream.merge_message(&mut msg);
            count += 1;
        }
        count
    });

    assert_eq!(random_data.len(), merged);
}

fn main() {
    let mut is = File::open(&Path::new("perftest_data.pbbin"));
    let perftest_data = protobuf::parse_from_reader::<PerftestData>(&mut is);
    test("test1", perftest_data.get_test1());
    test("test_repeated_bool", perftest_data.get_test_repeated_bool());
    test("test_repeated_messages", perftest_data.get_test_repeated_messages());
    test("test_optional_messages", perftest_data.get_test_optional_messages());
    test("test_strings", perftest_data.get_test_strings());
}<commit_msg>Option to specify which perftest to run<commit_after>extern crate protobuf;
extern crate rand;
extern crate time;

use std::io::File;
use std::io::MemWriter;
use std::io::BufReader;
use std::default::Default;
use std::os;

use rand::Rng;
use rand::StdRng;
use rand::SeedableRng;

use protobuf::Message;
use perftest_data::PerftestData;

mod perftest_data;

fn measure_ns<R>(f: || -> R) -> (u64, R) {
    let start = time::precise_time_ns();
    let r = f();
    (time::precise_time_ns() - start, r)
}

fn measure_and_print<R>(title: &str, iter: u64, f: || -> R) -> R {
    let (ns, r) = measure_ns(f);
    let ns_per_iter = ns / iter;
    println!("{}: {} ns per iter", title, ns_per_iter);
    r
}

fn run_test<M : Message>(name: &str, data: &[M]) {
    let mut rng: StdRng = SeedableRng::from_seed(&[10, 20, 30, 40]);
    let mut random_data: Vec<M> = Vec::new();

    let mut total_size = 0;
    while total_size < 1000000 {
        let ref item = data[rng.gen_range(0, data.len())];
        random_data.push(item.clone());
        total_size += item.serialized_size();
    }

    let mut writer = MemWriter::new();
    measure_and_print(format!("{}: write", name), random_data.len() as u64, || {
        for m in random_data.iter() {
            m.write_length_delimited_to_writer(&mut writer);
        }
    });

    let buf = writer.unwrap();

    let read_data = measure_and_print(format!("{}: read", name), random_data.len() as u64, || {
        let mut r = Vec::new();
        let mut reader = BufReader::new(buf.as_slice());
        let mut coded_input_stream = protobuf::CodedInputStream::new(&mut reader);
        while !coded_input_stream.eof() {
            r.push(protobuf::parse_length_delimited_from::<M>(&mut coded_input_stream));
        }
        r
    });

    assert_eq!(random_data, read_data);

    let merged = measure_and_print(format!("{}: read reuse", name), random_data.len() as u64, || {
        let mut reader = BufReader::new(buf.as_slice());
        let mut coded_input_stream = protobuf::CodedInputStream::new(&mut reader);
        let mut msg: M = Default::default();
        let mut count = 0;
        while !coded_input_stream.eof() {
            msg.clear();
            coded_input_stream.merge_message(&mut msg);
            count += 1;
        }
        count
    });

    assert_eq!(random_data.len(), merged);
}

struct TestRunner {
    selected: Option<~str>,
    any_matched: bool,
}

impl TestRunner {
    fn test<M : Message>(&mut self, name: &str, data: &[M]) {
        if self.selected.is_none() || name == *self.selected.get_ref() {
            run_test(name, data);
            self.any_matched = true;
        }
    }

    fn check(&self) {
        if !self.any_matched {
            let name = self.selected.as_ref().map(|s| s.as_slice()).unwrap_or("bug");
            fail!("no tests found with name {}", name);
        }
    }
}

fn main() {
    let selected = match os::args().as_slice() {
        [_] => None,
        [_, ref test] => Some(test.to_owned()),
        [ref argv0, ..] => fail!("usage: {} <test>", argv0),
        _ => fail!()
    };

    let mut runner = TestRunner { selected: selected, any_matched: false };

    let mut is = File::open(&Path::new("perftest_data.pbbin"));
    let test_data = protobuf::parse_from_reader::<PerftestData>(&mut is);

    runner.test("test1", test_data.get_test1());
    runner.test("test_repeated_bool", test_data.get_test_repeated_bool());
    runner.test("test_repeated_messages", test_data.get_test_repeated_messages());
    runner.test("test_optional_messages", test_data.get_test_optional_messages());
    runner.test("test_strings", test_data.get_test_strings());
    runner.check();
}
{'fraction_non_alphanumeric': 0.16028303337948008, 'fraction_numerical': 0.009844639286263652, 'mean_word_length': 3.2275682704811444, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1747015', 'n_tokens_mistral': 2289, 'n_tokens_neox': 2210, 'n_words': 477}