<commit_before>extern crate futures;
extern crate futures_cpupool;
extern crate time;
extern crate wikidata;

use std::sync::mpsc::channel;

use futures::Future;
use futures_cpupool::CpuPool;
use time::precise_time_ns;

use wikidata::parse::parse_item;
use wikidata::read::Streamer;
use wikidata::param::{Parameters,read_params};

fn main() {
    let t0 = precise_time_ns();

    let Parameters{input_file, languages} = read_params();
    println!("Input file: {}", input_file);
    println!("Languages: {:?}", languages);

    let (tx, rx) = channel();
    if let Ok(streamer) = Streamer::new(&input_file) {
        let t00 = precise_time_ns();
        let pool = CpuPool::new(4);
        let mut futs = Vec::new();
        for line in streamer {
            let l = line.clone();
            let langs = languages.clone();
            let tx = tx.clone();
            let t = pool.spawn_fn(move || {
                parse_item(&l, &langs)
                    .ok()
                    .and_then(|l| l)
                    .map(|e| {
                        let _ = tx.send(e);
                    });
                Ok::<(), ()>(())
            });
            futs.push(t);
        }
        for f in futs {
            let _ = f.wait();
        }
        let t10 = precise_time_ns();
        println!("Main loop: {} us", (t10 - t00) / 1000 );
    }

    let t000 = precise_time_ns();
    let mut same = 0;
    let mut different = 0;
    drop(tx); // Close the original sender, since we won't be sending anything from it
    for e in rx {
        if e[&languages[0]] == e[&languages[1]] {
            same += 1
        } else {
            different += 1
        }
    }
    let t100 = precise_time_ns();
    println!("Counting: {} us", (t100 - t000) / 1000 );

    let t1 = precise_time_ns();
    println!("Results - Same: {}, Different: {}", same, different);
    println!("Time: {} us", (t1 - t0) / 1000 )
}
<commit_msg>Can wait on the futures after we're done counting results<commit_after>extern crate futures;
extern crate futures_cpupool;
extern crate time;
extern crate wikidata;

use std::sync::mpsc::channel;

use futures::Future;
use futures_cpupool::CpuPool;
use time::precise_time_ns;

use wikidata::parse::parse_item;
use wikidata::read::Streamer;
use wikidata::param::{Parameters,read_params};

fn main() {
    let t0 = precise_time_ns();

    let Parameters{input_file, languages} = read_params();
    println!("Input file: {}", input_file);
    println!("Languages: {:?}", languages);

    let pool = CpuPool::new(4);
    let mut futs = Vec::new();
    let (tx, rx) = channel();
    if let Ok(streamer) = Streamer::new(&input_file) {
        let t00 = precise_time_ns();
        for line in streamer {
            let l = line.clone();
            let langs = languages.clone();
            let tx = tx.clone();
            let t = pool.spawn_fn(move || {
                parse_item(&l, &langs)
                    .ok()
                    .and_then(|l| l)
                    .map(|e| {
                        let _ = tx.send(e);
                    });
                Ok::<(), ()>(())
            });
            futs.push(t);
        }
        let t10 = precise_time_ns();
        println!("Main loop: {} us", (t10 - t00) / 1000 );
    }

    let t000 = precise_time_ns();
    let mut same = 0;
    let mut different = 0;
    drop(tx); // Close the original sender, since we won't be sending anything from it
    for e in rx {
        if e[&languages[0]] == e[&languages[1]] {
            same += 1
        } else {
            different += 1
        }
    }
    for f in futs {
        let _ = f.wait();
    }
    let t100 = precise_time_ns();
    println!("Counting: {} us", (t100 - t000) / 1000 );

    let t1 = precise_time_ns();
    println!("Results - Same: {}, Different: {}", same, different);
    println!("Time: {} us", (t1 - t0) / 1000 )
}

{'fraction_non_alphanumeric': 0.15136668385765858, 'fraction_numerical': 0.0221763795771016, 'mean_word_length': 1.9746932515337423, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3979918', 'n_tokens_mistral': 1327, 'n_tokens_neox': 1234, 'n_words': 321}