<commit_before>extern crate timely;
extern crate timely_sort;
extern crate differential_dataflow;

use timely::dataflow::*;
use timely::dataflow::operators::*;

use differential_dataflow::ring::RingPair;
use differential_dataflow::AsCollection;
use differential_dataflow::operators::Consolidate;

fn main() {

    let keys: usize = std::env::args().nth(1).unwrap().parse().unwrap();
    let batch: usize = std::env::args().nth(2).unwrap().parse().unwrap();

    // This computation demonstrates in-place accumulation of arbitrarily large 
    // volumes of input data, so long as the number of distinct keys is moderate.
    // 
    // Specifically, the program aims to accumulate (key, val) pairs where we are
    // interested in the sum of values for each key, as well as the count of the
    // number of records, so that we can produce the average.
    timely::execute_from_args(std::env::args().skip(4), move |computation| {

        let mut input = computation.scoped::<(), _, _>(|scope| {

            let (input, data) = scope.new_input::<((usize, isize), isize)>();

            // move `val` into the ring component.
            data.map(|((x,y),d)| (x, Default::default(), RingPair::new(y,d)))
				.as_collection()
                .consolidate();

            input
        });

        let timer = ::std::time::Instant::now();

        for val in 0 .. {
        	// introduce some data with bounded key.
        	input.send(((val % keys, (val as isize) % 100), 1 as isize));

        	// we must still give the computation the opportunity to act.
        	if val > 0 && val % batch == 0 { 
        		computation.step(); 
        		let elapsed = timer.elapsed();
        		let secs = elapsed.as_secs() as f64 + (elapsed.subsec_nanos() as f64)/1000000000.0;
        		println!("tuples: {:?},\telts/sec: {:?}", val, val as f64 / secs);
        	}
        }

    }).unwrap();
}<commit_msg>actually save edits this time<commit_after>extern crate timely;
extern crate timely_sort;
extern crate differential_dataflow;

use timely::dataflow::*;
use timely::dataflow::operators::*;

use differential_dataflow::difference::DiffPair;
use differential_dataflow::AsCollection;
use differential_dataflow::operators::Consolidate;

fn main() {

    let keys: usize = std::env::args().nth(1).unwrap().parse().unwrap();
    let batch: usize = std::env::args().nth(2).unwrap().parse().unwrap();

    // This computation demonstrates in-place accumulation of arbitrarily large 
    // volumes of input data, so long as the number of distinct keys is moderate.
    // 
    // Specifically, the program aims to accumulate (key, val) pairs where we are
    // interested in the sum of values for each key, as well as the count of the
    // number of records, so that we can produce the average.
    timely::execute_from_args(std::env::args().skip(4), move |computation| {

        let mut input = computation.scoped::<(), _, _>(|scope| {

            let (input, data) = scope.new_input::<((usize, isize), isize)>();

            // move `val` into the ring component.
            data.map(|((x,y),d)| (x, Default::default(), DiffPair::new(y,d)))
				.as_collection()
                .consolidate();

            input
        });

        let timer = ::std::time::Instant::now();

        for val in 0 .. {
        	// introduce some data with bounded key.
        	input.send(((val % keys, (val as isize) % 100), 1 as isize));

        	// we must still give the computation the opportunity to act.
        	if val > 0 && val % batch == 0 { 
        		computation.step(); 
        		let elapsed = timer.elapsed();
        		let secs = elapsed.as_secs() as f64 + (elapsed.subsec_nanos() as f64)/1000000000.0;
        		println!("tuples: {:?},\telts/sec: {:?}", val, val as f64 / secs);
        	}
        }

    }).unwrap();
}
{'fraction_non_alphanumeric': 0.14772132006286012, 'fraction_numerical': 0.014143530644316397, 'mean_word_length': 2.7258536585365856, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4973699', 'n_tokens_mistral': 1217, 'n_tokens_neox': 1124, 'n_words': 352}