<commit_before>var uuid = require('node-uuid');

var now = function(){
	return (new Date()).getTime();
};

//all.report
var createHeadOfStream = function(report){

};

var reportRunSet = function(set){
	
};

var reportPerfRunResults = function(all){

};

exports.create = function(data){

	var currentSet = {
		report: {}
		data: []
	};

	var all = {
		report: { 
			startTime: now(),
			endTime: null
			received: 1,
			runId: uuid.v4(),
			max: data._metadata.runSize
		},
		dataPoints: []
	};

	//create head of stream

	var generateReport = function(report,dataset){
		var toReturn = report;

		//gather some stats over dataset

		return toReturn;
	};

    var checkEnd = function(data){
    	//this is the end
    	if((report.max && received >= report.max)
    		|| data._metadata.endPerfRun){
    		report.endTime = now();
    		closeCurrentSet();
    		//getAllStats();
    		//saveAllStats();
    		//saveReport();
    	}

    };

    var closeCurrentSet = function(){
    	dataPoints.concat(currentSet.data);	
    	generateReport
    };

    var _perf = function(data){

    	currentSet.push(data);
	    received++;

	    if(received === 1){
	    	openCurrentSet();
	    }

	    if(received % 100 === 0){
	        console.log(received);
	        closeCurrentSet();
	        openCurrentSet();
	    }

	    checkEnd(data);
    };


	return {
		finished: false		
	};	
};<commit_msg>basic perfRun.js in place<commit_after>var uuid = require('node-uuid');

var now = function(){
	return (new Date()).getTime();
};

//THESE ALL SEND STATS TO REPORTING STREAM
//all.report
var createHeadOfStream = function(report){
	
};

var reportRunSet = function(set){

};

var reportPerfRunResults = function(all){

};

//GATHER STATISTICS AND ANALYSIS
var generateReport = function(report,dataset){
	var toReturn = report;

	//gather some stats over dataset

	return toReturn;
};

var statData = function(data){
	var toReturn = data;

	return data;
};

/*
To make reusable we also need
where to save
statData function
generateReport function
*/
exports.create = function(data){

	var obj = {
		finished: false
	};

	var currentSet = {
		report: {},
		data: []
	};

	var all = {
		report: {
			startTime: now(),
			endTime: null,
			received: 1,
			runId: uuid.v4(),
			max: data._metadata.runSize
		},
		sets: [],
		data: []
	};

	createHeadOfStream(all.report);

    var checkEnd = function(data){
    	//this is the end
    	if((report.max && received >= report.max)
    		|| data._metadata.endPerfRun){
    		all.report.endTime = now();
			obj.finished = true;
    		closeCurrentSet();
			all.report = generateReport(all.report,all.data);
			reportPerfRunResults(all);
    	}
    };

    var closeCurrentSet = function(){
		currentSet.report.endTime = now();
    	all.data.concat(currentSet.data);
    	currentSet.report = generateReport(currentSet.report,currentSet.data);
		reportRunSet(currentSet);
    };

	var openCurrentSet = function(){

		currentSet = {
			report: {
				startTime: now(),
				endTime: null,
				setId: uuid.v4()
			},
			data: []
		};

		all.sets.push(currentSet.report.setId);
	};

    var _perf = function(data){

		var toAdd = statData(data);

    	currentSet.push(toAdd);
	    received++;

	    if(received === 1){
	    	openCurrentSet();
	    }

	    if(received % 100 === 0){
	        console.log(received);
	        closeCurrentSet();
	        openCurrentSet();
	    }

	    checkEnd(data);
    };

	return obj;
};
{'fraction_non_alphanumeric': 0.14534883720930233, 'fraction_numerical': 0.00436046511627907, 'mean_word_length': 2.7854785478547854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4352412', 'n_tokens_mistral': 1281, 'n_tokens_neox': 1144, 'n_words': 250}