<commit_before>var argv = require('minimist')(process.argv.slice(2));
var log = require('../utils/log');

// create a promise for each message on redis queue
var Redis = require('redis');
var redis_client = Redis.createClient();
var queueName = argv.q || argv.queue || 'histograph-queue';

var loopRedis = function loopRedis(mkPromise)
{
	redis_client.blpop(queueName, 0, function(err,data) {
		var d = JSON.parse(data[1]);
		return mkPromise(d)
			.then(function(){
				// loop
				process.nextTick(loopRedis.bind(null, mkPromise));
			}, function(err){
				console.error(err);
			});
	});
}

// when passed an object, every fail that contains an object
// is converted into a JSON-string<commit_msg>hacked some logging that shows performance<commit_after>var argv = require('minimist')(process.argv.slice(2));
var log = require('../utils/log');

// create a promise for each message on redis queue
var Redis = require('redis');
var redis_client = Redis.createClient();
var queueName = argv.q || argv.queue || 'histograph-queue';

var i = 0;

var time = function(){
	var t = process.hrtime();
	var ns = (t[0] * 1e9 + t[1]);
	return ns / 1e9;
}

var t0 = time()

var loopRedis = function loopRedis(mkPromise)
{
	redis_client.blpop(queueName, 0, function(err,data) {
		var d = JSON.parse(data[1]);
		return mkPromise(d)
			.then(function(response){
				i += 1;
				try
				{
					// output
					var r = response.result;
					var per_sec = (argv.batchSize / (r.duration_ms / 1000)).toFixed(2);
					var errs = (r.errors && r.errors.length) || 0;
					var elapsed = time() - t0;
					var agg_rate = (i/elapsed).toFixed(2);
					console.log("Rate / Errors / Count / Agg. Rate:\t", per_sec,'\t',errs,'\t',i, '\t', agg_rate);
				} catch (e) {
				}

				// loop
				process.nextTick(loopRedis.bind(null, mkPromise));
			}, function(err){
				console.error(err);
			});
	});
}

// when passed an object, every fail that contains an object
// is converted into a JSON-string
{'fraction_non_alphanumeric': 0.1626842907981698, 'fraction_numerical': 0.011692933401118455, 'mean_word_length': 3.4829157175398633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1916131', 'n_tokens_mistral': 771, 'n_tokens_neox': 678, 'n_words': 176}