<commit_before>var redis = require('redis');
var config = require('histograph-config');
var client = redis.createClient(config.redis.port, config.redis.host);
var coreQueue = config.redis.queue;

function add(data) {
  var key;
  if (data.type === 'pit') {
    key = data.data.uri || (data.dataset + '/' + data.data.id);
  } else if (data.type === 'relation') {
    key = data.data.from + ' - ' + data.data.to;
  }

  console.log('Pushing \'' + data.type + '.' + data.action + '\' to queue: \'' + key + '\'');
  client.lpush(coreQueue, JSON.stringify(data));
}

module.exports.add = add;<commit_msg>avoid overflowing queue, added 'done' callback<commit_after>var redis = require('redis');
var config = require('histograph-config');
var client = redis.createClient(config.redis.port, config.redis.host);
var coreQueue = config.redis.queue;

// ensure we don't grow redis queue too much
// will only do_thing when queue < config.redis.maxQueueSize
function is_ready(do_thing)
{
  // try it again in three seconds
  var try_later = function() {
    setTimeout(function(){ is_ready(do_thing); }, 3000)
  };

  // check queue, if too big, wait...
  var qsize = client.llen(coreQueue, function(err, qsize){
    if(qsize > config.redis.maxQueueSize)
      try_later();
    else
      do_thing();
  });
}

// done callback is called when queue is empty enough
function add(data, done) {
  var key;
  if (data.type === 'pit') {
    key = data.data.uri || (data.dataset + '/' + data.data.id);
  } else if (data.type === 'relation') {
    key = data.data.from + ' - ' + data.data.to;
  }

  is_ready(function(){
    console.log('Pushing \'' + data.type + '.' + data.action + '\' to queue: \'' + key + '\'');
    var qsize = client.lpush(coreQueue, JSON.stringify(data), function(err, qsize){
      // report back that we are done
      if(done)
        done(qsize);
    });
  });
}

module.exports.add = add;
{'fraction_non_alphanumeric': 0.17606747496046388, 'fraction_numerical': 0.0021085925144965737, 'mean_word_length': 3.274774774774775, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '706045', 'n_tokens_mistral': 658, 'n_tokens_neox': 646, 'n_words': 153}