<commit_before>NANO.plugin(function(NANO){
  var default_params = {
    complete: function(){},
    after: function(){},
    iterator: function(data){return data;},
    delay: 1
  };

  var async_iterator = function(state, iterator, complete){
    var hold = false;
    var keys = _.keys(state.data);
    iterator = _.bind(iterator, state);
    
    var async_iteration = function(){
      if(hold){
        return;
      }
      var next_index = keys.shift();
      if(!_.isUndefined(next_index)){
        state.results[next_index] = iterator(state.data[next_index], next_index);
        setTimeout(async_iteration, state.delay || 0);
      }else{ 
        complete(state);
        state.is_complete = true;
      }
    };
    return {
      state: state, 
      start: function(){
        hold = false;
        setTimeout(async_iteration, state.delay || 0);
      },
      stop: function(){
        hold = true;
      }
    };
  };


  var async_worker = function(params){
    params = _.extend({}, default_params, params);
    var state = {
      is_complete: false
    };

    var get_working_state = function(data){
      return {
        data: data,
        results: _.isArray(data) ? [] : {},
        delay: params.delay,
        is_complete: false
      };
    };<commit_msg>appended bathching to async worker<commit_after>NANO.plugin(function(NANO){
  var default_params = {
    complete: function(){},
    after: function(){},
    iterator: function(data){return data;},
    delay: 1,
    batch_size: 20
  };

  var async_iterator = function(state, iterator, complete){
    var hold = false;
    var keys = _.keys(state.data);
    iterator = _.bind(iterator, state);
    var current_batch = 0;
    
    var async_iteration = function(){
      if(hold){
        return;
      }
      var next_index = keys.shift();
      if(!_.isUndefined(next_index)){
        state.results[next_index] = iterator(state.data[next_index], next_index);
        if(0 < current_batch){
          current_batch --;
          async_iteration();
        }else {
          current_batch = state.batch_size;
          setTimeout(async_iteration, state.delay || 0);
        }
      }else{ 
        complete(state);
        state.is_complete = true;
      }
    };
    return {
      state: state, 
      start: function(){
        hold = false;
        setTimeout(async_iteration, state.delay || 0);
      },
      stop: function(){
        hold = true;
      }
    };
  };


  var async_worker = function(params){
    params = _.extend({}, default_params, params);
    var state = {
      is_complete: false
    };

    var get_working_state = function(data){
      return {
        data: data,
        results: _.isArray(data) ? [] : {},
        delay: params.delay,
        batch_size: params.batch_size,
        is_complete: false
      };
    };
{'fraction_non_alphanumeric': 0.13166254853512177, 'fraction_numerical': 0.0035298270384751147, 'mean_word_length': 2.173572228443449, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7479408', 'n_tokens_mistral': 865, 'n_tokens_neox': 845, 'n_words': 182}