<commit_before>var d3 = require('d3');
var moment = require('moment');
var to_offset = require('./dateoffset.js').to_offset;
var date_range = require('./dateindex.js').date_range;

function time_bins(index, freq) {
  /*
   * 
   */
  var offset = to_offset(freq);
  var start = moment.utc(index[0]);
  var end = moment.utc(index[index.length-1]);
  var edge_start = offset.rollback(start);
  var edge_end = offset.rollforward(end);
  var binner = date_range(edge_start, edge_end, freq);
  var binvals = binner.values;

  var bins = [];
  var bin_index = 0;
  var current_edge = binvals[bin_index];
  for (var i=0; i < index.length; i++) {
    if (index[i] >= current_edge) {
      bins.push(i);
      bin_index++;
      current_edge = binvals[bin_index];
    }<commit_msg>added optinal start/end index so we don't have to slice and create copies of large arrays<commit_after>var d3 = require('d3');
var moment = require('moment');
var to_offset = require('./dateoffset.js').to_offset;
var date_range = require('./dateindex.js').date_range;

function time_bins(index, freq, config) {
  /*
   * 
   */
  if (!config) { config = {} }
  var start = 'undefined' === typeof(config.start) ? 0 : config.start;
  var end = 'undefined' === typeof(config.end) ? index.length-1 : config.end;

  var offset = to_offset(freq);
  var start_date = moment.utc(index[start]);
  var end_date = moment.utc(index[end]);
  var edge_start = offset.rollback(start_date);
  var edge_end = offset.rollforward(end_date);
  var binner = date_range(edge_start, edge_end, freq);
  var binvals = binner.values;

  var bins = [];
  var bin_index = 0;
  var current_edge = binvals[bin_index];
  for (var i=start; i <= end; i++) {
    if (index[i] >= current_edge) {
      bins.push(i);
      bin_index++;
      current_edge = binvals[bin_index];
    }
{'fraction_non_alphanumeric': 0.15456545654565457, 'fraction_numerical': 0.00605060506050605, 'mean_word_length': 3.8249336870026527, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779166', 'n_tokens_mistral': 654, 'n_tokens_neox': 637, 'n_words': 149}