<commit_before>// reduce() polyfill. This needs to move to a polyfill.js
if (!Array.prototype.reduce) {
  Array.prototype.reduce = function reduce(accumulator){
    if (this===null || this===undefined) throw new TypeError("Object is null or undefined");
    var i = 0, l = this.length >> 0, curr;

    if(typeof accumulator !== "function") // ES5 : "If IsCallable(callbackfn) is false, throw a TypeError exception."
      throw new TypeError("First argument is not callable");

    if(arguments.length < 2) {
      if (l === 0) throw new TypeError("Array length is 0 and no second argument");
      curr = this[0];
      i = 1; // start accumulating at the second element
    }
    else
      curr = arguments[1];

    while (i < l) {
      if(i in this) curr = accumulator.call(undefined, curr, this[i], i, this);
      ++i;
    }

    return curr;
  };
}

function get_object_fields(obj) {
  var field_array = [];
  obj = flatten_json(obj._source)
  for (field in obj) {
    field_array.push(field);
  }
  return field_array.sort();
}

function get_all_fields(json) {
  var field_array = [];
  var obj_fields;
  for (hit in json.hits.hits) {
    obj_fields = get_object_fields(json.hits.hits[hit]);
    for (index in obj_fields) {
      if ($.inArray(obj_fields[index],field_array) < 0) {
        field_array.push(obj_fields[index]);
      }
    }
  }
  return field_array.sort();
}

function has_field(obj,field) {
  var obj_fields = get_object_fields(obj);
  if ($.inArray(field,obj_fields) < 0) {
    return false;
  } else {
    return true;
  }
}

// Retuns a sorted array with duplicates removed
function array_unique(arr) {
  var sorted_arr = arr.sort();
  var results = [];
  for (var i = 0; i < arr.length - 1; i++) {
    if (sorted_arr[i + 1] != sorted_arr[i]) {
        results.push(sorted_arr[i]);
    }
  }
  return results
}

function get_objids_with_field(json,field) {
  var objid_array = [];
  for (hit in json.hits.hits) {
    if(has_field(json.hits.hits[hit],field)) {
      objid_array.push(hit);
    }
  }
  return objid_array;
}

function get_objids_with_field_value(json,field,value) {
  var objid_array = [];
  for (hit in json.hits.hits) {
    var hit_obj = json.hits.hits[hit];
    if(has_field(hit_obj,field)) {
      var field_val = get_field_value(hit_obj,field,'raw')
      if($.isArray(field_val)) {
        if($.inArray(value,field_val) >= 0) {
          objid_array.push(hit);
        }
      } else {
        if(field_val == value) {
          objid_array.push(hit);
        }
      }
    } else {
      if ( value == '')
        objid_array.push(hit);
    }
  }
  return objid_array;
}

function get_related_fields(json,field) {
  var field_array = []
  for (hit in json.hits.hits) {
    var obj_fields = get_object_fields(json.hits.hits[hit])
    //var obj_fields = jQuery.grep(get_object_fields(json.hits.hits[hit]), function(value){
    //  return (value.charAt(0) != '@');
    //});
    if ($.inArray(field,obj_fields) >= 0) {
      field_array.push.apply(field_array,obj_fields);
    }
  }
  var counts = count_values_in_array(field_array);
  return counts;
}

function recurse_field_dots(object,field) {
  var value;
  if (typeof object[field] != 'undefined')
    value = object[field];
  else if (nested = field.match(/(.*?)\.(.*)/))
      value = (typeof object[nested[1]][nested[2]] != 'undefined') ?
        object[nested[1]][nested[2]] : recurse_field_dots(object[nested[1]],nested[2]);
  else
    value = null;

  return value;<commit_msg>check for first level before second<commit_after>// reduce() polyfill. This needs to move to a polyfill.js
if (!Array.prototype.reduce) {
  Array.prototype.reduce = function reduce(accumulator){
    if (this===null || this===undefined) throw new TypeError("Object is null or undefined");
    var i = 0, l = this.length >> 0, curr;

    if(typeof accumulator !== "function") // ES5 : "If IsCallable(callbackfn) is false, throw a TypeError exception."
      throw new TypeError("First argument is not callable");

    if(arguments.length < 2) {
      if (l === 0) throw new TypeError("Array length is 0 and no second argument");
      curr = this[0];
      i = 1; // start accumulating at the second element
    }
    else
      curr = arguments[1];

    while (i < l) {
      if(i in this) curr = accumulator.call(undefined, curr, this[i], i, this);
      ++i;
    }

    return curr;
  };
}

function get_object_fields(obj) {
  var field_array = [];
  obj = flatten_json(obj._source)
  for (field in obj) {
    field_array.push(field);
  }
  return field_array.sort();
}

function get_all_fields(json) {
  var field_array = [];
  var obj_fields;
  for (hit in json.hits.hits) {
    obj_fields = get_object_fields(json.hits.hits[hit]);
    for (index in obj_fields) {
      if ($.inArray(obj_fields[index],field_array) < 0) {
        field_array.push(obj_fields[index]);
      }
    }
  }
  return field_array.sort();
}

function has_field(obj,field) {
  var obj_fields = get_object_fields(obj);
  if ($.inArray(field,obj_fields) < 0) {
    return false;
  } else {
    return true;
  }
}

// Retuns a sorted array with duplicates removed
function array_unique(arr) {
  var sorted_arr = arr.sort();
  var results = [];
  for (var i = 0; i < arr.length - 1; i++) {
    if (sorted_arr[i + 1] != sorted_arr[i]) {
        results.push(sorted_arr[i]);
    }
  }
  return results
}

function get_objids_with_field(json,field) {
  var objid_array = [];
  for (hit in json.hits.hits) {
    if(has_field(json.hits.hits[hit],field)) {
      objid_array.push(hit);
    }
  }
  return objid_array;
}

function get_objids_with_field_value(json,field,value) {
  var objid_array = [];
  for (hit in json.hits.hits) {
    var hit_obj = json.hits.hits[hit];
    if(has_field(hit_obj,field)) {
      var field_val = get_field_value(hit_obj,field,'raw')
      if($.isArray(field_val)) {
        if($.inArray(value,field_val) >= 0) {
          objid_array.push(hit);
        }
      } else {
        if(field_val == value) {
          objid_array.push(hit);
        }
      }
    } else {
      if ( value == '')
        objid_array.push(hit);
    }
  }
  return objid_array;
}

function get_related_fields(json,field) {
  var field_array = []
  for (hit in json.hits.hits) {
    var obj_fields = get_object_fields(json.hits.hits[hit])
    //var obj_fields = jQuery.grep(get_object_fields(json.hits.hits[hit]), function(value){
    //  return (value.charAt(0) != '@');
    //});
    if ($.inArray(field,obj_fields) >= 0) {
      field_array.push.apply(field_array,obj_fields);
    }
  }
  var counts = count_values_in_array(field_array);
  return counts;
}

function recurse_field_dots(object,field) {
  var value = null;
  if (typeof object[field] != 'undefined')
    value = object[field];
  else if (nested = field.match(/(.*?)\.(.*)/))
    if(typeof object[nested[1]] != 'undefined')
      value = (typeof object[nested[1]][nested[2]] != 'undefined') ?
        object[nested[1]][nested[2]] : recurse_field_dots(object[nested[1]],nested[2]);

  return value;
{'fraction_non_alphanumeric': 0.15011383039271486, 'fraction_numerical': 0.006687535571997723, 'mean_word_length': 3.081881533101045, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3480026', 'n_tokens_mistral': 2522, 'n_tokens_neox': 2484, 'n_words': 531}