<commit_before>/* Convenient functions for handling arrays */
var Arr = {};

/** @param {!Array} source
    @param {!Array} toRemove
    @return {!Array}
*/
Arr.difference = function(source, toRemove) {
    source = $.makeArray(source); toRemove = $.makeArray(toRemove);
    var result = [];
    $.each(source, function(_,val){
        if (!Arr.contains(toRemove,val))
            result.push(val);
    })
    return result;
}

/** @param {!Array} a
    @param {!Array} b
    @return {!Array}
*/
Arr.union = function(a, b) {
    a = $.makeArray(a); b = $.makeArray(b);
    var result = [];
    $.each(a, function(_,val) {
        if (Arr.contains(b,val))
            result.push(val);
    });
    return result;
}

/** @param {!Array} a
    @param {!number} i
*/
Arr.removeAt = function(a, i) {
    assert(i >= 0, "Arr.removeAt i>=0");
    assert(i <= a.length, "Arr.removeAt i<=a.length");
    return a.slice(0,i).concat(a.slice(i+1));
}

/** Is value in array?
  * @param {!Array} array
  * @param {*} value
  * @return {!boolean}
  */
Arr.contains = function(array, value) {
    return $.inArray(value, array) !== -1;
}

/* Returns a copy of the array with those elements of that
 * don't satisfy the predicate filtered out
 * 
 * @param {!Array} array
 * @param {!function(*):boolean} predicate
 * @return {!Array}
 */
Arr.filter = function(array, predicate) {
    return $.grep(array, predicate);
}

/** Returns two new arrays, the first with those elements that satisfy the 
  * predicate, the second with those that don't 
  * 
  * @param {!Array} array
  * @param {!function(*):boolean} predicate
  * @return {!Array.<!Array>}
*/
Arr.partition = function(array, predicate) {
    var good = [];
    var bad = [];
    $.each(array, function(i, val) {
        if (predicate(val))
            good.push(val);
        else
            bad.push(val);
    });
    return [good,bad];
}

/**
  * @param {!Array} array
  * @param {!function(*):boolean=} predicate
  * @return {!boolean}
  */
Arr.all = function(array, predicate) {
    if (!predicate) predicate = identity;
    for (var i = 0; i < array.length; i++)
        if (!predicate(array[i]))
            return false;
    return true;
}

/**
  * @param {!Array} array
  * @param {!function(*):boolean=} predicate
  * @return {!boolean}
  */
Arr.any = function(array, predicate) {
    if (!predicate) predicate = identity;
    for (var i = 0; i < array.length; i++)
        if (predicate(array[i]))
            return true;
    return false;
}

/**
  * @param {!Array} array
  * @param {!function(*):boolean=} predicate
  * @return {!boolean}
  */<commit_msg>Weaken type signature to mollify type checker<commit_after>/* Convenient functions for handling arrays */
var Arr = {};

/** @param {!Array} source
    @param {!Array} toRemove
    @return {!Array}
*/
Arr.difference = function(source, toRemove) {
    source = $.makeArray(source); toRemove = $.makeArray(toRemove);
    var result = [];
    $.each(source, function(_,val){
        if (!Arr.contains(toRemove,val))
            result.push(val);
    })
    return result;
}

/** @param {!Array} a
    @param {!Array} b
    @return {!Array}
*/
Arr.union = function(a, b) {
    a = $.makeArray(a); b = $.makeArray(b);
    var result = [];
    $.each(a, function(_,val) {
        if (Arr.contains(b,val))
            result.push(val);
    });
    return result;
}

/** @param {!Array} a
    @param {!number} i
*/
Arr.removeAt = function(a, i) {
    assert(i >= 0, "Arr.removeAt i>=0");
    assert(i <= a.length, "Arr.removeAt i<=a.length");
    return a.slice(0,i).concat(a.slice(i+1));
}

/** Is value in array?
  * @param {!Array} array
  * @param {*} value
  * @return {!boolean}
  */
Arr.contains = function(array, value) {
    return $.inArray(value, array) !== -1;
}

/* Returns a copy of the array with those elements of that
 * don't satisfy the predicate filtered out
 * 
 * @param {!Array} array
 * @param {!function(*):boolean} predicate
 * @return {!Array}
 */
Arr.filter = function(array, predicate) {
    return $.grep(array, predicate);
}

/** Returns two new arrays, the first with those elements that satisfy the 
  * predicate, the second with those that don't 
  * 
  * @param {!Array} array
  * @param {!function(*):boolean} predicate
  * @return {!Array.<!Array>}
*/
Arr.partition = function(array, predicate) {
    var good = [];
    var bad = [];
    $.each(array, function(i, val) {
        if (predicate(val))
            good.push(val);
        else
            bad.push(val);
    });
    return [good,bad];
}

/**
  * @param {!Array} array
  * @param {!function(*):boolean=} predicate
  * @return {!boolean}
  */
Arr.all = function(array, predicate) {
    if (!predicate) predicate = identity;
    for (var i = 0; i < array.length; i++)
        if (!predicate(array[i]))
            return false;
    return true;
}

/**
  * @param {!Array} array
  * @param {function()=} predicate
  * @return {!boolean}
  */
Arr.any = function(array, predicate) {
    if (!predicate) predicate = identity;
    for (var i = 0; i < array.length; i++)
        if (predicate(array[i]))
            return true;
    return false;
}

/**
  * @param {!Array} array
  * @param {!function(*):boolean=} predicate
  * @return {!boolean}
  */
{'fraction_non_alphanumeric': 0.18114285714285713, 'fraction_numerical': 0.0026666666666666666, 'mean_word_length': 2.818909090909091, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7274715', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1760, 'n_words': 470}