<commit_before>"use strict";


/**
 * Normalize all values in an array to be a four-element vector in range [0..1]
 * Substitue a fiven number without any processing instead of null.
 *
 * @param sourceArray -- array of numbers or strings
 * @param nullSub
 * @returns {Array}
 */
function gm$smartNormalizeValues(sourceArray, nullSub) {
    var sorted = sourceArray.slice().sort(),
        uniques = sorted.map(function(e,i){if(e==sorted[i+1]){return null;}else{return e;}}).filter(function(e){return e!=null;}),
        categorical = uniques.some(function(e){return e != null && e.constructor == String});
    if(categorical){
        if (uniques.every(function(s) {return (/^(#[A-Fa-f0-9]{6})$/).test(s);})) {
          return [sourceArray.map(function(e){
               if (e) {
                 var x = parseInt(e.substr(1), 16);
                 return [((x & 0xff0000) >> 16) / 255.0, ((x & 0x00ff00) >> 8) / 255.0, (x & 0x0000ff) / 255.0, 1];
               } else {
                 return [100/255.0, 100/255.0, 100/255.0, 200/255.0];
               }
            }), false];
        } else {
          return [gm$normalizeValues(sourceArray.map(function(e){ return e != null ? uniques.indexOf(e) : null;}), nullSub), true];
        }
    } else {
        return [gm$normalizeValues(sourceArray, nullSub), true];
    }
}

/**
 * Normalize all values in an array to be a four-element vector in range [0..1]
 * Substitue a fiven number without any processing instead of null.
 *
 * @param sourceArray -- array of numbers
 * @param nullSub
 * @returns {Array}
 */<commit_msg>Add js helper for smart conversion from hex to RGBA<commit_after>"use strict";


/**
 * Normalize all values in an array to be a four-element vector in range [0..1]
 * Substitue a fiven number without any processing instead of null.
 *
 * @param sourceArray -- array of numbers or strings
 * @param nullSub
 * @returns {Array}
 */
function gm$smartNormalizeValues(sourceArray, nullSub) {
    var sorted = sourceArray.slice().sort(),
        uniques = sorted.map(function(e,i){if(e==sorted[i+1]){return null;}else{return e;}}).filter(function(e){return e!=null;}),
        categorical = uniques.some(function(e){return e != null && e.constructor == String});
    if(categorical){
        if (uniques.every(function(s) {return (/^(#[A-Fa-f0-9]{6})$/).test(s);})) {
          return [sourceArray.map(function(e){
               if (e) {
                 var x = parseInt(e.substr(1), 16);
                 return [((x & 0xff0000) >> 16) / 255.0, ((x & 0x00ff00) >> 8) / 255.0, (x & 0x0000ff) / 255.0, 1];
               } else {
                 return [100/255.0, 100/255.0, 100/255.0, 200/255.0];
               }
            }), false];
        } else {
          return [gm$normalizeValues(sourceArray.map(function(e){ return e != null ? uniques.indexOf(e) : null;}), nullSub), true];
        }
    } else {
        return [gm$normalizeValues(sourceArray, nullSub), true];
    }
}

function gm$smartCityObjectColor(obj) {
    if(obj.hasOwnProperty('properties') && 
            obj['properties'].hasOwnProperty('viewColor')) {
        var color = obj['properties']['viewColor'];
        if (color.match(/^(#[A-Fa-f0-9]{6})$/)) {
            var x = parseInt(color.substr(1), 16);
            return [((x & 0xff0000) >> 16) / 255.0, 
                    ((x & 0x00ff00) >> 8) / 255.0, 
                    (x & 0x0000ff) / 255.0, 1];
        }
    }
}

/**
 * Normalize all values in an array to be a four-element vector in range [0..1]
 * Substitue a fiven number without any processing instead of null.
 *
 * @param sourceArray -- array of numbers
 * @param nullSub
 * @returns {Array}
 */
{'fraction_non_alphanumeric': 0.16412629286880784, 'fraction_numerical': 0.04817637452367991, 'mean_word_length': 2.4669811320754715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6156188', 'n_tokens_mistral': 1308, 'n_tokens_neox': 1177, 'n_words': 305}