<commit_before>// layer.js
// Most of the code to handle the layer data.

var app = app || {};
(function (hex) { // jshint ignore: line
Layer = (function () { // jshint ignore: line

    var selection_prefix = 'Selection';
    
    function metadata_counts_check (where, layer_name, n, positives) {
        var hexCount = Object.keys(polygons).length;
        if (n > hexCount && hexCount > 0) {
            console.log('TODO: bad counts at', where, ': layer, n, hexCount:',
                layer_name, layers[layer_name].n, hexCount);
            if (!_.isUndefined(positives)) {
                console.log('    positives:', positives);
            }
        }
    }
    
    function make_layer_name_unique (layer_name) {
 
        // We're done if the name is unique
        if (layers[layer_name] === undefined) { return layer_name; }

        var last_suffix,
            name = layer_name,
            seq = 1;
 
        // Special case a default selection layer name
        if (name.startsWith(selection_prefix)) {
            name = selection_prefix;
        }
 
        // Keep looking for a name until it is unique
        while (true) {
 
            // We're done if the name is unique
            if (layers[name] === undefined) { break; }

            // Find any previously tried sequence suffix
            if (seq > 1) {
                last_suffix = ' ' + (seq - 1);
                if (name.endsWith(last_suffix)) {
 
                    // Remove the existing sequence suffix
                    name = name.slice(0, name.length - last_suffix.length);
                }
            }
            name += ' ' + seq;
            seq += 1;
        }
        return name;
    }
 
    function ask_user_to_name_layer(layer_name, dup_name) {
 
        // Give the user a chance to name the layer
        var promptString = (dup_name) ?
                            dup_name + ' is in use, how about this one?'
                            : 'Please provide a label for this new layer',
            text = prompt(promptString, layer_name);
        if (text) {
            return text.trim();
        } else {
            return undefined;
        }
    }
 
    function let_user_name_layer(layer_name) {
 
        var name = layer_name;
 
        // if no layer name was supplied, assume this is a selection
        if (!name) {
            name = selection_prefix;
        } else {
            name = name.trim();
        }
 
        // Start with a unique name as a suggestion
        name = make_layer_name_unique(name);
 
        var unique_name,
            dup_name;
 
        // Keep asking the user for a name until it is unique or she cancels
        while (true) {
            name = ask_user_to_name_layer(name, dup_name);
            
            // We're done if she cancels
            if (name === undefined) { break; }
 
            // We're done if the name is unique
            unique_name = make_layer_name_unique(name);
            if (unique_name === name) { break; }

            // Suggest another unique name
            dup_name = name;
            name = unique_name;
        }
        return name;
    }
 
    function load_dynamic_colormap (name, layer, cats) {

        // Load the colormap for dynamic categorical or binary attributes.
        if ('colormap' in layer) {
        
            // Load the supplied colormap
            colormaps[name] = _.map(layer.colormap, function (vals, i) {
                return {
                    name: vals[0],
                    color: new Color(vals[1]),
                    fileColor: new Color(vals[1]),
                }
            });

            if (cats) {
         
                // Replace category string values with indices
                layer.data = _.object(
                    _.keys(layer.data),
                    _.map(layer.data, function (strVal, key) {
                        return cats.indexOf(strVal);
                    })
                );
            }
        } else if (cats && cats.length) {
        
            // Generate a colormap
            var jpColormap = _.map(
                jPalette.ColorMap.get('hexmap')(cats.length + 1).map,
                function (val, key) {
            
                    // Ignore alpha, taking the default of one.
                    return {r: val.r, g: val.g, b: val.b};
                }
            );
         
            // Remove the repeating red at the end
            jpColormap.splice(cats.length, 1);
         
            // Load this generated colormap
            colormaps[name] = _.map(jpColormap, function(color, i) {
                return {
                    name: cats[i],
                    color: Color(color), // operating color in map
                    fileColor: Color(color), // color from orig file
                };
            });
         
            // Replace category string values with indices
            layer.data = _.object(
                _.keys(layer.data),
                _.map(layer.data, function (strVal, key) {
                    return cats.indexOf(strVal);
                })
            );

        } else {
        
            // The default binary colormap for non-string values
            colormaps[name] = [];
        }
        
        // Remove these that are no longer needed.
        delete layer.colormap;
    }

    function load_dynamic_data_type (name, layer) {
    
        // Load the data type for dynamic attributes.
        var uniqueVals,
            hasStrings;

        // Skip any layers with no values.
        
        if (_.keys(layer.data).length < 1) { return; }

        // Drop any values used to indicate no value.
        var drop = ['', '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN', 
            '-NAN', '1.#IND', '1.#QNAN', 'N/A', 'NA', 'NULL', 'NAN'];
        _.each(layer.data, function (val, key) {
            if (drop.indexOf(val.toString().toUpperCase()) > -1) {
                delete layer.data[key];
            }
        });
 
        if (!('dataType' in layer)) {
        
            // Determine the data type since it was not supplied.
         
            // If they are any strings, this gets a colormap
            // and call it categorical for now. It may be binary.
            var aString = _.find(layer.data, function (value) {
                    return _.isNaN(parseFloat(value));
                });
            if (aString && aString.length > 0) {
                layer.dataType = 'categorical';
                hasStrings = true;
            }
         
            // Find the count of each unique value.
            uniqueVals = _.countBy(layer.data, function (value) {
                return value;
            });<commit_msg>handle non values in dynamic attributes<commit_after>// layer.js
// Most of the code to handle the layer data.

var app = app || {};
(function (hex) { // jshint ignore: line
Layer = (function () { // jshint ignore: line

    var selection_prefix = 'Selection';
    
    function metadata_counts_check (where, layer_name, n, positives) {
        var hexCount = Object.keys(polygons).length;
        if (n > hexCount && hexCount > 0) {
            console.log('TODO: bad counts at', where, ': layer, n, hexCount:',
                layer_name, layers[layer_name].n, hexCount);
            if (!_.isUndefined(positives)) {
                console.log('    positives:', positives);
            }
        }
    }
    
    function make_layer_name_unique (layer_name) {
 
        // We're done if the name is unique
        if (layers[layer_name] === undefined) { return layer_name; }

        var last_suffix,
            name = layer_name,
            seq = 1;
 
        // Special case a default selection layer name
        if (name.startsWith(selection_prefix)) {
            name = selection_prefix;
        }
 
        // Keep looking for a name until it is unique
        while (true) {
 
            // We're done if the name is unique
            if (layers[name] === undefined) { break; }

            // Find any previously tried sequence suffix
            if (seq > 1) {
                last_suffix = ' ' + (seq - 1);
                if (name.endsWith(last_suffix)) {
 
                    // Remove the existing sequence suffix
                    name = name.slice(0, name.length - last_suffix.length);
                }
            }
            name += ' ' + seq;
            seq += 1;
        }
        return name;
    }
 
    function ask_user_to_name_layer(layer_name, dup_name) {
 
        // Give the user a chance to name the layer
        var promptString = (dup_name) ?
                            dup_name + ' is in use, how about this one?'
                            : 'Please provide a label for this new layer',
            text = prompt(promptString, layer_name);
        if (text) {
            return text.trim();
        } else {
            return undefined;
        }
    }
 
    function let_user_name_layer(layer_name) {
 
        var name = layer_name;
 
        // if no layer name was supplied, assume this is a selection
        if (!name) {
            name = selection_prefix;
        } else {
            name = name.trim();
        }
 
        // Start with a unique name as a suggestion
        name = make_layer_name_unique(name);
 
        var unique_name,
            dup_name;
 
        // Keep asking the user for a name until it is unique or she cancels
        while (true) {
            name = ask_user_to_name_layer(name, dup_name);
            
            // We're done if she cancels
            if (name === undefined) { break; }
 
            // We're done if the name is unique
            unique_name = make_layer_name_unique(name);
            if (unique_name === name) { break; }

            // Suggest another unique name
            dup_name = name;
            name = unique_name;
        }
        return name;
    }
 
    function load_dynamic_colormap (name, layer, cats) {

        // Load the colormap for dynamic categorical or binary attributes.
        if ('colormap' in layer) {
        
            // Load the supplied colormap
            colormaps[name] = _.map(layer.colormap, function (vals, i) {
                return {
                    name: vals[0],
                    color: new Color(vals[1]),
                    fileColor: new Color(vals[1]),
                }
            });

            if (cats) {
         
                // Replace category string values with indices
                layer.data = _.object(
                    _.keys(layer.data),
                    _.map(layer.data, function (strVal, key) {
                        return cats.indexOf(strVal);
                    })
                );
            }
        } else if (cats && cats.length) {
        
            // Generate a colormap
            var jpColormap = _.map(
                jPalette.ColorMap.get('hexmap')(cats.length + 1).map,
                function (val, key) {
            
                    // Ignore alpha, taking the default of one.
                    return {r: val.r, g: val.g, b: val.b};
                }
            );
         
            // Remove the repeating red at the end
            jpColormap.splice(cats.length, 1);
         
            // Load this generated colormap
            colormaps[name] = _.map(jpColormap, function(color, i) {
                return {
                    name: cats[i],
                    color: Color(color), // operating color in map
                    fileColor: Color(color), // color from orig file
                };
            });
         
            // Replace category string values with indices
            layer.data = _.object(
                _.keys(layer.data),
                _.map(layer.data, function (strVal, key) {
                    return cats.indexOf(strVal);
                })
            );

        } else {
        
            // The default binary colormap for non-string values
            colormaps[name] = [];
        }
        
        // Remove these that are no longer needed.
        delete layer.colormap;
    }

    function load_dynamic_data_type (name, layer) {
    
        // Load the data type for dynamic attributes.
        var uniqueVals,
            hasStrings;

        // Skip any layers with no values.
        
        if (_.keys(layer.data).length < 1) { return; }

        // Drop any nulls or values used to indicate no value.
        var drop = ['', '#N/A', '#N/A N/A', '#NA', '-1.#IND', '-1.#QNAN', 
            '-NAN', '1.#IND', '1.#QNAN', 'N/A', 'NA', 'NULL', 'NAN'];
        _.each(layer.data, function (val, key) {
            if (_.isNull(val) || _.isUndefined(val) || _.isNaN(val)) {
                delete layer.data[key];
            } else if (drop.indexOf(val.toString().toUpperCase()) > -1) {
                delete layer.data[key];
            }
        });
 
        if (!('dataType' in layer)) {
        
            // Determine the data type since it was not supplied.
         
            // If they are any strings, this gets a colormap
            // and call it categorical for now. It may be binary.
            var aString = _.find(layer.data, function (value) {
                    return _.isNaN(parseFloat(value));
                });
            if (aString && aString.length > 0) {
                layer.dataType = 'categorical';
                hasStrings = true;
            }
         
            // Find the count of each unique value.
            uniqueVals = _.countBy(layer.data, function (value) {
                return value;
            });
{'fraction_non_alphanumeric': 0.10294657946946874, 'fraction_numerical': 0.0026453082518921303, 'mean_word_length': 1.2939490982639474, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1636764', 'n_tokens_mistral': 3765, 'n_tokens_neox': 3529, 'n_words': 1076}