<commit_before>/**@file dashboard.js
   Creates the 'bgexplorer.dashboard' namespace. This will load the model's
   datatable TSV
   export from the given endpoint and create a series of crossfilter groups
   and d3.js hierarchies based on the column headings.

   This library requires both d3.js and crossfilter and must be loaded
   after them.
*/

(function(bgexplorer){
//set up the 'dashboard' sub-namespace
bgexplorer.dashboard = bgexplorer.dashboard || {}
dashboard = bgexplorer.dashboard

//various configuration settings can be changed here
dashboard.config = {
    'tableprecision': 2,
    'tabledecimals':  3,
    'titleprecision': 5,
    'defaulttabledepth': 1,
    'maxchartdepth': 3,
    'minnodesize': 0.003,
    'mintextsize':0.05,
    'defaultcharttype': 'pie',
    'transitionduration':1000,
};

//top-level crossfilter instance
dashboard.cf = crossfilter();
//crossfilter dimensions and groups. Will be keyed by the column headings
dashboard.cfdimensions = {};
dashboard.cfgroups = {};
dashboard.cfgroupAll = dashboard.cf.groupAll();
dashboard.cffilters = {};
//d3.js hierarchies. will be keyed by the column headings
dashboard.hierarchies = {};
//sorting functions for groups
dashboard.groupsort = {};

dashboard.valuetypes = [];

//list of interactive display objects that should be updated when filters change
dashboard.displays = {
    'tables': [],
    'charts': [],
};

// base url for generating spectra
dashboard.basespecurl = null;

//register a function to be called when data finishes loading
var onload = []
dashboard.onLoad = function(callback){
    onload.push(callback);
    if(dashboard.dataloaded){
        callback();
        onload.pop();
    }
};


//parse a value column that has uncertainties in it
dashboard.parsevalstring = function(val){
    //strings are of the form "X+/-E" or "(X+/-E)eY" with an optional leading <
    var islimit = val.startsWith('<');
    if(islimit){
        val = val.substr(1);
    }
    var expo="e0";
    if(val.startsWith("(")){
        var split = val.split("e");
        expo = "e"+split[1];
        val = split[0].slice(1,-1);
    }
    var split = val.split("+/-");
    if(split.length == 1)
        split[1] = "0";
    return [+(split[0]+expo), +(split[1]+expo), islimit];
};


//parse the rows in the tsv datatable. Should be passed as the second argument
//to d3.tsv
var splitkey = '___';
var groups = [];
dashboard.parserow = function(row){
    if(groups.length == 0){//construct the list of groups
        for(var col in row){
            if(col.startsWith("G_"))
                groups.push(col.substr(2));
            else if(col.startsWith("V_"))
                dashboard.valuetypes.push(col.substr(2));
        }
    }
    var out = { 'ID': row.ID, 'groups': {}, 'values': {}, 'variance': {} };
    groups.forEach(function(g){ out.groups[g] = row['G_'+g].split(splitkey); });
    dashboard.valuetypes.forEach(function(v){
        parsed = dashboard.parsevalstring(row['V_'+v])
        out.values[v] = parsed[0];
        out.values[v+"_sig2"] = parsed[1]*parsed[1];
        out.values[v+"_lim"] = parsed[2]*parsed[0];
    });
    return out;
};

// override this to filter the data before building the crossfilter
dashboard.prefilterfunc = null;


//take the list of objects returned from parserows and construct the crossfiler
// and d3 structures
dashboard.processtable = function(error,rows){
    if(error){alert("An error occurred: "+error); throw error;}
    if(dashboard.prefilterfunc){
        console.log("using prefilter function, initial size "+rows.length);
        rows = rows.filter(dashboard.prefilterfunc);
        console.log("filtered size "+rows.length)
    }
    dashboard.data = rows

    var first = rows[0];
    if(groups.length == 0){
        //we didn't build up the groups list
        groups = keys(first.groups);
        dashboard.valuetypes = keys(first.values);
    }

    //crossfilter group reduce functions
    function reduceInitial(){ return {count:0}; }
    function reduceAdd(p, v){
        for(var val in v.values){
            p[val] = (val in p ? p[val] : 0.) + v.values[val];
        }
        p.count += 1;
        return p;
    }
    function reduceRemove(p,v){
        for(var val in v.values){
            p[val] = p[val] - v.values[val];
        }<commit_msg>more helpful error message<commit_after>/**@file dashboard.js
   Creates the 'bgexplorer.dashboard' namespace. This will load the model's
   datatable TSV
   export from the given endpoint and create a series of crossfilter groups
   and d3.js hierarchies based on the column headings.

   This library requires both d3.js and crossfilter and must be loaded
   after them.
*/

(function(bgexplorer){
//set up the 'dashboard' sub-namespace
bgexplorer.dashboard = bgexplorer.dashboard || {}
dashboard = bgexplorer.dashboard

//various configuration settings can be changed here
dashboard.config = {
    'tableprecision': 2,
    'tabledecimals':  3,
    'titleprecision': 5,
    'defaulttabledepth': 1,
    'maxchartdepth': 3,
    'minnodesize': 0.003,
    'mintextsize':0.05,
    'defaultcharttype': 'pie',
    'transitionduration':1000,
};

//top-level crossfilter instance
dashboard.cf = crossfilter();
//crossfilter dimensions and groups. Will be keyed by the column headings
dashboard.cfdimensions = {};
dashboard.cfgroups = {};
dashboard.cfgroupAll = dashboard.cf.groupAll();
dashboard.cffilters = {};
//d3.js hierarchies. will be keyed by the column headings
dashboard.hierarchies = {};
//sorting functions for groups
dashboard.groupsort = {};

dashboard.valuetypes = [];

//list of interactive display objects that should be updated when filters change
dashboard.displays = {
    'tables': [],
    'charts': [],
};

// base url for generating spectra
dashboard.basespecurl = null;

//register a function to be called when data finishes loading
var onload = []
dashboard.onLoad = function(callback){
    onload.push(callback);
    if(dashboard.dataloaded){
        callback();
        onload.pop();
    }
};


//parse a value column that has uncertainties in it
dashboard.parsevalstring = function(val){
    //strings are of the form "X+/-E" or "(X+/-E)eY" with an optional leading <
    var islimit = val.startsWith('<');
    if(islimit){
        val = val.substr(1);
    }
    var expo="e0";
    if(val.startsWith("(")){
        var split = val.split("e");
        expo = "e"+split[1];
        val = split[0].slice(1,-1);
    }
    var split = val.split("+/-");
    if(split.length == 1)
        split[1] = "0";
    return [+(split[0]+expo), +(split[1]+expo), islimit];
};


//parse the rows in the tsv datatable. Should be passed as the second argument
//to d3.tsv
var splitkey = '___';
var groups = [];
dashboard.parserow = function(row){
    if(groups.length == 0){//construct the list of groups
        for(var col in row){
            if(col.startsWith("G_"))
                groups.push(col.substr(2));
            else if(col.startsWith("V_"))
                dashboard.valuetypes.push(col.substr(2));
        }
    }
    var out = { 'ID': row.ID, 'groups': {}, 'values': {}, 'variance': {} };
    groups.forEach(function(g){ out.groups[g] = row['G_'+g].split(splitkey); });
    dashboard.valuetypes.forEach(function(v){
        parsed = dashboard.parsevalstring(row['V_'+v])
        out.values[v] = parsed[0];
        out.values[v+"_sig2"] = parsed[1]*parsed[1];
        out.values[v+"_lim"] = parsed[2]*parsed[0];
    });
    return out;
};

// override this to filter the data before building the crossfilter
dashboard.prefilterfunc = null;


//take the list of objects returned from parserows and construct the crossfiler
// and d3 structures
dashboard.processtable = function(error,rows){
    if(error){
        console.log("An error occurred: ");
        console.log(error);
        d3.select("#loading").classed("text-danger", true)
          .html("<p>Error "+error.target.status+" downloading datatable:</p>\n"
                +error.target.responseText);
        throw error;
    }
    if(dashboard.prefilterfunc){
        console.log("using prefilter function, initial size "+rows.length);
        rows = rows.filter(dashboard.prefilterfunc);
        console.log("filtered size "+rows.length)
    }
    dashboard.data = rows

    var first = rows[0];
    if(groups.length == 0){
        //we didn't build up the groups list
        groups = keys(first.groups);
        dashboard.valuetypes = keys(first.values);
    }

    //crossfilter group reduce functions
    function reduceInitial(){ return {count:0}; }
    function reduceAdd(p, v){
        for(var val in v.values){
            p[val] = (val in p ? p[val] : 0.) + v.values[val];
        }
        p.count += 1;
        return p;
    }
    function reduceRemove(p,v){
        for(var val in v.values){
            p[val] = p[val] - v.values[val];
        }
{'fraction_non_alphanumeric': 0.14217741021864733, 'fraction_numerical': 0.01053585589668064, 'mean_word_length': 2.9445933869526364, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7546481', 'n_tokens_mistral': 2793, 'n_tokens_neox': 2766, 'n_words': 741}