<commit_before>// Takes an object in the form of:
// { name: "blah", children: [ { name: "blah 2", children: [ ... ] } ] }
// and counts the number of objects without children
function sf_viz_countTailNodes(arg) {
    var data = arg;
    var count = 0;

    for (var i = 0; i < data.length; i++) {
        for (var p in data[i]) {
            if (p == "children" && data[i].children == null) {
                count++;
                continue;
            }
            if (p == "children" && data[i].children != null) {
                count += sf_viz_countTailNodes(data[i].children);
            }
        }
    }

    return count;
}

// As above but counts the total number of objects
function sf_viz_countTotalNodes(arg) {
    var data = arg;
    var count = 0;

    for (var i = 0; i < data.length; i++) {
        for (var p in data[i]) {
            if (p == "name") {
                count++;
                continue;
            }
            if (p == "children" && data[i].children != null) {
                count += sf_viz_countTotalNodes(data[i].children);
            }
        }
    }

    return count;
}

// As above but counts the highest number of levels
function sf_viz_countLevels(arg, levelsDeep, maxLevels) {
    var data = arg;
    var levels = levelsDeep;
    var max = maxLevels;

    for (var i = 0; i < data.length; i++) {
        for (var p in data[i]) {
            // We've hit a member with children..
            if (p == "children" && data[i].children != null) {
                levels++;
                [ levels, max ] = sf_viz_countLevels(data[i].children, levels, max);
            }

            if (p == "children" && data[i].children == null) {
                if (levels > max) {
                    //alert("max = " + levels);
                    max = levels;
                }
            }
        }

        // Reset to the level we're at as we iterate through the next child.
        levels = levelsDeep;
    }

    return [ levels, max ];
}

function sf_viz_dendrogram(targetId, data) {
    var plotData = data['tree'];
    var dataMap = data['data'];
    console.dir(dataMap);
    var width = sf_viz_countLevels([plotData], 0, 0)[1] * 120;
    var height = sf_viz_countTailNodes([plotData]) * 20;

    if (width < 600) {
        width = 600;
    }
    if (height < 600) {
        height = 600;
    }  

    var cluster = d3.layout.cluster()
        .size([height, width - 160]);

    var diagonal = d3.svg.diagonal()
        .projection(function(d) { return [d.y, d.x]; });

    var svg = d3.select(targetId).append("svg")
        .attr("width", width)
        .attr("height", height)
        .append("g")
        .attr("transform", "translate(40,0)");

    var nodes = cluster.nodes(plotData),
        links = cluster.links(nodes);

    var link = svg.selectAll(".link")
        .data(links)
        .enter().append("path")
        .attr("class", "dend-link")
        .attr("d", diagonal);

    var node = svg.selectAll(".node")
        .data(nodes)
        .enter().append("g")
        .attr("class", "dend-node")
        .attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; })
        .on("mouseover", function(d, i) {
            d3.select(this).style("fill", "silver");
            showToolTip(buildPopupMessage(dataMap[d.name]), d3.event.pageX+10, d3.event.pageY+10,true);
        })
        .on("mouseout", function() {
            d3.select(this).style("fill", "black");
            showToolTip(" ",0,0,false);
        });

    node.append("circle")
        .attr("r", 4.5);

    node.append("text")
        .attr("dx", function(d) { 
            if (d.depth == 0) { 
                return 50;
            }

            return d.children ? -8 : 8; 
        })
        .attr("dy", 3)
        .style("text-anchor", function(d) { return d.children ? "end" : "start"; })
        .text(function(d) { 
            if (dataMap[d.name][1].length > 20) {
                return dataMap[d.name][1].substring(0, 20) + "...";
            } else {
                return dataMap[d.name][1];
            }
        });

    d3.select(targetId).style("height", height + "px");

    function buildPopupMessage(data) {
        if (data[1].length > 200) {
            data[1] = data[1].substring(0, 200) + "...";<commit_msg>Raw data clean up in visualization.<commit_after>// Takes an object in the form of:
// { name: "blah", children: [ { name: "blah 2", children: [ ... ] } ] }
// and counts the number of objects without children
function sf_viz_countTailNodes(arg) {
    var data = arg;
    var count = 0;

    for (var i = 0; i < data.length; i++) {
        for (var p in data[i]) {
            if (p == "children" && data[i].children == null) {
                count++;
                continue;
            }
            if (p == "children" && data[i].children != null) {
                count += sf_viz_countTailNodes(data[i].children);
            }
        }
    }

    return count;
}

// As above but counts the total number of objects
function sf_viz_countTotalNodes(arg) {
    var data = arg;
    var count = 0;

    for (var i = 0; i < data.length; i++) {
        for (var p in data[i]) {
            if (p == "name") {
                count++;
                continue;
            }
            if (p == "children" && data[i].children != null) {
                count += sf_viz_countTotalNodes(data[i].children);
            }
        }
    }

    return count;
}

// As above but counts the highest number of levels
function sf_viz_countLevels(arg, levelsDeep, maxLevels) {
    var data = arg;
    var levels = levelsDeep;
    var max = maxLevels;

    for (var i = 0; i < data.length; i++) {
        for (var p in data[i]) {
            // We've hit a member with children..
            if (p == "children" && data[i].children != null) {
                levels++;
                [ levels, max ] = sf_viz_countLevels(data[i].children, levels, max);
            }

            if (p == "children" && data[i].children == null) {
                if (levels > max) {
                    //alert("max = " + levels);
                    max = levels;
                }
            }
        }

        // Reset to the level we're at as we iterate through the next child.
        levels = levelsDeep;
    }

    return [ levels, max ];
}

function sf_viz_dendrogram(targetId, data) {
    var plotData = data['tree'];
    var dataMap = data['data'];
    console.dir(dataMap);
    var width = sf_viz_countLevels([plotData], 0, 0)[1] * 120;
    var height = sf_viz_countTailNodes([plotData]) * 20;

    if (width < 600) {
        width = 600;
    }
    if (height < 600) {
        height = 600;
    }  

    var cluster = d3.layout.cluster()
        .size([height, width - 160]);

    var diagonal = d3.svg.diagonal()
        .projection(function(d) { return [d.y, d.x]; });

    var svg = d3.select(targetId).append("svg")
        .attr("width", width)
        .attr("height", height)
        .append("g")
        .attr("transform", "translate(40,0)");

    var nodes = cluster.nodes(plotData),
        links = cluster.links(nodes);

    var link = svg.selectAll(".link")
        .data(links)
        .enter().append("path")
        .attr("class", "dend-link")
        .attr("d", diagonal);

    var node = svg.selectAll(".node")
        .data(nodes)
        .enter().append("g")
        .attr("class", "dend-node")
        .attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; })
        .on("mouseover", function(d, i) {
            d3.select(this).style("fill", "silver");
            showToolTip(buildPopupMessage(dataMap[d.name]), d3.event.pageX+10, d3.event.pageY+10,true);
        })
        .on("mouseout", function() {
            d3.select(this).style("fill", "black");
            showToolTip(" ",0,0,false);
        });

    node.append("circle")
        .attr("r", 4.5);

    node.append("text")
        .attr("dx", function(d) { 
            if (d.depth == 0) { 
                return 50;
            }

            return d.children ? -8 : 8; 
        })
        .attr("dy", 3)
        .style("text-anchor", function(d) { return d.children ? "end" : "start"; })
        .text(function(d) { 
            if (dataMap[d.name][4] == "RAW_DATA") {
                return "Raw Data";
            }
            if (dataMap[d.name][1].length > 20) {
                return dataMap[d.name][1].substring(0, 20) + "...";
            } else {
                return dataMap[d.name][1];
            }
        });

    d3.select(targetId).style("height", height + "px");

    function buildPopupMessage(data) {
        if (data[1].length > 200) {
            data[1] = data[1].substring(0, 200) + "...";
{'fraction_non_alphanumeric': 0.1643127798828798, 'fraction_numerical': 0.01664944310483408, 'mean_word_length': 1.7041291524371314, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5956985', 'n_tokens_mistral': 2913, 'n_tokens_neox': 2722, 'n_words': 653}