<commit_before><!doctype html>
<canvas id="canvas" width="710" height="710" />
<script src="../rbush.js"></script>
<script>
    var canvas = document.getElementById('canvas');
    var ctx = canvas.getContext('2d');

    var N = 10000,
        W = 700,
        data = [];

    function randBox(size) {
        var x = Math.random() * (700 - size),
            y = Math.random() * (700 - size);
        return [x, y,
            x + size * Math.random(),
            y + size * Math.random()];
    }

    for (var i = 0; i < N; i++) {
        data[i] = randBox(1);
    }

    var tree = rbush(14).load(data);

    var colors = ['#ff0000', 'green', '#fcc'];

    var rects = [];

    function drawTree(node, level) {
        var rect = [];

        if (level) {
            rect.push(colors[(level - 1) % colors.length]);
            rect.push([
                Math.round(node.bbox[0]) + 0.5,
                Math.round(node.bbox[1]) + 0.5,
                Math.round(node.bbox[2] - node.bbox[0]),
                Math.round(node.bbox[3] - node.bbox[1])
            ]);
        }

        rects.push(rect);

        if (node.leaf) return;
        //if (level === 3) { return; }

        for (var i = 0; i < node.children.length; i++) {
            drawTree(node.children[i], level + 1);
        }
    }

    drawTree(tree.data, 0);

    for (var i = rects.length - 1; i >= 0; i--) {
        ctx.strokeStyle = rects[i][0];
        ctx.strokeRect.apply(ctx, rects[i][1]);
    }
</script><commit_msg>one more viz update<commit_after><!doctype html>
<canvas id="canvas" width="710" height="710" />
<script src="../rbush.js"></script>
<script>
    var canvas = document.getElementById('canvas');
    var ctx = canvas.getContext('2d');

    var N = 50000,
        W = 700,
        data = [];

    function randBox(size) {
        var x = Math.random() * (700 - size),
            y = Math.random() * (700 - size);
        return [x, y,
            x + size * Math.random(),
            y + size * Math.random()];
    }

    for (var i = 0; i < N; i++) {
        data[i] = randBox(1);
    }

    var tree = rbush(10).load(data);

    var colors = ['red', 'blue', 'green'];

    var rects = [];

    function drawTree(node, level) {
        var rect = [];

        if (level) {
            rect.push(colors[(level - 1) % colors.length]);
            rect.push(1 / level);
            rect.push([
                Math.round(node.bbox[0]) + 0.5,
                Math.round(node.bbox[1]) + 0.5,
                Math.round(node.bbox[2] - node.bbox[0]),
                Math.round(node.bbox[3] - node.bbox[1])
            ]);
        }

        rects.push(rect);

        if (node.leaf) return;
        //if (level === 8) { return; }

        for (var i = 0; i < node.children.length; i++) {
            drawTree(node.children[i], level + 1);
        }
    }

    drawTree(tree.data, 0);

    for (var i = rects.length - 1; i >= 0; i--) {
        ctx.strokeStyle = rects[i][0];
        ctx.globalAlpha = rects[i][1];
        ctx.strokeRect.apply(ctx, rects[i][2]);
    }
</script>
{'fraction_non_alphanumeric': 0.18466557911908646, 'fraction_numerical': 0.03066884176182708, 'mean_word_length': 1.6777292576419214, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5580777', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1028, 'n_words': 230}