<commit_before>/*jshint browser: true, jquery: true */
/*global clique, _ */

function bigramGraph() {
    "use strict";

    return $.getJSON("assets/bigram.json").then(function (bigram) {
        var alphabet = "abcdefghijklmnopqrstuvwxyz",
            nodes = [],
            links = [],
            threshold = 3 * (1 / 676),
            aCodePoint = "a".codePointAt(0);

        _.each(alphabet, function (letter) {
            nodes.push({
                name: letter
            });
        });

        _.each(alphabet, function (first) {
            _.each(alphabet, function (second) {
                if (first !== second && bigram[first + second] > threshold) {
                    links.push({
                        source: first.codePointAt(0) - aCodePoint,
                        target: second.codePointAt(0) - aCodePoint
                    });
                }<commit_msg>Include some undirected links in the graph<commit_after>/*jshint browser: true, jquery: true */
/*global clique, _ */

function bigramGraph() {
    "use strict";

    // Construct a graph based on bigram frequency in the English language. The
    // nodes each represent a letter of the alphabet, while a link from one
    // letter to another means the bigram frequency of those two letter is
    // greater than three times that predicted by chance alone. If the reverse
    // bigram also has this property, the link will be undirected.
    return $.getJSON("assets/bigram.json").then(function (bigram) {
        var alphabet = "abcdefghijklmnopqrstuvwxyz",
            nodes = [],
            links = [],
            threshold = 3 * (1 / 676),
            done = {},
            aCodePoint = "a".codePointAt(0);

        // Construct the node set, one per English letter.
        _.each(alphabet, function (letter) {
            nodes.push({
                name: letter
            });
        });

        // Iterate through each possible bigram.
        _.each(alphabet, function (first) {
            _.each(alphabet, function (second) {
                var key = first + second,
                    rev = second + first,
                    link;

                // Omitting double letters (not supported by Clique cola view),
                // check to see whether the bigram has high enough frequence
                // (and wasn't already processed as a frequent reverse of an
                // earlier bigram).
                if (first !== second && !_.has(done, key) && bigram[key] > threshold) {
                    link = {
                        source: first.codePointAt(0) - aCodePoint,
                        target: second.codePointAt(0) - aCodePoint
                    };

                    // Make the link undirected if the reverse bigram is
                    // sufficiently frequent as well.
                    if (bigram[rev] > threshold) {
                        link.undirected = true;
                        done[rev] = null;
                    }

                    links.push(link);
                }
{'fraction_non_alphanumeric': 0.1015445284258955, 'fraction_numerical': 0.0052579691094314825, 'mean_word_length': 1.4528605962933119, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1823927', 'n_tokens_mistral': 802, 'n_tokens_neox': 743, 'n_words': 240}