<commit_before>$(document).ready(function () {
    trigramTable = {}; // For tags.length > 3
    unicodeTable = {};

    // {{{ load data
    $.ajax({
        url: 'data-nounihan.json',
        dataType: 'json',
        success: function(data) {
            $.each(data, function (index, unicodeChar) {
                unicodeTable[unicodeChar.c] = unicodeChar;

                // Use name as tags
                tagsToTrigrams(unicodeChar.c, unicodeChar.n.split(" "));

                // Use alias as tags
                if (unicodeChar.a) {
                    tagsToTrigrams(unicodeChar.c, unicodeChar.a);
                }
            });
        },
        error: function () {
            console.log("could not load data.json:", arguments);
        }
    });
    // }}}

    // {{{ wordToTrigrams(word)
    function wordToTrigrams(word) {
        if (word.length === 0) {
            return [];
        } else if (word.length < 3) {
            return [word];
        }
        var triGrams = [];
        for (var i=0; i<=word.length-3; i++) {
            triGrams.push(word.substr(i, 3));
        }
        return triGrams;
    }
    // }}}

    // {{{ tagsToTrigrams(unicodeChar, tag)
    function tagsToTrigrams(unicodeChar, tags) {
        for(var i=0; i<tags.length; i++) {
            tagToTrigrams(unicodeChar, tags[i]);
        }
    }
    // }}}

    // {{{ tagToTrigrams(unicodeChar, tag)
    function tagToTrigrams(unicodeChar, tag) {
        function addOrCreate(table, key, value) {
            table[key] = table[key] || [];
            table[key].push(value);
        }

        var tgs = wordToTrigrams(tag);

        if (tgs.indexOf("") !== -1) {
            console.log("''", unicodeChar);
        }

        $.each(wordToTrigrams(tag), function (index, triGram) {
            addOrCreate(trigramTable, triGram, unicodeChar);
        });
    }
    // }}}

    // {{{ search
    function search(text) {
        var words = text.split(" "),
            triGrams = [],
            charCodesFound = {};

        // Convert search terms to tri-grams
        $.each(words, function (index, word) {
            triGrams = triGrams.concat(wordToTrigrams(word));
        });

        // Look up tri-grams
        $.each(triGrams, function (index, triGram) {
            if (triGram in trigramTable) {
                var charCodes = trigramTable[triGram];
                for(var i=0; i<charCodes.length; i++) {
                    var charCode = charCodes[i];
                    charCodesFound[charCode] = charCodesFound[charCode] || 0;
                    charCodesFound[charCode] = charCodesFound[charCode] + 1;
                }
            }
        });

        // Score the results for how many matches there were
        var sortedOutput = [];
        $.each(charCodesFound, function(charCode, matchingTriGrams) {
            sortedOutput.push({code: charCode, score: matchingTriGrams});
        });

        // Sort in reverse
        sortedOutput.sort(function (a, b) {
            return b.score - a.score;
        });

        // Strip everything but the actual code
        return $.map(sortedOutput, function (data) {
            return data.code;
        });
    }
    // }}}

    // Handle search input
    $("#searchField").keyup(function (e) {
        // TODO: Check for non-text keypresses and empty strings
        var searchTerm = e.srcElement.value;

        var codes = search(searchTerm);

        var html = ['<div class="row">'];
        $.each(codes, function (index, code) {
            html.push('<div class="span1 resultChar">&#' + code + ';</div>');
        });
        html.push('</div>');
        $("#results").html(html.join(" "));


    });
});<commit_msg>Wait ¼ second without input starting search.<commit_after>$(document).ready(function () {
    trigramTable = {}; // For tags.length > 3
    unicodeTable = {};

    // {{{ load data
    $.ajax({
        url: 'data-nounihan.json',
        dataType: 'json',
        success: function(data) {
            $.each(data, function (index, unicodeChar) {
                unicodeTable[unicodeChar.c] = unicodeChar;

                // Use name as tags
                tagsToTrigrams(unicodeChar.c, unicodeChar.n.split(" "));

                // Use alias as tags
                if (unicodeChar.a) {
                    tagsToTrigrams(unicodeChar.c, unicodeChar.a);
                }
            });
        },
        error: function () {
            console.log("could not load data.json:", arguments);
        }
    });
    // }}}

    // {{{ wordToTrigrams(word)
    function wordToTrigrams(word) {
        if (word.length === 0) {
            return [];
        } else if (word.length < 3) {
            return [word];
        }
        var triGrams = [];
        for (var i=0; i<=word.length-3; i++) {
            triGrams.push(word.substr(i, 3));
        }
        return triGrams;
    }
    // }}}

    // {{{ tagsToTrigrams(unicodeChar, tag)
    function tagsToTrigrams(unicodeChar, tags) {
        for(var i=0; i<tags.length; i++) {
            tagToTrigrams(unicodeChar, tags[i]);
        }
    }
    // }}}

    // {{{ tagToTrigrams(unicodeChar, tag)
    function tagToTrigrams(unicodeChar, tag) {
        function addOrCreate(table, key, value) {
            table[key] = table[key] || [];
            table[key].push(value);
        }

        var tgs = wordToTrigrams(tag);

        if (tgs.indexOf("") !== -1) {
            console.log("''", unicodeChar);
        }

        $.each(wordToTrigrams(tag), function (index, triGram) {
            addOrCreate(trigramTable, triGram, unicodeChar);
        });
    }
    // }}}

    // {{{ search
    function search(text) {
        var words = text.split(" "),
            triGrams = [],
            charCodesFound = {};

        // Convert search terms to tri-grams
        $.each(words, function (index, word) {
            triGrams = triGrams.concat(wordToTrigrams(word));
        });

        // Look up tri-grams
        $.each(triGrams, function (index, triGram) {
            if (triGram in trigramTable) {
                var charCodes = trigramTable[triGram];
                for(var i=0; i<charCodes.length; i++) {
                    var charCode = charCodes[i];
                    charCodesFound[charCode] = charCodesFound[charCode] || 0;
                    charCodesFound[charCode] = charCodesFound[charCode] + 1;
                }
            }
        });

        // Score the results for how many matches there were
        var sortedOutput = [];
        $.each(charCodesFound, function(charCode, matchingTriGrams) {
            sortedOutput.push({code: charCode, score: matchingTriGrams});
        });

        // Sort in reverse
        sortedOutput.sort(function (a, b) {
            return b.score - a.score;
        });

        // Strip everything but the actual code
        return $.map(sortedOutput, function (data) {
            return data.code;
        });
    }
    // }}}

    function searchAndShow(text) {
        var codes = search(text);

        var html = ['<div class="row">'];
        $.each(codes, function (index, code) {
            html.push('<div class="span1 resultChar">&#' + code + ';</div>');
        });
        html.push('</div>');
        $("#results").html(html.join(" "));
    }

    // Handle search input
    var timeout;
    $("#searchField").keyup(function (e) {
        if (timeout) {
            console.log("cancel timeout");
            window.clearTimeout(timeout);
        }
        timeout = window.setTimeout(function () {
            console.log("timeout done");
            searchAndShow(e.srcElement.value);
        }, 250);
    });
});
{'fraction_non_alphanumeric': 0.14723526343244653, 'fraction_numerical': 0.0035211267605633804, 'mean_word_length': 1.656390717007274, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6969009', 'n_tokens_mistral': 2378, 'n_tokens_neox': 2233, 'n_words': 456}