<commit_before>(function($){

    /* Show theme to the user */
    function selectTheme(themeId){
        var selection = window.getSelection();
        selection.removeAllRanges();

        var range = document.createRange();
        var s = $(".motyw[theme-class='" + themeId + "']")[0];
        var e = $(".end[theme-class='" + themeId + "']")[0];

        if (s && e) {
            range.setStartAfter(s);
            range.setEndBefore(e);
            selection.addRange(range);
        }
    };

    /* Verify insertion port for annotation or theme */
    function verifyTagInsertPoint(node){
        if (node.nodeType == 3) { // Text Node
            node = node.parentNode;
        }

        if (node.nodeType != 1) {
            return false;
        }

        node = $(node);
        var xtype = node.attr('x-node');

        if (!xtype || (xtype.search(':') >= 0) ||
        xtype == 'motyw' ||
        xtype == 'begin' ||
        xtype == 'end') {
            return false;
        }

        // don't allow themes inside annotations
        if (node.is('*[x-annotation-box] *'))
            return false;

        return true;
    }

    /* Convert HTML fragment to plaintext */
    var ANNOT_FORBIDDEN = ['pt', 'pa', 'pr', 'pe', 'begin', 'end', 'theme'];

    function html2plainText(fragment){
        var text = "";

        $(fragment.childNodes).each(function(){
            if (this.nodeType == 3) // textNode
                text += this.nodeValue;
            else {
                if (this.nodeType == 1 &&
                        $.inArray($(this).attr('x-node'), ANNOT_FORBIDDEN) == -1) {<commit_msg>Fixes #789: Motifs in annotations<commit_after>(function($){

    /* Show theme to the user */
    function selectTheme(themeId){
        var selection = window.getSelection();
        selection.removeAllRanges();

        var range = document.createRange();
        var s = $(".motyw[theme-class='" + themeId + "']")[0];
        var e = $(".end[theme-class='" + themeId + "']")[0];

        if (s && e) {
            range.setStartAfter(s);
            range.setEndBefore(e);
            selection.addRange(range);
        }
    };

    /* Verify insertion port for annotation or theme */
    function verifyTagInsertPoint(node){
        if (node.nodeType == 3) { // Text Node
            node = node.parentNode;
        }

        if (node.nodeType != 1) {
            return false;
        }

        node = $(node);
        var xtype = node.attr('x-node');

        if (!xtype || (xtype.search(':') >= 0) ||
        xtype == 'motyw' ||
        xtype == 'begin' ||
        xtype == 'end') {
            return false;
        }

        // don't allow themes inside annotations
        if (node.is('*[x-annotation-box] *'))
            return false;

        return true;
    }

    /* Convert HTML fragment to plaintext */
    var ANNOT_FORBIDDEN = ['pt', 'pa', 'pr', 'pe', 'begin', 'end', 'motyw'];

    function html2plainText(fragment){
        var text = "";

        $(fragment.childNodes).each(function(){
            if (this.nodeType == 3) // textNode
                text += this.nodeValue;
            else {
                if (this.nodeType == 1 &&
                        $.inArray($(this).attr('x-node'), ANNOT_FORBIDDEN) == -1) {
{'fraction_non_alphanumeric': 0.1624923640806353, 'fraction_numerical': 0.006414172266340868, 'mean_word_length': 1.7223607647547796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3305452', 'n_tokens_mistral': 1012, 'n_tokens_neox': 959, 'n_words': 228}