<commit_before>/**
 * @FileOverview: Dom clean up routines
 * depends on: jQuery-dom-traversal, jQuery-dom-manipulation
 */

(function($) {
    $.Arte = $.Arte || {};
    $.Arte.dom = $.Arte.dom || {};

    // Cache references
    var dom = $.Arte.dom;
    var configuration = $.Arte.configuration;
    var constants = $.Arte.constants;
    var util = $.Arte.util;

    $.extend(configuration, {
        /**
         * A set of tagNames to which a style/class can be styled.
         * If a tagName is not styleable, the styles/classes will be applied to all of its
         * children or the parent depending on the markup.
         */
        styleableTags: {
            SPAN: 1,
            DIV: 1,
            P: 1,
            LI: 1,
            UL: 1,
            OL: 1
        },
        supportedTags: {
            "P": 1,
            "UL": 1,
            "OL": 1,
            "LI": 1,
            "SPAN": 1,
            "BR": 1 // Chrome add BR to keep a space
        }
    });

    $.extend(configuration, {
        cleanup: {
            options: {
                removeNonPrintableCharacters: true,
                removeEmptyElements: true,
                removeRedundantMarkup: true,
                mergeAdjacentLists: true
            },

            invalidTagHandlers: {
                "B": {
                    applierTagName: "span",
                    styleName: "font-weight",
                    styleValue: "bold"
                },
                "I": {
                    applierTagName: "span",
                    styleName: "font-style",
                    styleValue: "italic"
                },
                "U": {
                    applierTagName: "span",
                    styleName: "text-decoration",
                    styleValue: "underline"
                },
                "DIV": {
                    applierTagName: "P"
                }
            },

            /**
             * During the cleanup phase, the elements with tagName specified with Key can be merged
             * with the parent element specified by the values.
             * For example, A SPAN can be merged with SPAN/DIV/P/LI while a LI can't be merged with anything
             */
            mergableTags: {
                SPAN: {
                    SPAN: 1,
                    DIV: 1,
                    P: 1,
                    LI: 1
                },
                DIV: {
                    DIV: 1,
                    P: 1,
                    LI: 1
                },
                P: {
                    DIV: 1,
                    P: 1,
                    LI: 1
                },
                LI: {},
                OL: {},
                UL: {},
                B: {
                    B: 1
                },
                U: {
                    U: 1
                },
                I: {
                    I: 1
                },
                STRONG: {
                    STRONG: 1
                },
                SUB: {
                    SUB: 1
                },
                SUP: {
                    SUP: 1
                },
                H1: {
                    H2: 1,
                    H3: 1,
                    H4: 1,
                    H5: 1,
                    H6: 1
                },
                H2: {
                    H1: 1,
                    H3: 1,
                    H4: 1,
                    H5: 1,
                    H6: 1
                },
                H3: {
                    H1: 1,
                    H2: 1,
                    H4: 1,
                    H5: 1,
                    H6: 1
                },
                H4: {
                    H1: 1,
                    H2: 1,
                    H3: 1,
                    H5: 1,
                    H6: 1
                },
                H5: {
                    H1: 1,
                    H2: 1,
                    H3: 1,
                    H4: 1,
                    H6: 1
                },
                H6: {
                    H1: 1,
                    H2: 1,
                    H3: 1,
                    H4: 1,
                    H5: 1
                }
            },

            /**
             * Collection of invalid characters and character ranges
             */
            invalidCharacterRegex: [
                "\u0000-\u001F", // Control Characters
                "\u0080-\u009F", // Latin-Supplement many control characters in this range
                "\u2000-\u200F", // Invisible Puntuation
                "\uE000-\uF8FF" // Private use
            ]
        }
    });

    var cleanupConfig = configuration.cleanup;

    var mergeLists = function(tagName, lists) {
        var filter = function(index, node) {
            return !$(node).is(":emptyTextOrRangySpan");
        };
        // Start from the last element in the list and start merging backward
        while (lists.length) {
            var currentList = $(lists[lists.length - 1]);
            lists.splice(lists.length - 1, 1);
            var prevNode = dom.prevSiblingIncludingTextNodes(currentList, filter);

            // If the previous element has same list tagName, merge both of these elements
            if (prevNode && prevNode.prop("tagName") === tagName) {
                // Move the current node's li children to previous node
                currentList.children().appendTo(prevNode);
                currentList.remove();
            }
        }
    };

    /**
     * Merge adjacent lists within the set of matched element
     * For example <ul><li>1</li><ul><ul><li>2</li></ul> => <ul><li>1</li><li>2</li></ul>
     */
    var mergeAdjacentLists = function(jNodes) {
        jNodes.each(function() {
            mergeLists(constants.tagName.OL, $(this).find(constants.tagName.OL));
            mergeLists(constants.tagName.UL, $(this).find(constants.tagName.UL));
        });
    };

    var seekDirection = {
        Next: 0,<commit_msg>Fix mergeLists() to work after jQuery 1.10<commit_after>/**
 * @FileOverview: Dom clean up routines
 * depends on: jQuery-dom-traversal, jQuery-dom-manipulation
 */

(function($) {
    $.Arte = $.Arte || {};
    $.Arte.dom = $.Arte.dom || {};

    // Cache references
    var dom = $.Arte.dom;
    var configuration = $.Arte.configuration;
    var constants = $.Arte.constants;
    var util = $.Arte.util;

    $.extend(configuration, {
        /**
         * A set of tagNames to which a style/class can be styled.
         * If a tagName is not styleable, the styles/classes will be applied to all of its
         * children or the parent depending on the markup.
         */
        styleableTags: {
            SPAN: 1,
            DIV: 1,
            P: 1,
            LI: 1,
            UL: 1,
            OL: 1
        },
        supportedTags: {
            "P": 1,
            "UL": 1,
            "OL": 1,
            "LI": 1,
            "SPAN": 1,
            "BR": 1 // Chrome add BR to keep a space
        }
    });

    $.extend(configuration, {
        cleanup: {
            options: {
                removeNonPrintableCharacters: true,
                removeEmptyElements: true,
                removeRedundantMarkup: true,
                mergeAdjacentLists: true
            },

            invalidTagHandlers: {
                "B": {
                    applierTagName: "span",
                    styleName: "font-weight",
                    styleValue: "bold"
                },
                "I": {
                    applierTagName: "span",
                    styleName: "font-style",
                    styleValue: "italic"
                },
                "U": {
                    applierTagName: "span",
                    styleName: "text-decoration",
                    styleValue: "underline"
                },
                "DIV": {
                    applierTagName: "P"
                }
            },

            /**
             * During the cleanup phase, the elements with tagName specified with Key can be merged
             * with the parent element specified by the values.
             * For example, A SPAN can be merged with SPAN/DIV/P/LI while a LI can't be merged with anything
             */
            mergableTags: {
                SPAN: {
                    SPAN: 1,
                    DIV: 1,
                    P: 1,
                    LI: 1
                },
                DIV: {
                    DIV: 1,
                    P: 1,
                    LI: 1
                },
                P: {
                    DIV: 1,
                    P: 1,
                    LI: 1
                },
                LI: {},
                OL: {},
                UL: {},
                B: {
                    B: 1
                },
                U: {
                    U: 1
                },
                I: {
                    I: 1
                },
                STRONG: {
                    STRONG: 1
                },
                SUB: {
                    SUB: 1
                },
                SUP: {
                    SUP: 1
                },
                H1: {
                    H2: 1,
                    H3: 1,
                    H4: 1,
                    H5: 1,
                    H6: 1
                },
                H2: {
                    H1: 1,
                    H3: 1,
                    H4: 1,
                    H5: 1,
                    H6: 1
                },
                H3: {
                    H1: 1,
                    H2: 1,
                    H4: 1,
                    H5: 1,
                    H6: 1
                },
                H4: {
                    H1: 1,
                    H2: 1,
                    H3: 1,
                    H5: 1,
                    H6: 1
                },
                H5: {
                    H1: 1,
                    H2: 1,
                    H3: 1,
                    H4: 1,
                    H6: 1
                },
                H6: {
                    H1: 1,
                    H2: 1,
                    H3: 1,
                    H4: 1,
                    H5: 1
                }
            },

            /**
             * Collection of invalid characters and character ranges
             */
            invalidCharacterRegex: [
                "\u0000-\u001F", // Control Characters
                "\u0080-\u009F", // Latin-Supplement many control characters in this range
                "\u2000-\u200F", // Invisible Puntuation
                "\uE000-\uF8FF" // Private use
            ]
        }
    });

    var cleanupConfig = configuration.cleanup;

    var mergeLists = function(tagName, lists) {
        var filter = function(index, node) {
            // cwkTODO this is copied from getContentNodes
            // refactor it to its own method
            if (node.nodeType === 1) {
                return !$(node).is(":emptyTextOrRangySpan");
            } else {
                // Starting in jQuery 1.10,
                // filter() only works on nodeType 1 (ELEMENT_NODE)
                // (callStack: is() -> winnow() -> filter() ),
                // For other nodeTypes, e.g. 3 (TEXT_NODE),
                // we must manually do the !emptyTextOrRangySpan check

                var jQueryExpr = $.expr[":"];

                // These methods are added in jquery-dom-traversal
                var isEmptyText = (typeof jQueryExpr.emptyText === "function") ?
                    jQueryExpr.emptyText(node) : false;
                var isRangySpan = (typeof jQueryExpr.rangySpan(node) === "function") ?
                    jQueryExpr.rangySpan(node) : false;

                return !(isEmptyText || isRangySpan);
            }
        };
        // Start from the last element in the list and start merging backward
        while (lists.length) {
            var currentList = $(lists[lists.length - 1]);
            lists.splice(lists.length - 1, 1);
            var prevNode = dom.prevSiblingIncludingTextNodes(currentList, filter);

            // If the previous element has same list tagName, merge both of these elements
            if (prevNode && prevNode.prop("tagName") === tagName) {
                // Move the current node's li children to previous node
                currentList.children().appendTo(prevNode);
                currentList.remove();
            }
        }
    };

    /**
     * Merge adjacent lists within the set of matched element
     * For example <ul><li>1</li><ul><ul><li>2</li></ul> => <ul><li>1</li><li>2</li></ul>
     */
    var mergeAdjacentLists = function(jNodes) {
        jNodes.each(function() {
            mergeLists(constants.tagName.OL, $(this).find(constants.tagName.OL));
            mergeLists(constants.tagName.UL, $(this).find(constants.tagName.UL));
        });
    };

    var seekDirection = {
        Next: 0,
{'fraction_non_alphanumeric': 0.10183820677887226, 'fraction_numerical': 0.020398665942759638, 'mean_word_length': 0.850459242250287, 'pattern_counts': {'":': 23, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2009857', 'n_tokens_mistral': 3679, 'n_tokens_neox': 3215, 'n_words': 873}