<commit_before>define([
    'jquery',
], function ($) {
    function createSet(list) {
        var set = {};
        $.each(list, function (i, el) {
            set[el] = true;
        });
        return set;
    }

    var CleanInputMixin = {
        keep_tags: ["b", "i", "a", "p", "br", "strong", "em"],

        processNode: function(node, keep, wrapper) {
            var nodes, clean_node,
                node_name;

            switch (node.nodeType) {
                case 3: // Text node
                    return node.parentNode ? node.parentNode.removeChild(node) : node;
                case 1:
                    node_name = node.nodeName.toLowerCase();
                    if (keep[node_name] || wrapper) {
                        clean_node = document.createElement(node_name);
                        if (node_name === "a") { // Special case
                            $(clean_node).attr("href", $(node).attr("href"));
                        }
                    } else {
                        clean_node = document.createDocumentFragment();
                    }
                    nodes = node.childNodes;
                    while(nodes.length) {
                        clean_node.appendChild(this.processNode(nodes[0], keep));
                    }
                    // Remove old one to prevent infinite loop
                    if (node.parentNode) {
                        node.parentNode.removeChild(node);
                    }
                    return clean_node;
                default:
                    // Drop other elements (comments...)
                    node.parentNode.removeChild(node);
            }
        },

        cleanInput: function (dirty) {
            var $el = $("<div>"),
                keep = createSet(this.keep_tags),
                el;<commit_msg>Fix: processedNode in clearinput-mixin can return null<commit_after>define([
    'jquery',
], function ($) {
    function createSet(list) {
        var set = {};
        $.each(list, function (i, el) {
            set[el] = true;
        });
        return set;
    }

    var CleanInputMixin = {
        keep_tags: ["b", "i", "a", "p", "br", "strong", "em"],

        processNode: function(node, keep, wrapper) {
            var nodes, clean_node, processed_node,
                node_name;

            switch (node.nodeType) {
                case 3: // Text node
                    return node.parentNode ? node.parentNode.removeChild(node) : node;
                case 1:
                    node_name = node.nodeName.toLowerCase();
                    if (keep[node_name] || wrapper) {
                        clean_node = document.createElement(node_name);
                        if (node_name === "a") { // Special case
                            $(clean_node).attr("href", $(node).attr("href"));
                        }
                    } else {
                        clean_node = document.createDocumentFragment();
                    }
                    nodes = node.childNodes;
                    while(nodes.length) {
                        processed_node = this.processNode(nodes[0], keep);
                        if (processed_node) {
                            clean_node.appendChild(processed_node);
                        }
                    }
                    // Remove old one to prevent infinite loop
                    if (node.parentNode) {
                        node.parentNode.removeChild(node);
                    }
                    return clean_node;
                default:
                    // Drop other elements (comments...)
                    node.parentNode.removeChild(node);
            }
        },

        cleanInput: function (dirty) {
            var $el = $("<div>"),
                keep = createSet(this.keep_tags),
                el;
{'fraction_non_alphanumeric': 0.11241523213354199, 'fraction_numerical': 0.001564945226917058, 'mean_word_length': 1.1211283185840708, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4326110', 'n_tokens_mistral': 954, 'n_tokens_neox': 898, 'n_words': 198}