<commit_before>/**
 * @namespace CSS минимизатор.
 */
var $cssm = {

    /**
     * TODO:
     *
     * @param s
     */
    isComment: function(s) {
        return s.length > 3 && s.substr(0, 2) === '/*' && s.slice(-2) === '*/';
    },

    /**
     * TODO:
     *
     * @param tree
     */
    minimize: function(tree) {
        var nodes = [];

        $cssm.copyTree(tree, nodes);
        $cssm.processComments(nodes);
        $cssm._minimize(nodes);

        return { nodes: $cssm.restructure(nodes) };
    },

    /**
     * TODO:
     *
     * @param nodes
     */
    processComments: function(nodes) {
        var node, i = 0, j, c, t, mode = { protect: false, order: false };

        for (; i < nodes.length; i++) {
            node = nodes[i];
            if (typeof node === 'string') {
                $cssm.checkMode(node, mode);
                nodes.splice(i, 1);
                i--;
            } else {
                if (node.value.length) {
                    node.value.forEach(function(value) {
                        $cssm.cleanComments(value);
                    });
                }
                if (node.content.length) {
                    for (j = 0; j < node.content.length; j++) {
                        c = node.content[j];
                        if (typeof c === 'string') {
                            $cssm.checkMode(c, mode);
                            if (!mode.protect && c === '/*p*/') {
                                if ((t = node.content[j + 1]) !== undefined && typeof t !== 'string') t.protect = true;
                            } else if (!mode.order && c === '/*o*/') {
                                if ((t = node.content[j + 1]) !== undefined && typeof t !== 'string') t.order = true;
                            }

                            node.content.splice(j, 1);
                            j--;
                        } else {
                            if (mode.protect) c.protect = true;
                            if (!mode.order && !c.order) c.n = c.n - 1000000;
                            $cssm.cleanComments(c.name);
                            $cssm.cleanComments(c.value);
                        }
                    }
                }
                if (node.nodes && node.nodes.length) $cssm.processComments(node.nodes);
            }
        }
    },

    /**
     * TODO:
     *
     * @param s
     * @param mode
     */
    checkMode: function(s, mode) {
        switch (s) {
            case '/*p<*/':
                mode.protect = true;
                break;
            case '/*>p*/':
                mode.protect = false;
                break;
            case '/*o<*/':
                mode.order = true;
                break;
            case '/*>o*/':
                mode.order = false;
                break;
        }
    },

    /**
     * TODO:
     *
     * @param tokens
     */
    cleanComments: function(tokens) {
        for (var i = 0; i < tokens.length; i++) {
            if ($cssm.isComment(tokens[i])) {
                if (tokens[i + 1] === ' ') tokens.splice(i + 1, 1);
                if (tokens[i - 1] === ' ') tokens.splice(i - 1, 1), i--;
                tokens[i] = ' ';
                if (i === tokens.length - 1 || i === 0) tokens.splice(i, 1);
            }
        }
    },

    /**
     * TODO:
     *
     * @param nodes
     */
    _minimize: function(nodes) {
        var i, node;

        for (i = 0; i < nodes.length; i++) {
            node = nodes[i];
            if (node.type === 'ruleset') {
                node.value.forEach(function(value) {
                    $cssm.minTokens(value); // exclude URLs, etc
                });
                node.content.forEach(function(d) {
                    $cssm.minTokens(d.value); // exclude URLs, etc
                    if (d.name.join() === 'font-weight') $cssm.minFontWeight(d.value);
                });
            } else if (node.type === 'atrule' && node.value[1] === 'media') {
                $cssm._minimize(node.nodes);
            }

            if (node.nodes.length) $cssm._minimize(node.nodes);
        }
    },

    /**
     * TODO:
     *
     * @param src
     * @param dst
     */
    copyTree: function(src, dst) {
        var r = dst || [], t,
            charset = false,
            node,
            n = 1,
            i = 0;

        for (; i < src.nodes.length; i++) {
            node = src.nodes[i];
            if (typeof node === 'string') {
                r.push(node);
            } else if ($cssm.isContentNode(node)) {
                if (node.value.length && node.content.length) {
                    t = { type: node.type, value: node.value.slice(), content: [], nodes: [] };
                    node.content.forEach(function(d) {
                        if (typeof d === 'string') t.content.push(d);
                        else if (d.name.length && d.value.length) t.content.push({ name: d.name.slice(), value: d.value.slice(), important: d.important, n: n++ });
                    });
                    r.push(t);
                }
            } else {
                switch (node.value[1]) {
                    case 'charset':
                        if (!charset) charset = true;
                        else break;
                    case 'import':
                    case 'namespace':
                        if (!node.content.length && !node.nodes.length) {
                            r.push({ type: 'atrule', value: node.value.slice(), content: [], nodes: [] });
                        }
                        break;
                    case 'media':
                        if (node.nodes.length) {
                            t = { type: 'atrule', value: node.value.slice(), content: [], nodes: $cssm.copyTree(node) };
                            r.push(t);
                        }
                        break;
                }
            }
        }

        return r;
    },<commit_msg>fix @charset once again<commit_after>/**
 * @namespace CSS минимизатор.
 */
var $cssm = {

    /**
     * TODO:
     *
     * @param s
     */
    isComment: function(s) {
        return s.length > 3 && s.substr(0, 2) === '/*' && s.slice(-2) === '*/';
    },

    /**
     * TODO:
     *
     * @param tree
     */
    minimize: function(tree) {
        var nodes = [];

        $cssm.copyTree(tree, nodes);
        $cssm.processComments(nodes);
        $cssm._minimize(nodes);

        return { nodes: $cssm.restructure(nodes) };
    },

    /**
     * TODO:
     *
     * @param nodes
     */
    processComments: function(nodes) {
        var node, i = 0, j, c, t, mode = { protect: false, order: false };

        for (; i < nodes.length; i++) {
            node = nodes[i];
            if (typeof node === 'string') {
                $cssm.checkMode(node, mode);
                nodes.splice(i, 1);
                i--;
            } else {
                if (node.value.length) {
                    node.value.forEach(function(value) {
                        $cssm.cleanComments(value);
                    });
                }
                if (node.content.length) {
                    for (j = 0; j < node.content.length; j++) {
                        c = node.content[j];
                        if (typeof c === 'string') {
                            $cssm.checkMode(c, mode);
                            if (!mode.protect && c === '/*p*/') {
                                if ((t = node.content[j + 1]) !== undefined && typeof t !== 'string') t.protect = true;
                            } else if (!mode.order && c === '/*o*/') {
                                if ((t = node.content[j + 1]) !== undefined && typeof t !== 'string') t.order = true;
                            }

                            node.content.splice(j, 1);
                            j--;
                        } else {
                            if (mode.protect) c.protect = true;
                            if (!mode.order && !c.order) c.n = c.n - 1000000;
                            $cssm.cleanComments(c.name);
                            $cssm.cleanComments(c.value);
                        }
                    }
                }
                if (node.nodes && node.nodes.length) $cssm.processComments(node.nodes);
            }
        }
    },

    /**
     * TODO:
     *
     * @param s
     * @param mode
     */
    checkMode: function(s, mode) {
        switch (s) {
            case '/*p<*/':
                mode.protect = true;
                break;
            case '/*>p*/':
                mode.protect = false;
                break;
            case '/*o<*/':
                mode.order = true;
                break;
            case '/*>o*/':
                mode.order = false;
                break;
        }
    },

    /**
     * TODO:
     *
     * @param tokens
     */
    cleanComments: function(tokens) {
        for (var i = 0; i < tokens.length; i++) {
            if ($cssm.isComment(tokens[i])) {
                if (tokens[i + 1] === ' ') tokens.splice(i + 1, 1);
                if (tokens[i - 1] === ' ') tokens.splice(i - 1, 1), i--;
                tokens[i] = ' ';
                if (i === tokens.length - 1 || i === 0) tokens.splice(i, 1);
            }
        }
    },

    /**
     * TODO:
     *
     * @param nodes
     */
    _minimize: function(nodes) {
        var i, node;

        for (i = 0; i < nodes.length; i++) {
            node = nodes[i];
            if (node.type === 'ruleset') {
                node.value.forEach(function(value) {
                    $cssm.minTokens(value); // exclude URLs, etc
                });
                node.content.forEach(function(d) {
                    $cssm.minTokens(d.value); // exclude URLs, etc
                    if (d.name.join() === 'font-weight') $cssm.minFontWeight(d.value);
                });
            } else if (node.type === 'atrule' && node.value[1] === 'media') {
                $cssm._minimize(node.nodes);
            }

            if (node.nodes.length) $cssm._minimize(node.nodes);
        }
    },

    /**
     * TODO:
     *
     * @param src
     * @param dst
     */
    copyTree: function(src, dst) {
        var r = dst || [], t,
            skipCharset = false,
            node,
            n = 1,
            i = 0;

        for (; i < src.nodes.length; i++) {
            node = src.nodes[i];
            if (typeof node === 'string') {
                r.push(node);
            } else if ($cssm.isContentNode(node)) {
                if (node.value.length && node.content.length) {
                    t = { type: node.type, value: node.value.slice(), content: [], nodes: [] };
                    node.content.forEach(function(d) {
                        if (typeof d === 'string') t.content.push(d);
                        else if (d.name.length && d.value.length) t.content.push({ name: d.name.slice(), value: d.value.slice(), important: d.important, n: n++ });
                    });
                    r.push(t);
                    skipCharset = true;
                }
            } else {
                switch (node.value[1]) {
                    case 'charset':
                        if (!skipCharset) skipCharset = true;
                        else break;
                    case 'import':
                    case 'namespace':
                        if (!node.content.length && !node.nodes.length) {
                            r.push({ type: 'atrule', value: node.value.slice(), content: [], nodes: [] });
                            skipCharset = true;
                        }
                        break;
                    case 'media':
                        if (node.nodes.length) {
                            t = { type: 'atrule', value: node.value.slice(), content: [], nodes: $cssm.copyTree(node) };
                            r.push(t);
                            skipCharset = true;
                        }
                        break;
                }
            }
        }

        return r;
    },
{'fraction_non_alphanumeric': 0.1525887819449054, 'fraction_numerical': 0.005310321938267508, 'mean_word_length': 0.9840329218106996, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2164181', 'n_tokens_mistral': 3554, 'n_tokens_neox': 3304, 'n_words': 703}