<commit_before>/*
 * grunt-glob-html
 * https://github.com/gevgeny/grunt-glob-html
 *
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

module.exports = function (grunt) {
    var cheerio = require('cheerio'),
        path = require('path');

    var isDir = function (path) {
        return grunt.util._.endsWith(path, '/');
    };

    /**
     * Expand element with glob to matched values.
     * @param {object} $element Element with glob.
     * @param {string} attr Resource attr. (src, href).
     * @return {Array<object>} matched values.
     * */
    var expandElement = function ($element, attr) {
        var path = $element.attr(attr),
            expandedPaths = grunt.file.expand(path);

        grunt.log.writeln('Expand ' + path.cyan + ' to ' + expandedPaths.length.toString().cyan + ' files');

        return expandedPaths.map(function (path) {
            return $element.clone().attr(attr, path).toString();
        });
    };

    /**
     * Checks whether element should be handled and return appropriate attr in accordance with element type:
     * link - href, script - src.
     * @param {object} $element element to handle.
     * @return {string} attr to handle. ('src', 'href', undefined)
     * */
    var getAttrToHandle = function ($element) {
        var attr;

        if ($element.is('script')) {
            attr = 'src';
        } else if ($element.is('link')) {
            attr = 'href';
        }

        if (($element.attr(attr) || '').indexOf('*') !== -1) {
            return attr;
        }
    };

    /**
     * Fetches element's indent.
     * @param {object} $element element to handle.
     * @return {string} indent;
     * */
    var fetchIndent = function ($element) {
        var indent = '';

        // Check whether prev element is whitespace.
        if (/^\s+$/.test($element[0].prev.data)) {
            indent = $element[0].prev.data;
            console.log('space1', grunt.util._.map(indent, function (item) { return item.charCodeAt(0); }));
            // Cut line break.
            indent = grunt.util._.last(indent.split(/\n|\r/));
            console.log('space2', grunt.util._.map(indent, function (item) { return item.charCodeAt(0); }));
        }

        return '\n' + indent;
    };

    /**
     * */
    var processFile = function (content) {
        var $ = cheerio.load(content),
            $elements = $('script,link'),
            noElementsFound = true;

        $elements.each(function (i, element) {
            var $element = $(element), indent,
                attr = getAttrToHandle($element);

            if (!attr) {
                return;
            }
            noElementsFound = false;
            indent = fetchIndent($element);

            // Expand initial script to matched ones.
            expandElement($element, attr).forEach(function (element, i) {
                if (i === 0) {
                    $element.before(element);
                } else {
                    $element.before(indent + element);
                }
            });

            // Remove the handled element with its indent.
//            if (indent) {
//                $element[0].prev.data = '';
//            }
            $element.remove();
        });

        if (noElementsFound) {
            grunt.log.writeln('No scripts with wildcard found');
            return content;
        }

        return $.html();
    };


    grunt.registerMultiTask('globhtml', 'Add globbing to your HTML', function () {
        var gruntDir = process.cwd(),
            dest = this.data.dest;

        if (!this.filesSrc.length) {
            grunt.log.error('No sources found');
            return;
        }
        if (!dest) {
            grunt.log.error('Destination is not specified');
            return;
        }
        this.filesSrc.forEach(function (filePath) {
            var content = grunt.file.read(filePath), processedContent, resultDest;

            grunt.log.writeln('Processing ' + filePath.cyan);

            // Set processing file dir as cwd because script tag source related to the HTML and not to the Grunt file.
            grunt.file.setBase(path.dirname(filePath));
            
            // Expand scripts with wildcards.
            processedContent = processFile(content);
            
            console.log(processedContent);
            // Reset base dir to default.
            grunt.file.setBase(gruntDir);

            // Save processed content.
            resultDest = isDir(dest) ? path.join(dest, path.basename(filePath)) : dest;
            grunt.file.write(resultDest, processedContent);
            grunt.log.writeln('Created ' + resultDest.cyan + '\n');
        });
    });
};<commit_msg>remove debug logs<commit_after>/*
 * grunt-glob-html
 * https://github.com/gevgeny/grunt-glob-html
 *
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

module.exports = function (grunt) {
    var cheerio = require('cheerio'),
        path = require('path');

    var isDir = function (path) {
        return grunt.util._.endsWith(path, '/');
    };

    /**
     * Expand element with glob to matched values.
     * @param {object} $element Element with glob.
     * @param {string} attr Resource attr. (src, href).
     * @return {Array<object>} matched values.
     * */
    var expandElement = function ($element, attr) {
        var path = $element.attr(attr),
            expandedPaths = grunt.file.expand(path);

        grunt.log.writeln('Expand ' + path.cyan + ' to ' + expandedPaths.length.toString().cyan + ' files');

        return expandedPaths.map(function (path) {
            return $element.clone().attr(attr, path).toString();
        });
    };

    /**
     * Checks whether element should be handled and return appropriate attr in accordance with element type:
     * link - href, script - src.
     * @param {object} $element element to handle.
     * @return {string} attr to handle. ('src', 'href', undefined)
     * */
    var getAttrToHandle = function ($element) {
        var attr;

        if ($element.is('script')) {
            attr = 'src';
        } else if ($element.is('link')) {
            attr = 'href';
        }

        if (($element.attr(attr) || '').indexOf('*') !== -1) {
            return attr;
        }
    };

    /**
     * Fetches element's indent.
     * @param {object} $element element to handle.
     * @return {string} indent;
     * */
    var fetchIndent = function ($element) {
        var indent = '';

        // Check whether prev element is whitespace.
        if (/^\s+$/.test($element[0].prev.data)) {
            indent = $element[0].prev.data;

            // Cut line break.
            indent = grunt.util._.last(indent.split(/\n|\r/));
        }

        return '\n' + indent;
    };

    /**
     * */
    var processFile = function (content) {
        var $ = cheerio.load(content),
            $elements = $('script,link'),
            noElementsFound = true;

        $elements.each(function (i, element) {
            var $element = $(element), indent,
                attr = getAttrToHandle($element);

            if (!attr) {
                return;
            }
            noElementsFound = false;
            indent = fetchIndent($element);

            // Expand initial script to matched ones.
            expandElement($element, attr).forEach(function (element, i) {
                if (i === 0) {
                    $element.before(element);
                } else {
                    $element.before(indent + element);
                }
            });

            // Remove the handled element with its indent.
//            if (indent) {
//                $element[0].prev.data = '';
//            }
            $element.remove();
        });

        if (noElementsFound) {
            grunt.log.writeln('No scripts with wildcard found');
            return content;
        }

        return $.html();
    };


    grunt.registerMultiTask('globhtml', 'Add globbing to your HTML', function () {
        var gruntDir = process.cwd(),
            dest = this.data.dest;

        if (!this.filesSrc.length) {
            grunt.log.error('No sources found');
            return;
        }
        if (!dest) {
            grunt.log.error('Destination is not specified');
            return;
        }
        this.filesSrc.forEach(function (filePath) {
            var content = grunt.file.read(filePath), processedContent, resultDest;

            grunt.log.writeln('Processing ' + filePath.cyan);

            // Set processing file dir as cwd because script tag source related to the HTML and not to the Grunt file.
            grunt.file.setBase(path.dirname(filePath));
            
            // Expand scripts with wildcards.
            processedContent = processFile(content);

            // Reset base dir to default.
            grunt.file.setBase(gruntDir);

            // Save processed content.
            resultDest = isDir(dest) ? path.join(dest, path.basename(filePath)) : dest;
            grunt.file.write(resultDest, processedContent);
            grunt.log.writeln('Created ' + resultDest.cyan + '\n');
        });
    });
};
{'fraction_non_alphanumeric': 0.13799088739422868, 'fraction_numerical': 0.002386634844868735, 'mean_word_length': 1.740487514863258, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5587481', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2511, 'n_words': 631}