<commit_before>/*
 * grunt-demux-file
 * https://github.com/yongtw123/grunt-demux-file
 *
 * Copyright (c) 2015 <NAME>
 * Licensed under the MIT license.
 */
'use strict';

module.exports = function(grunt) {

    // Please see the Grunt documentation for more information regarding task
    // creation: http://gruntjs.com/creating-tasks

    grunt.registerMultiTask('demux_file', 'Spawns multiple files from delimiter-separated values in source', function() {

        var _ = require('underscore'),
            chalk = require('chalk'),
            regexesc = require('escape-string-regexp');

        // Merge task-specific and/or target-specific options with these defaults.
        var options = this.options({
            prefixes: undefined, //an array is preferred, or a number designating length
            delimiters: ['[[[', ']]]'],
            separator: ','
        });

        
        // Insist on supplying prefixes
        if (_.isNumber(options.prefixes)) {
            grunt.log.writeln(chalk.yellow('No prefix supplied. Default to numbers.'));
            options.prefixes = _.range(parseInt(options.prefixes, 10)).map(function(v) {
                return v.toString();
            });
        } else if (_.isUndefined(options.prefixes)) {
            grunt.warn(chalk.red('Prefixes must be supplied!'));
        }

        
        // Setup regular expressions
        if (!_.isArray(options.delimiters) || options.delimiters.length !== 2) {
            grunt.warn(chalk.red('Delimiters must be an array of 2 strings.'));
        }
        options.delimiters = options.delimiters.map(function(delim) {
            return regexesc(delim);
        });
        options.delimiters.splice(1, 0, '(.*)');
        var delim = new RegExp(options.delimiters.join(''), 'g');


        // Iterate over all specified file groups.
        this.files.forEach(function(f) {
            var isExpanded = f.orig.expand || false;
            f.src.filter(function(filepath) {
                // Filter for existing files
                if (!grunt.file.exists(filepath)) {
                    grunt.log.warn('Source file "' + filepath + '" not found. Ignore.');
                    return false;
                }
                return true;<commit_msg>capturing group now captures lazily (in case multiple delim pairs on same line)<commit_after>/*
 * grunt-demux-file
 * https://github.com/yongtw123/grunt-demux-file
 *
 * Copyright (c) 2015 <NAME>
 * Licensed under the MIT license.
 */
'use strict';

module.exports = function(grunt) {

    // Please see the Grunt documentation for more information regarding task
    // creation: http://gruntjs.com/creating-tasks

    grunt.registerMultiTask('demux_file', 'Spawns multiple files from delimiter-separated values in source', function() {

        var _ = require('underscore'),
            chalk = require('chalk'),
            regexesc = require('escape-string-regexp');

        // Merge task-specific and/or target-specific options with these defaults.
        var options = this.options({
            prefixes: undefined, //an array is preferred, or a number designating length
            delimiters: ['[[[', ']]]'],
            separator: ','
        });

        
        // Insist on supplying prefixes
        if (_.isNumber(options.prefixes)) {
            grunt.log.writeln(chalk.yellow('No prefix supplied. Default to numbers.'));
            options.prefixes = _.range(parseInt(options.prefixes, 10)).map(function(v) {
                return v.toString();
            });
        } else if (_.isUndefined(options.prefixes)) {
            grunt.warn(chalk.red('Prefixes must be supplied!'));
        }

        
        // Setup regular expressions
        if (!_.isArray(options.delimiters) || options.delimiters.length !== 2) {
            grunt.warn(chalk.red('Delimiters must be an array of 2 strings.'));
        }
        options.delimiters = options.delimiters.map(function(delim) {
            return regexesc(delim);
        });
        options.delimiters.splice(1, 0, '(.*?)');
        var delim = new RegExp(options.delimiters.join(''), 'g');


        // Iterate over all specified file groups.
        this.files.forEach(function(f) {
            var isExpanded = f.orig.expand || false;
            f.src.filter(function(filepath) {
                // Filter for existing files
                if (!grunt.file.exists(filepath)) {
                    grunt.log.warn('Source file "' + filepath + '" not found. Ignore.');
                    return false;
                }
                return true;
{'fraction_non_alphanumeric': 0.12846237731733914, 'fraction_numerical': 0.0056706652126499455, 'mean_word_length': 2.071667782987274, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6917749', 'n_tokens_mistral': 1313, 'n_tokens_neox': 1246, 'n_words': 303}