<commit_before>var path = require('path');
var fs = require('fs');
var EOL = require('os').EOL;

var through = require('through2');
var gutil = require('gulp-util');
var glob = require('glob');

module.exports = function(options) {
  options = options || {};

  var startReg = /<!--\s*build:(\w+)(?:\(([^\)]+?)\))?\s+(\/?([^\s]+?))\s*-->/gim;
  var endReg = /<!--\s*endbuild\s*-->/gim;
  var jsReg = /script.+src\s*=\s*['"]([^"']+)['"]/mi;
  var cssReg = /link.+href\s*=\s*['"]([^"']+)['"]/mi;
  var basePath, mainPath, mainName, alternatePath;

  function createFile(name, content) {
    return new gutil.File({
      path: path.join(path.relative(basePath, mainPath), name),
      contents: new Buffer(content)
    });
  }

  function getBlockType(content) {
    return jsReg.test(content) ? 'js' : 'css';
  }

  function getFiles(content, reg) {
    var paths = [];
    var files = [];

    content
      .replace(/<!--(?:(?:.|\r|\n)*?)-->/gim, '')
      .replace(reg, function (a, b) {
        var filePath = path.resolve(path.join(alternatePath || mainPath, b));

        if (options.assetsDir) {
          filePath = path.resolve(path.join(options.assetsDir, path.relative(basePath, filePath)));
        }

        paths.push(filePath);
      });

    for (var i = 0, l = paths.length; i < l; ++i) {
      var filepath = glob.sync(paths[i])[0];
      if (filepath === undefined) {
        throw new gutil.PluginError('gulp-jade-usemin', 'Path ' + paths[i] + ' not found!');
      }
      files.push(new gutil.File({
        path: filepath,
        contents: fs.readFileSync(filepath)
      }));
    }

    return files;
  }

  function concat(files, name) {
    var buffer = [];

    files.forEach(function(file) {
      buffer.push(String(file.contents));
    });

    return createFile(name, buffer.join(EOL));
  }

  function processTask(index, tasks, name, files, callback) {
    var newFiles = [];

    function write(file) {
      newFiles.push(file);
    }

    if (tasks[index] === 'concat') {
      newFiles = [concat(files, name)];
    } else {
      var stream = tasks[index];


      stream.on('data', write);
      files.forEach(function(file) {
        stream.write(file);
      });
      stream.removeListener('data', write);
    }

    if (tasks[++index]) {
      processTask(index, tasks, name, newFiles, callback);
    } else {
      newFiles.forEach(callback);
    }
  }

  function process(name, files, pipelineId, callback) {
    var tasks = options[pipelineId] || [];
    if (tasks.indexOf('concat') === -1) {
      tasks.unshift('concat');
    }

    processTask(0, tasks, name, files, callback);
  }

  function processJade(content, push, callback) {
    var jade = [];
    var sections = content.split(endReg);

    for (var i = 0, l = sections.length; i < l; ++i)
      if (sections[i].match(startReg)) {
        var section = sections[i].split(startReg);
        alternatePath = section[2];

        jade.push(section[0]);

        if (getBlockType(section[5]) === 'js') {
          process(section[4], getFiles(section[5], jsReg), section[1], function(name, file) {
            push(file);
            if (path.extname(file.path) === '.js')
              jade.push('script(src="' + name.replace(path.basename(name), path.basename(file.path)) + '")');
          }.bind(this, section[3]));
        } else {
          process(section[4], getFiles(section[5], cssReg), section[1], function(name, file) {
            push(file);
            jade.push('link(rel="stylesheet", href="' + name.replace(path.basename(name), path.basename(file.path)) + '")');
          }.bind(this, section[3]));
        }
      } else {
        jade.push(sections[i]);
      }

    process(mainName, [createFile(mainName, jade.join(''))], 'jade', function(file) {
      push(file);
      callback();
    });
  }

  return through.obj(function(file, enc, callback) {
    if (file.isNull()) {
      this.push(file); // Do nothing if no contents
      callback();
    } else if (file.isStream()) {
      this.emit('error', new gutil.PluginError('gulp-jade-usemin', 'Streams are not supported!'));
      callback();
    } else {
      basePath = file.base;
      mainPath = path.dirname(file.path);
      mainName = path.basename(file.path);

      processJade(String(file.contents), this.push.bind(this), callback);
    }
  });
};
<commit_msg>Fixed regexp for replacing all occurences of js/css, closes #7<commit_after>var path = require('path');
var fs = require('fs');
var EOL = require('os').EOL;

var through = require('through2');
var gutil = require('gulp-util');
var glob = require('glob');

module.exports = function(options) {
  options = options || {};

  var startReg = /<!--\s*build:(\w+)(?:\(([^\)]+?)\))?\s+(\/?([^\s]+?))\s*-->/gim;
  var endReg = /<!--\s*endbuild\s*-->/gim;
  var jsReg = /script.+src\s*=\s*['"]([^"']+)['"]/gim;
  var cssReg = /link.+href\s*=\s*['"]([^"']+)['"]/gim;
  var basePath, mainPath, mainName, alternatePath;

  function createFile(name, content) {
    return new gutil.File({
      path: path.join(path.relative(basePath, mainPath), name),
      contents: new Buffer(content)
    });
  }

  function getBlockType(content) {
    return jsReg.test(content) ? 'js' : 'css';
  }

  function getFiles(content, reg) {
    var paths = [];
    var files = [];
    content
      .replace(/<!--(?:(?:.|\r|\n)*?)-->/gim, '')
      .replace(reg, function (a, b) {
        var filePath = path.resolve(path.join(alternatePath || mainPath, b));

        if (options.assetsDir) {
          filePath = path.resolve(path.join(options.assetsDir, path.relative(basePath, filePath)));
        }

        paths.push(filePath);
      });

    for (var i = 0, l = paths.length; i < l; ++i) {
      var filepath = glob.sync(paths[i])[0];
      if (filepath === undefined) {
        throw new gutil.PluginError('gulp-jade-usemin', 'Path ' + paths[i] + ' not found!');
      }
      files.push(new gutil.File({
        path: filepath,
        contents: fs.readFileSync(filepath)
      }));
    }

    return files;
  }

  function concat(files, name) {
    var buffer = [];

    files.forEach(function(file) {
      buffer.push(String(file.contents));
    });

    return createFile(name, buffer.join(EOL));
  }

  function processTask(index, tasks, name, files, callback) {
    var newFiles = [];

    function write(file) {
      newFiles.push(file);
    }

    if (tasks[index] === 'concat') {
      newFiles = [concat(files, name)];
    } else {
      var stream = tasks[index];


      stream.on('data', write);
      files.forEach(function(file) {
        stream.write(file);
      });
      stream.removeListener('data', write);
    }

    if (tasks[++index]) {
      processTask(index, tasks, name, newFiles, callback);
    } else {
      newFiles.forEach(callback);
    }
  }

  function process(name, files, pipelineId, callback) {
    var tasks = options[pipelineId] || [];
    if (tasks.indexOf('concat') === -1) {
      tasks.unshift('concat');
    }

    processTask(0, tasks, name, files, callback);
  }

  function processJade(content, push, callback) {
    var jade = [];
    var sections = content.split(endReg);

    for (var i = 0, l = sections.length; i < l; ++i)
      if (sections[i].match(startReg)) {
        var section = sections[i].split(startReg);
        alternatePath = section[2];
        jade.push(section[0]);

        if (getBlockType(section[5]) === 'js') {
          process(section[4], getFiles(section[5], jsReg), section[1], function(name, file) {
            push(file);
            if (path.extname(file.path) === '.js')
              jade.push('script(src="' + name.replace(path.basename(name), path.basename(file.path)) + '")');
          }.bind(this, section[3]));
        } else {
          process(section[4], getFiles(section[5], cssReg), section[1], function(name, file) {
            push(file);
            jade.push('link(rel="stylesheet", href="' + name.replace(path.basename(name), path.basename(file.path)) + '")');
          }.bind(this, section[3]));
        }
      } else {
        jade.push(sections[i]);
      }

    process(mainName, [createFile(mainName, jade.join(''))], 'jade', function(file) {
      push(file);
      callback();
    });
  }

  return through.obj(function(file, enc, callback) {
    if (file.isNull()) {
      this.push(file); // Do nothing if no contents
      callback();
    } else if (file.isStream()) {
      this.emit('error', new gutil.PluginError('gulp-jade-usemin', 'Streams are not supported!'));
      callback();
    } else {
      basePath = file.base;
      mainPath = path.dirname(file.path);
      mainName = path.basename(file.path);

      processJade(String(file.contents), this.push.bind(this), callback);
    }
  });
};

{'fraction_non_alphanumeric': 0.19313893653516295, 'fraction_numerical': 0.004002287021154946, 'mean_word_length': 2.71380042462845, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3118658', 'n_tokens_mistral': 2950, 'n_tokens_neox': 2876, 'n_words': 563}