<commit_before>var path = require('path');
var fs = require('fs');
var EOL = require('os').EOL;

var through = require('through2');
var gutil = require('gulp-util');
var glob = require('glob');
var minimatch = require('minimatch');
var when = require('when');
var es = require('event-stream');
var Readable = require('stream').Readable;
var _ = require('lodash');

module.exports = function (options) {
  options = options || {};
  var startReg = /<!--\s*build:(\w+)(?:\(([^\)]+?)\))?\s+(\/?([^\s]+?))?\s*-->/gim;
  var endReg = /<!--\s*endbuild\s*-->/gim;
  var jsReg = /<\s*script\s+.*?src\s*=\s*("|')((?:[^\1]|\\\1)+?)\1.*?><\s*\/\s*script\s*>/gi;
  var cssReg = /<\s*link\s+.*?href\s*=\s*("|')((?:[^\1]|\\\1)+?)\1.*?>/gi;
  var startCondReg = /<!--\[[^\]]+\]>/gim;
  var endCondReg = /<!\[endif\]-->/gim;
  var basePath, mainPath, mainName, alternatePath;


  function getPath(name) {
    var filePath = path.join(path.relative(basePath, mainPath), name);
    var isStatic = name.split('.').pop() === 'js' || name.split('.').pop() === 'css';

    if (options.outputRelativePath && isStatic)
      filePath = options.outputRelativePath + name;
    return filePath;
  }

  function createFile(name, content) {
    return new gutil.File({
      path: getPath(name),
      contents: new Buffer(content)
    })
  }

  function getBlockType(content) {
    return !cssReg.test(content) ? 'js' : 'css';
  }

  function readStream(streamCtor, callback) {
    var files = [];
    var deferred = when.defer();

    var stream = streamCtor();
    stream.on('data', function (file) {
      if (file.isStream()) {
        this.emit('error', gutil.PluginError('gulp-usemin', 'Streams in assets are not supported!'));
      }

      if (file.isBuffer()) {
        file.base = path.resolve(file.base);
        file.path = path.resolve(file.path);
        files.push(file);
      }
    });

    stream.on('end', function () {
      if (options.debugStreamFiles) {
        console.log('asssets:\n', files.map(function (f) {
          return f.base + ' :: ' + f.path;
        }).join('\n '));
      }
      deferred.resolve(files);
    });

    return deferred.promise;
  }

  function produceMatcher(fileArray) {
    var allFiles = fileArray.map(function (file) {
      return file.path;
    });

    var filesByPath = fileArray.reduce(function (obj, file) {
      obj[file.path] = file;
      return obj;
    }, {});
<commit_msg>pass null files into asset pipe<commit_after>var path = require('path');
var fs = require('fs');
var EOL = require('os').EOL;

var through = require('through2');
var gutil = require('gulp-util');
var glob = require('glob');
var minimatch = require('minimatch');
var when = require('when');
var es = require('event-stream');
var Readable = require('stream').Readable;
var _ = require('lodash');

module.exports = function (options) {
  options = options || {};
  var startReg = /<!--\s*build:(\w+)(?:\(([^\)]+?)\))?\s+(\/?([^\s]+?))?\s*-->/gim;
  var endReg = /<!--\s*endbuild\s*-->/gim;
  var jsReg = /<\s*script\s+.*?src\s*=\s*("|')((?:[^\1]|\\\1)+?)\1.*?><\s*\/\s*script\s*>/gi;
  var cssReg = /<\s*link\s+.*?href\s*=\s*("|')((?:[^\1]|\\\1)+?)\1.*?>/gi;
  var startCondReg = /<!--\[[^\]]+\]>/gim;
  var endCondReg = /<!\[endif\]-->/gim;
  var basePath, mainPath, mainName, alternatePath;


  function getPath(name) {
    var filePath = path.join(path.relative(basePath, mainPath), name);
    var isStatic = name.split('.').pop() === 'js' || name.split('.').pop() === 'css';

    if (options.outputRelativePath && isStatic)
      filePath = options.outputRelativePath + name;
    return filePath;
  }

  function createFile(name, content) {
    return new gutil.File({
      path: getPath(name),
      contents: new Buffer(content)
    })
  }

  function getBlockType(content) {
    return !cssReg.test(content) ? 'js' : 'css';
  }

  function readStream(streamCtor, callback) {
    var files = [];
    var deferred = when.defer();

    var stream = streamCtor();
    stream.on('data', function (file) {
      if (file.isStream()) {
        this.emit('error', gutil.PluginError('gulp-usemin', 'Streams in assets are not supported!'));
      }

      file.base = path.resolve(file.base);
      file.path = path.resolve(file.path);
      files.push(file);
    });

    stream.on('end', function () {
      if (options.debugStreamFiles) {
        console.log('asssets:\n', files.map(function (f) {
          return f.base + ' :: ' + f.path;
        }).join('\n '));
      }
      deferred.resolve(files);
    });

    return deferred.promise;
  }

  function produceMatcher(fileArray) {
    var allFiles = fileArray.map(function (file) {
      return file.path;
    });

    var filesByPath = fileArray.reduce(function (obj, file) {
      obj[file.path] = file;
      return obj;
    }, {});

{'fraction_non_alphanumeric': 0.22206035379812694, 'fraction_numerical': 0.0029136316337148803, 'mean_word_length': 3.0016652789342215, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2778812', 'n_tokens_mistral': 1707, 'n_tokens_neox': 1664, 'n_words': 323}