<commit_before>var async = require('async'),
  _ = require('underscore'),
  extend = require('./extend'),
  processor = extend.processor.list(),
  util = require('./util'),
  file = util.file,
  sourceDir = hexo.source_dir;

module.exports = function(files, callback){
  if (!_.isArray(files)) files = [files];

  hexo.emit('processBefore');

  async.forEach(files, function(item, next){
    file.read(sourceDir + item, function(err, content){
      if (err) throw new Error('Failed to read file: ' + sourceDir + item);

      var tasks = [],
        params;

      processor.forEach(function(proc){
        var rule = proc.rule;

        if (_.isRegExp(rule)){
          var exec = rule.exec(item);
          if (exec){
            params = exec;
            tasks.push(proc);
          }
        } else {
          if (rule === ''){
            tasks.push(proc);
          } else {
            var arr = [];
            rule = rule.replace('/', '\\/').replace(/:(\w+)/g, function(match, arg){
              arr.push(arg);
              return '([^\\/]+)';
            });
            var regex = new RegExp(rule),
              exec = regex.exec(item);

            for (var i=0, len=arr.length; i<len; i++){
              params[arr[i]] = exec[i + 1];
            }
          }
        }
      });

      async.forEach(tasks, function(task, next){
        var obj = {
          source: sourceDir + item,
          path: item,
          content: content
        };

        if (params) obj.params = params;

        task(obj, next);
      }, next);
    });
  }, function(){
    hexo.emit('processAfter');
    callback();
  });
};<commit_msg>Processor: read file after confirming tasks<commit_after>var async = require('async'),
  _ = require('underscore'),
  extend = require('./extend'),
  processor = extend.processor.list(),
  util = require('./util'),
  file = util.file,
  sourceDir = hexo.source_dir;

module.exports = function(files, callback){
  if (!_.isArray(files)) files = [files];

  hexo.emit('processBefore');

  async.forEach(files, function(item, next){
    var tasks = [];

    processor.forEach(function(item){
      var rule = item.rule,
        match = false,
        params;

      if (_.isRegExp(rule)){
        var exec = rule.exec(item);
        if (exec){
          params = exec;
          match = true;
        }
      } else {
        if (rule === ''){
          match = true;
        } else {
          var arr = [];
          rule = rule.replace('/', '\\/').replace(/:(\w+)/g, function(match, arg){
            arr.push(arg);
            return '([^\\/]+)';
          });
          var regex = new RegExp(rule),
            exec = regex.exec(item);

          if (exec){
            match = true;
            params = [];
            for (var i=0, len=arr.length; i<len; i++){
              params[arr[i]] = exec[i + 1];
            }
          }
        }
      }

      if (match){
        item.params = params;
        tasks.push(item);
      }
    });

    if (!tasks.length) return next();

    file.read(sourceDir + item, function(err, content){
      if (err) throw new Error('Failed to read file: ' + sourceDir + item);

      async.forEach(tasks, function(task, next){
        var obj = {
          source: sourceDir + item,
          path: item,
          content: content,
          params: task.params
        };

        task(obj, next);
      }, next);
    });
  }, function(){
    hexo.emit('processAfter');
    callback();
  });
};
{'fraction_non_alphanumeric': 0.16695402298850576, 'fraction_numerical': 0.0011494252873563218, 'mean_word_length': 1.7280564263322884, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5236357', 'n_tokens_mistral': 1043, 'n_tokens_neox': 1018, 'n_words': 235}