<commit_before>var path = require('path'),
    _ = require('underscore'),
    async = require('async'),
    jsonfile = require('jsonfile'),
    MD5 = require('md5'),
    minimatch = require('minimatch');
jsonfile.spaces = 4;

var defaults = {
  'verbose': false,
  'updatedFile': '.updated.json',
  'ignoreKeys': ["draft"]
}

function processConfig(config, src) {
  config = config || {};
  config = _.extend(_.clone(defaults), config);
  if (src) {
    config.updatedFile = path.join(src, config.updatedFile);
  }
  return config;
}

function removeFiles(files, config) {
  if (files[config.updatedFile]) {
    delete(files[config.updatedFile]);
  }
}

function updated(config) {

  return function(files, metalsmith, done) {
    
    config = processConfig(config);
    
    var metadata = metalsmith.metadata();
    var check_time = metadata.date || new Date();
    
    var updatedFiles = files;
    if (config.filePatterns && config.filePatterns.length > 0) {
      updatedFiles = _.pick(files, function(file, filename) {
        return _.some(config.filePatterns, function (pattern) {
          return minimatch(filename, pattern);
        });
      });
    }
    if (_.keys(updatedFiles).length === 0) {
      if (config.verbose) {
        console.log("No files to compute updates for.");
      }
      removeFiles();
      done();
    }

    updates = {};
    try {
      updates = JSON.parse(files[config.updatedFile].contents);
    } catch (err) {
      if (config.verbose) {
        console.log("Missing " + config.updatedFile);
      }
    }

    async.forEachOf(updatedFiles, function (file, filename, finished) {
      if (filename == config.updatedFile) {
        finished();
        return;
      }<commit_msg>fixes call to removeFiles with no arguments<commit_after>var path = require('path'),
    _ = require('underscore'),
    async = require('async'),
    jsonfile = require('jsonfile'),
    MD5 = require('md5'),
    minimatch = require('minimatch');
jsonfile.spaces = 4;

var defaults = {
  'verbose': false,
  'updatedFile': '.updated.json',
  'ignoreKeys': ["draft"]
}

function processConfig(config, src) {
  config = config || {};
  config = _.extend(_.clone(defaults), config);
  if (src) {
    config.updatedFile = path.join(src, config.updatedFile);
  }
  return config;
}

function removeFiles(files, config) {
  if (files[config.updatedFile]) {
    delete(files[config.updatedFile]);
  }
}

function updated(config) {

  return function(files, metalsmith, done) {
    
    config = processConfig(config);
    
    var metadata = metalsmith.metadata();
    var check_time = metadata.date || new Date();
    
    var updatedFiles = files;
    if (config.filePatterns && config.filePatterns.length > 0) {
      updatedFiles = _.pick(files, function(file, filename) {
        return _.some(config.filePatterns, function (pattern) {
          return minimatch(filename, pattern);
        });
      });
    }
    if (_.keys(updatedFiles).length === 0) {
      if (config.verbose) {
        console.log("No files to compute updates for.");
      }
      removeFiles(files, config);
      done();
    }

    updates = {};
    try {
      updates = JSON.parse(files[config.updatedFile].contents);
    } catch (err) {
      if (config.verbose) {
        console.log("Missing " + config.updatedFile);
      }
    }

    async.forEachOf(updatedFiles, function (file, filename, finished) {
      if (filename == config.updatedFile) {
        finished();
        return;
      }
{'fraction_non_alphanumeric': 0.14678111587982834, 'fraction_numerical': 0.002861230329041488, 'mean_word_length': 2.8672566371681416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7076166', 'n_tokens_mistral': 1059, 'n_tokens_neox': 1041, 'n_words': 227}