<commit_before>var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var yaml = require('js-yaml');
var async = require('async');
var gaze = require('gaze');

module.exports = function (acetate, next){
  acetate.log.verbose('extension', 'adding data to pages');

  var cache = {};

  // watch everything in the source folder that looks like a data file
  gaze(path.join(acetate.src, '**/*.+(js|json|yaml|yml)'), function(error, watcher){
    watcher.on('all', function(event, filepath){
      // get the path of the data file relative to the source folder
      relativepath = filepath.replace(path.join(process.cwd(),acetate.src) + path.sep, '');

      acetate.log.info('data', '%s changed', relativepath);

      delete cache[relativepath];

      // find all pages with this data file
      var pages = _.filter(acetate.pages, function(page){
        return  page.data && _.contains(page.data, relativepath);
      });

      // rebuild each of those pages
      if(pages.length){
        acetate.log.info('data', 'rebuilding %d pages useing %s', pages.length, relativepath);
        for (var i = pages.length - 1; i >= 0; i--) {
          var dirtyPage = _.remove(acetate.pages, {template: pages[i].template})[0];

          if(dirtyPage){
            dirtyPage.clean();
          }

          acetate.loadPage(dirtyPage.template, function(error, page){
            acetate.pages.push(page);
          });
        };
        acetate.build();
      }
    });
  });

  function cacheResult(key, callback){
    return function(error, data){
      cache[key] = data;
      callback(error, data);
    }
  }

  function loadData(filepath, callback) {
    var ext = path.extname(filepath);

    if(cache[filepath]){
      acetate.log.debug('data', '%s loaded from cache', filepath);
      callback(null, cache[filepath])
      return;
    }

    if(ext === '.js') {
      parseModuleData(filepath, cacheResult(filepath, callback));
      return;
    }

    if(ext === '.json') {
      parseJsonData(filepath, cacheResult(filepath, callback));
      return;
    }

    if(ext === '.yaml' || ext === '.yml') {
      parseYamlData(filepath, cacheResult(filepath, callback));
      return;
    }

    // callback with an error
    callback('Data file not found');
  };

  function parseYamlData(filepath, callback){
    var fullpath = path.join(acetate.root, acetate.src, filepath)
    var data = {};

    fs.readFile(fullpath, function(error, content){
      if(error){
        acetate.log.warn('data', 'error reading %s - %s', filepath, e);
        acetate.buildStatus('warn');
        callback(null, {});
        return null;
      }

      try {
        data = yaml.safeLoad(content.toString(), {
          strict: true
        });
      } catch (e) {
        acetate.log.warn('data', 'invalid YAML in %s', filepath);
        acetate.buildStatus('warn');
      }

      callback(null, data);
    });
  };

  function parseModuleData(filepath, callback){
    var fullpath = path.join(acetate.root, acetate.src, filepath)
    var data = {};

    if(require.cache[fullpath]){
      delete require.cache[fullpath];
    }

    try{
      require(fullpath)(function(error, result){
        data = result;
        if(error){
          acetate.log.error('data', 'error loading %s - %s', filepath, error);
          acetate.buildStatus('error');
        }
      });
    } catch (e) {
      acetate.log.error('data', 'error thrown loading %s - %s', filepath, e.message || e);
      acetate.log.stacktrace(e.stack);
      acetate.buildStatus('error');
    }
    callback(null, data);
  };

  function parseJsonData(filepath, callback){
    var fullpath = path.join(acetate.root, acetate.src, filepath)
    var data = {};

    fs.readFile(fullpath, function(error, content){
      if(error){
        acetate.log.warn('data', 'error reading %s - %s', filepath, e);
        acetate.buildStatus('warn');
        callback(null, {});
        return null;
      }

      try {
        data = JSON.parse(content.toString());
      } catch (e) {
        acetate.log.warn('data', 'invalid JSON in %s', filepath);
        acetate.buildStatus('warn');
      }

      callback(null, data);
    });
  };

  async.each(acetate.pages, function(page, loaded){
    if(page.data){
      async.parallel(_.mapValues(page.data , function(filepath) { return _.partial(loadData, filepath) }), function(error, pageData){
        _.merge(page, pageData);
        loaded();
      });
    } else {
      loaded();
    }
  }, next);

};<commit_msg>fix data extension<commit_after>var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var yaml = require('js-yaml');
var async = require('async');
var chokidar = require('chokidar');

module.exports = function (acetate, next){
  acetate.log.verbose('extension', 'adding data to pages');

  var cache = {};

  function handler(filepath) {
    // get the path of the data file relative to the source folder
    relativepath = filepath.replace(path.join(process.cwd(),acetate.src) + path.sep, '');

    acetate.log.info('data', '%s changed', relativepath);

    delete cache[relativepath];

    // find all pages with this data file
    var pages = _.filter(acetate.pages, function(page){
      return  page.data && _.contains(page.data, relativepath);
    });

    // rebuild each of those pages
    if(pages.length){
      acetate.log.info('data', 'rebuilding %d pages useing %s', pages.length, relativepath);
      for (var i = pages.length - 1; i >= 0; i--) {
        var dirtyPage = _.remove(acetate.pages, {src: pages[i].src})[0];

        if(dirtyPage){
          dirtyPage._clean();

          acetate.loadPage(path.join(acetate.root, acetate.src,dirtyPage.src), function(error, page){
            acetate.pages.push(page);
          });
        }
      };
      acetate.build();
    }
  }

  // watch everything in the source folder that looks like a data file
  var watcher = chokidar.watch(path.join(acetate.root, acetate.src,  '**/*.+(js|json|yaml|yml)'), {
    ignoreInitial: true
  })
  .on('change', handler)
  .on('add', handler)
  .on('unlink', handler);

  function cacheResult(key, callback){
    return function(error, data){
      cache[key] = data;
      callback(error, data);
    }
  }

  function loadData(filepath, callback) {
    var ext = path.extname(filepath);

    if(cache[filepath]){
      acetate.log.debug('data', '%s loaded from cache', filepath);
      callback(null, cache[filepath])
      return;
    }

    if(ext === '.js') {
      parseModuleData(filepath, cacheResult(filepath, callback));
      return;
    }

    if(ext === '.json') {
      parseJsonData(filepath, cacheResult(filepath, callback));
      return;
    }

    if(ext === '.yaml' || ext === '.yml') {
      parseYamlData(filepath, cacheResult(filepath, callback));
      return;
    }

    // callback with an error
    callback('Data file not found');
  };

  function parseYamlData(filepath, callback){
    var fullpath = path.join(acetate.root, acetate.src, filepath)
    var data = {};

    fs.readFile(fullpath, function(error, content){
      if(error){
        acetate.log.warn('data', 'error reading %s - %s', filepath, e);
        acetate.buildStatus('warn');
        callback(null, {});
        return null;
      }

      try {
        data = yaml.safeLoad(content.toString(), {
          strict: true
        });
      } catch (e) {
        acetate.log.warn('data', 'invalid YAML in %s', filepath);
        acetate.buildStatus('warn');
      }

      callback(null, data);
    });
  };

  function parseModuleData(filepath, callback){
    var fullpath = path.join(acetate.root, acetate.src, filepath)
    var data = {};

    if(require.cache[fullpath]){
      delete require.cache[fullpath];
    }

    try{
      require(fullpath)(function(error, result){
        data = result;
        if(error){
          acetate.log.error('data', 'error loading %s - %s', filepath, error);
          acetate.buildStatus('error');
        }
      });
    } catch (e) {
      acetate.log.error('data', 'error thrown loading %s - %s', filepath, e.message || e);
      acetate.log.stacktrace(e.stack);
      acetate.buildStatus('error');
    }
    callback(null, data);
  };

  function parseJsonData(filepath, callback){
    var fullpath = path.join(acetate.root, acetate.src, filepath)
    var data = {};

    fs.readFile(fullpath, function(error, content){
      if(error){
        acetate.log.warn('data', 'error reading %s - %s', filepath, e);
        acetate.buildStatus('warn');
        callback(null, {});
        return null;
      }

      try {
        data = JSON.parse(content.toString());
      } catch (e) {
        acetate.log.warn('data', 'invalid JSON in %s', filepath);
        acetate.buildStatus('warn');
      }

      callback(null, data);
    });
  };

  async.each(acetate.pages, function(page, loaded){
    if(page.data){
      async.parallel(_.mapValues(page.data , function(filepath) { return _.partial(loadData, filepath) }), function(error, pageData){
        _.merge(page, pageData);
        loaded();
      });
    } else {
      loaded();
    }
  }, next);

};
{'fraction_non_alphanumeric': 0.1585792349726776, 'fraction_numerical': 0.0006557377049180328, 'mean_word_length': 2.6084384858044163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3357032', 'n_tokens_mistral': 2956, 'n_tokens_neox': 2753, 'n_words': 631}