<commit_before>'use strict';

const fs = require('fs');
const Path = require('path');

const Yaml = require('js-yaml');
const _ = require('lodash');
const globby = require('globby');

const Script = require('./script');

class Models {
  constructor(models, options, storage) {
    this.models = models;
    this.storage = storage;
    this.scriptsDir = options.scriptsDir;
    this.databases = options.databases;
    this.engines = {};
    this.instances = {};
    this.plugins = {};
    this.scripts = {};
    this.pluginFields = {};
    this.preprocessors = {};
    this.postprocessors = {};

    this.ready = new Promise(resolve => {
      globby([Path.join(__dirname, '../engines/**/*.js')]).then(files => {
        files.forEach(file => {
          const load = require;
          const engine = load(file);
          this.engines[engine.name] = engine;
        });
        Object.keys(models).forEach(name => {
          const databaseName = this.models[name].database;
          if (typeof this.databases[databaseName] === 'undefined') {
            throw new Error('Unknown database ' + databaseName + ' in model ' + name);
          }
          const database = this.databases[databaseName];
          const engine = database.engine;
          if (typeof this.engines[engine] === 'undefined') {
            throw new Error('Unknown engine ' + engine + ' in database ' + databaseName);
          }
          try {
            this.instances[name] = new this.engines[engine](this.models[name], database, this.databases.internal, this.storage);
          } catch (err) {
            console.log(err.stack);
          }
          this.preprocessors[name] = [];
          this.postprocessors[name] = [];
        });
        return globby([Path.join(__dirname, '../plugins/**/*.js')]);
      }).then(files => {
        files.forEach(file => {
          const name = file.match(/\/([^\/]+)\.js$/)[1];
          const load = require;
          this.plugins[name] = new (load(file))(this.models);
        });
        Object.keys(this.plugins).forEach(name => {
          const plugin = this.plugins[name];

          // Fields.
          const fields = {};
          const names = plugin.getFields();
          names.forEach(name => {
            fields[name] = {
              plugin
            };
          });
          this.pluginFields = _.merge(this.pluginFields, fields);

          // Preprocess functions.
          plugin.getPreprocessors().forEach(name => {
            this.preprocessors[name].push(plugin);
          });

          // Postprocess functions.<commit_msg>Pass storage object to plugin constructor<commit_after>'use strict';

const fs = require('fs');
const Path = require('path');

const Yaml = require('js-yaml');
const _ = require('lodash');
const globby = require('globby');

const Script = require('./script');

class Models {
  constructor(models, options, storage) {
    this.models = models;
    this.storage = storage;
    this.scriptsDir = options.scriptsDir;
    this.databases = options.databases;
    this.engines = {};
    this.instances = {};
    this.plugins = {};
    this.scripts = {};
    this.pluginFields = {};
    this.preprocessors = {};
    this.postprocessors = {};

    this.ready = new Promise(resolve => {
      globby([Path.join(__dirname, '../engines/**/*.js')]).then(files => {
        files.forEach(file => {
          const load = require;
          const engine = load(file);
          this.engines[engine.name] = engine;
        });
        Object.keys(models).forEach(name => {
          const databaseName = this.models[name].database;
          if (typeof this.databases[databaseName] === 'undefined') {
            throw new Error('Unknown database ' + databaseName + ' in model ' + name);
          }
          const database = this.databases[databaseName];
          const engine = database.engine;
          if (typeof this.engines[engine] === 'undefined') {
            throw new Error('Unknown engine ' + engine + ' in database ' + databaseName);
          }
          try {
            this.instances[name] = new this.engines[engine](this.models[name], database, this.databases.internal, this.storage);
          } catch (err) {
            console.log(err.stack);
          }
          this.preprocessors[name] = [];
          this.postprocessors[name] = [];
        });
        return globby([Path.join(__dirname, '../plugins/**/*.js')]);
      }).then(files => {
        files.forEach(file => {
          const name = file.match(/\/([^\/]+)\.js$/)[1];
          const load = require;
          this.plugins[name] = new (load(file))(this.models, this.storage);
        });
        Object.keys(this.plugins).forEach(name => {
          const plugin = this.plugins[name];

          // Fields.
          const fields = {};
          const names = plugin.getFields();
          names.forEach(name => {
            fields[name] = {
              plugin
            };
          });
          this.pluginFields = _.merge(this.pluginFields, fields);

          // Preprocess functions.
          plugin.getPreprocessors().forEach(name => {
            this.preprocessors[name].push(plugin);
          });

          // Postprocess functions.
{'fraction_non_alphanumeric': 0.1562380038387716, 'fraction_numerical': 0.00038387715930902113, 'mean_word_length': 2.0797872340425534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3345702', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1412, 'n_words': 275}