<commit_before>var assert = require('chai').assert,
  fs = require('fs'),
  missingKeys = require('../bin/missing-keys'),
  async = require('async');

//test for all necessary configuration options
describe('Testing config.json', function() {
  var cfg = require('../config.json');
  var dirs = ['batches', 'creds', 'destinations', 'input', 'logs', 'output', 'sources'];

  it("Should have batches, creds, destinations, input, logs, output, sources", function(done) {
    var msg = missingKeys(cfg.dirs, dirs);
    if (!msg) {
      assert(true);
      return done();
    }
    assert(false, 'Missing: ' + msg.join(', '));
    done();
  });

  it('Those folders in config should exist', function() {
    dirs.forEach(function(d) {
      assert(fs.lstatSync(cfg.dirs[d]).isDirectory(), 'Not a directory or not exists: ' + cfg.dirs[d]);
    });
  })

  it("Should have some default bind variables", function(done) {
    assert(typeof(cfg.defaultBindVars) == 'undefined' ? false : true, typeof(cfg.defaultBindVars));
    done();
  });

  it('Should have logto option', function(done) {
    assert(typeof(cfg.logto) == 'undefined' ? false : true, typeof(cfg.logto));
    done();
  });

  it('If schedule defined, file exists', function() {
    if (typeof(cfg.schedule) == 'undefined') return assert(true);
    assert(fs.lstatSync(cfg.schedule).isFile(), 'Not a file ' + cfg.schedule);
  });

  it('If schedule defined, pm2 config has valid log filename.', function() {
    if (typeof(cfg.schedule) == 'undefined') return assert(true);
    try {
      var pm2cfg = require('../pm2.json');
    } catch(e) {
      assert(false,'Couldn\'t find file pm2.json. '+e.toString())
    }
    var p = pm2cfg.apps[0];
    var logdir = p.out_file.split('/').slice(0, -1).join('/');
    var errordir = p.error_file.split('/').slice(0, -1).join('/');
    assert(fs.lstatSync(logdir).isDirectory(), 'Log directory doesn\'t exist, bad log value in pm2.json ' + p.out_file);
    assert(fs.lstatSync(errordir).isDirectory(), 'Error log directory doesn\'t exist, bad error log value in pm2.json ' + p.error_file);
  })

});<commit_msg>beautify<commit_after>var assert = require('chai').assert,
  fs = require('fs'),
  missingKeys = require('../bin/missing-keys'),
  async = require('async');

//test for all necessary configuration options
describe('Testing config.json', function() {
  var cfg = require('../config.json');
  var dirs = ['batches', 'creds', 'destinations', 'input', 'logs', 'output', 'sources'];

  it("Should have batches, creds, destinations, input, logs, output, sources", function(done) {
    var msg = missingKeys(cfg.dirs, dirs);
    if (!msg) {
      assert(true);
      return done();
    }
    assert(false, 'Missing: ' + msg.join(', '));
    done();
  });

  it('Those folders in config should exist', function() {
    dirs.forEach(function(d) {
      assert(fs.lstatSync(cfg.dirs[d]).isDirectory(), 'Not a directory or not exists: ' + cfg.dirs[d]);
    });
  })

  it("Should have some default bind variables", function(done) {
    assert(typeof(cfg.defaultBindVars) == 'undefined' ? false : true, typeof(cfg.defaultBindVars));
    done();
  });

  it('Should have logto option', function(done) {
    assert(typeof(cfg.logto) == 'undefined' ? false : true, typeof(cfg.logto));
    done();
  });

  it('If schedule defined, file exists', function() {
    if (typeof(cfg.schedule) == 'undefined') return assert(true);
    assert(fs.lstatSync(cfg.schedule).isFile(), 'Not a file ' + cfg.schedule);
  });

  it('If schedule defined, pm2 config has valid log filename.', function() {
    if (typeof(cfg.schedule) == 'undefined') return assert(true);
    try {
      var pm2cfg = require('../pm2.json');
    } catch (e) {
      assert(false, 'Couldn\'t find file pm2.json. ' + e.toString())
    }
    var p = pm2cfg.apps[0];
    var logdir = p.out_file.split('/').slice(0, -1).join('/');
    var errordir = p.error_file.split('/').slice(0, -1).join('/');
    assert(fs.lstatSync(logdir).isDirectory(), 'Log directory doesn\'t exist, bad log value in pm2.json ' + p.out_file);
    assert(fs.lstatSync(errordir).isDirectory(), 'Error log directory doesn\'t exist, bad error log value in pm2.json ' + p.error_file);
  })

});
{'fraction_non_alphanumeric': 0.18772220905427828, 'fraction_numerical': 0.005688551789523584, 'mean_word_length': 3.7256438969764836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '454211', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1356, 'n_words': 340}