<commit_before>var Datastore = require('../lib/datastore')
  , testDb = 'workspace/test.db'
  , fs = require('fs')
  , path = require('path')
  , customUtils = require('../lib/customUtils')
  , should = require('chai').should()
  , assert = require('chai').assert
  , _ = require('underscore')
  , async = require('async')
  , model = require('../lib/model')
  ;


describe('Database', function () {
  var d = new Datastore(testDb);

  beforeEach(function (done) {
    async.waterfall([
      function (cb) {
        customUtils.ensureDirectoryExists(path.dirname(testDb), function () {
          fs.exists(testDb, function (exists) {
            if (exists) {
              fs.unlink(testDb, cb);
            } else { return cb(); }
          });
        });
      }
    , function (cb) {
        d.loadDatabase(function (err) {
          assert.isNull(err);
          return cb();
        });
      }
    ], done);

  });


  describe.only('Loading the database data from file', function () {

    it('Every line represents a document', function () {
      var now = new Date()
        , rawData = model.serialize({a: 2, ages: [1, 5, 12]}) + '\n' +
                    model.serialize({hello: 'world'}) + '\n' +
                    model.serialize({nested: { today: now }})
        , treatedData = Datastore.treatRawData(rawData)
        ;

      treatedData.length.should.equal(3);
      _.isEqual(treatedData[0], { a: 2, ages: [1, 5, 12] }).should.equal(true);
      _.isEqual(treatedData[1], { hello: 'world' }).should.equal(true);
      _.isEqual(treatedData[2], { nested: { today: now } }).should.equal(true);
    });

  });   // ==== End of 'Loading the database data from file' ==== //

<commit_msg>Tested that garbage lines have no impact on data treatment<commit_after>var Datastore = require('../lib/datastore')
  , testDb = 'workspace/test.db'
  , fs = require('fs')
  , path = require('path')
  , customUtils = require('../lib/customUtils')
  , should = require('chai').should()
  , assert = require('chai').assert
  , _ = require('underscore')
  , async = require('async')
  , model = require('../lib/model')
  ;


describe('Database', function () {
  var d = new Datastore(testDb);

  beforeEach(function (done) {
    async.waterfall([
      function (cb) {
        customUtils.ensureDirectoryExists(path.dirname(testDb), function () {
          fs.exists(testDb, function (exists) {
            if (exists) {
              fs.unlink(testDb, cb);
            } else { return cb(); }
          });
        });
      }
    , function (cb) {
        d.loadDatabase(function (err) {
          assert.isNull(err);
          return cb();
        });
      }
    ], done);

  });


  describe.only('Loading the database data from file', function () {

    it('Every line represents a document', function () {
      var now = new Date()
        , rawData = model.serialize({a: 2, ages: [1, 5, 12]}) + '\n' +
                    model.serialize({hello: 'world'}) + '\n' +
                    model.serialize({nested: { today: now }})
        , treatedData = Datastore.treatRawData(rawData)
        ;

      treatedData.length.should.equal(3);
      _.isEqual(treatedData[0], { a: 2, ages: [1, 5, 12] }).should.equal(true);
      _.isEqual(treatedData[1], { hello: 'world' }).should.equal(true);
      _.isEqual(treatedData[2], { nested: { today: now } }).should.equal(true);
    });

    it('Badly formatted lines have no impact on the treated data', function () {
      var now = new Date()
        , rawData = model.serialize({a: 2, ages: [1, 5, 12]}) + '\n' +
                    'garbage\n' +
                    model.serialize({nested: { today: now }})
        , treatedData = Datastore.treatRawData(rawData)
        ;

      treatedData.length.should.equal(2);
      _.isEqual(treatedData[0], { a: 2, ages: [1, 5, 12] }).should.equal(true);
      _.isEqual(treatedData[1], { nested: { today: now } }).should.equal(true);
    });

  });   // ==== End of 'Loading the database data from file' ==== //


{'fraction_non_alphanumeric': 0.17638266068759342, 'fraction_numerical': 0.010214250124564025, 'mean_word_length': 2.337489609310058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3048138', 'n_tokens_mistral': 1302, 'n_tokens_neox': 1234, 'n_words': 272}