<commit_before>'use strict';

require('should');

var eml = require('../lib/');
var AnyfetchClient = require('anyfetch');
var anyfetchFileHydrater = require('anyfetch-file-hydrater');
var hydrationError = anyfetchFileHydrater.hydrationError;

process.env.ANYFETCH_API_URL = 'http://localhost:1338';
var countFile = 0;
var cb = function(url){
  if (url.indexOf("/file") !== -1) {
    countFile += 1;
  }
};

// Create a fake HTTP server
var apiServer = AnyfetchClient.debug.createTestApiServer(cb);
apiServer.listen(1338);
after(function(){
  apiServer.close();
});

describe('Test EML', function() {
  it('returns basic datas', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/sample.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('document_type', "email");
      changes.should.have.property('metadatas');
      changes.metadatas.should.have.property('to').and.eql([ { address: '<EMAIL>', name: '' } ]);
      changes.metadatas.should.have.property('cc').and.eql([]);
      changes.metadatas.should.have.property('bcc').and.eql([ { address: '<EMAIL>', name: '' }, { address: '<EMAIL>', name: '<NAME>' } ]);
      changes.metadatas.should.have.property('from').and.eql([ { address: '<EMAIL>', name: '<NAME>' } ]);
      changes.metadatas.should.have.property('subject', 'sample');
      changes.metadatas.should.have.property('text', 'Hello there!');
      changes.should.have.property('creation_date');

      done();
    });
  });

  it('returns text and html for multipart text / html', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/html-text.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('text').and.include('Hello there!');
      changes.should.have.property('datas').with.property('html', '<div dir="ltr">Hello there! In html.<br></div>');

      done();
    });
  });

  it('escapes html in text', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/html-text.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('text', 'Hello there! &lt;3');

      done();
    });
  });

  it('returns datas for html only version', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/html-only.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('text', "Vu qu'elles sont supprimées même si elles servent à plusieurs personnes \n Bonjour Ca va ?");
      changes.should.have.property('datas').with.property('html', "<p>Vu qu'elles sont supprimées même si elles servent à plusieurs personnes</p>\n<p>Bonjour</p><p>Ca va ?</p>");

      done();
    });
  });


  it('create new documents for each attachment', function(done) {
    var document = {
      datas: {},
      metadatas: {},
      access_token: "<PASSWORD>",
      identifier: "azerty",
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/attachment.eml", document, changes, function(err) {
      if(err) {
        throw err;
      }
      countFile.should.eql(1);
      done();
    });
  });

  it('should include cid images into base64 HTML', function(done){
    var document = {
      datas: {},
      metadatas: {},
      access_token: "<PASSWORD>",
      identifier: "azerty",
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/image-included.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('datas').with.property('html').and.include("6WuNurkZ3XOx8a5UuO+fDm66FRskS+RmVqwFo9");

      done();
    });
  });

  it('should have a date', function(done){
    var document = {
      datas: {},
      metadatas: {},
      access_token: "<PASSWORD>",
      identifier: "azerty",
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/image-included.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('date', new Date('Fri Jan 18 2013 12:09:13 GMT+0100 (CET)'));
      changes.should.have.property('creation_date', new Date('Fri Jan 18 2013 12:09:13 GMT+0100 (CET)'));
      done();
    });
  });

});
<commit_msg>remove hydration error garbage<commit_after>'use strict';

require('should');

var eml = require('../lib/');
var AnyfetchClient = require('anyfetch');
var anyfetchFileHydrater = require('anyfetch-file-hydrater');

process.env.ANYFETCH_API_URL = 'http://localhost:1338';
var countFile = 0;
var cb = function(url){
  if (url.indexOf("/file") !== -1) {
    countFile += 1;
  }
};

// Create a fake HTTP server
var apiServer = AnyfetchClient.debug.createTestApiServer(cb);
apiServer.listen(1338);
after(function(){
  apiServer.close();
});

describe('Test EML', function() {
  it('returns basic datas', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/sample.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('document_type', "email");
      changes.should.have.property('metadatas');
      changes.metadatas.should.have.property('to').and.eql([ { address: '<EMAIL>', name: '' } ]);
      changes.metadatas.should.have.property('cc').and.eql([]);
      changes.metadatas.should.have.property('bcc').and.eql([ { address: '<EMAIL>', name: '' }, { address: '<EMAIL>', name: '<NAME>' } ]);
      changes.metadatas.should.have.property('from').and.eql([ { address: '<EMAIL>', name: '<NAME>' } ]);
      changes.metadatas.should.have.property('subject', 'sample');
      changes.metadatas.should.have.property('text', 'Hello there!');
      changes.should.have.property('creation_date');

      done();
    });
  });

  it('returns text and html for multipart text / html', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/html-text.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('text').and.include('Hello there!');
      changes.should.have.property('datas').with.property('html', '<div dir="ltr">Hello there! In html.<br></div>');

      done();
    });
  });

  it('escapes html in text', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/html-text.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('text', 'Hello there! &lt;3');

      done();
    });
  });

  it('returns datas for html only version', function(done) {
    var document = {
      datas: {},
      metadatas: {},
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/html-only.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('text', "Vu qu'elles sont supprimées même si elles servent à plusieurs personnes \n Bonjour Ca va ?");
      changes.should.have.property('datas').with.property('html', "<p>Vu qu'elles sont supprimées même si elles servent à plusieurs personnes</p>\n<p>Bonjour</p><p>Ca va ?</p>");

      done();
    });
  });


  it('create new documents for each attachment', function(done) {
    var document = {
      datas: {},
      metadatas: {},
      access_token: "123",
      identifier: "azerty",
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/attachment.eml", document, changes, function(err) {
      if(err) {
        throw err;
      }
      countFile.should.eql(1);
      done();
    });
  });

  it('should include cid images into base64 HTML', function(done){
    var document = {
      datas: {},
      metadatas: {},
      access_token: "<PASSWORD>",
      identifier: "azerty",
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/image-included.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('datas').with.property('html').and.include("6WuNurkZ3XOx8a5UuO+fDm66FRskS+RmVqwFo9");

      done();
    });
  });

  it('should have a date', function(done){
    var document = {
      datas: {},
      metadatas: {},
      access_token: "<PASSWORD>",
      identifier: "azerty",
    };

    var changes = anyfetchFileHydrater.defaultChanges();

    eml(__dirname + "/samples/image-included.eml", document, changes, function(err, changes) {
      if(err) {
        throw err;
      }

      changes.should.have.property('metadatas').with.property('date', new Date('Fri Jan 18 2013 12:09:13 GMT+0100 (CET)'));
      changes.should.have.property('creation_date', new Date('Fri Jan 18 2013 12:09:13 GMT+0100 (CET)'));
      done();
    });
  });

});

{'fraction_non_alphanumeric': 0.17351274787535412, 'fraction_numerical': 0.01123027114528531, 'mean_word_length': 3.1239048811013768, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5164199', 'n_tokens_mistral': 3328, 'n_tokens_neox': 3099, 'n_words': 612}