<commit_before>var Moment = require( 'moment' );
var MailParser = require( 'mailparser' ).MailParser;
var NLPServiceIsOn = engine.config.nlpClient.NLPServiceIsOn;
var insertAnalyzedData = engine.config.nlpClient.insertAnalyzedData; 

var strip = require('htmlstrip-native');

var stripOptions = {
        include_script : false,
        include_style : false,
        compact_whitespace : true
};

function ImapMessageTranslator() {
}
ImapMessageTranslator.prototype.getMatchPatterns = function() {
	return [ '^acct:((imap)|(gmail)):.+', '/message/.*' ];
}
ImapMessageTranslator.prototype.translate = function( uri, owner, rawDoc, callback ) {
	var self = this;
	try {
		if( rawDoc.data )
		{
			var parsedUri = uri.match( /[^:]*:\/\/.*\/\/@([^\/]*)\/message\/.*/ );
			var source = parsedUri[1];
			
			var parser = new MailParser();
			parser.on( "end", function( parsedMessage ) {
				var translatedMessages = [];
				var accountUri = 'ldengine://' + owner + '//@' + source;
				
				parsedMessage[ 'x-gm-thrid' ] = rawDoc.data[ 'x-gm-thrid' ]; 
				parsedMessage[ 'x-gm-msgid' ] = rawDoc.data[ 'x-gm-msgid' ];
				parsedMessage[ 'x-gm-labels' ] = rawDoc.data[ 'x-gm-labels' ];
				
				var msg = self.translateMessage( uri, accountUri, parsedMessage  );

				var data = {
						'uri': uri,
						'data': msg,
						'category': 'message'
					};
				if( msg.updated )
					data.time = msg.updated;
				else
					data.time = 0;

				NLPService.addAnalysisIfMissing( data, function( error, result ) {
		        	process.nextTick( function() {
	            		callback( error, result.data );
		        	});
		   		});
			});

			for( var i in rawDoc.data.chunks )<commit_msg>Removed references to the NLP module from the ImapMessageTranslator.<commit_after>var Moment = require( 'moment' );
var MailParser = require( 'mailparser' ).MailParser;

function ImapMessageTranslator() {
}
ImapMessageTranslator.prototype.getMatchPatterns = function() {
	return [ '^acct:((imap)|(gmail)):.+', '/message/.*' ];
}
ImapMessageTranslator.prototype.translate = function( uri, owner, rawDoc, callback ) {
	var self = this;
	try {
		if( rawDoc.data )
		{
			var parsedUri = uri.match( /[^:]*:\/\/.*\/\/@([^\/]*)\/message\/.*/ );
			var source = parsedUri[1];
			
			var parser = new MailParser();
			parser.on( "end", function( parsedMessage ) {
				var translatedMessages = [];
				var accountUri = 'ldengine://' + owner + '//@' + source;
				
				parsedMessage[ 'x-gm-thrid' ] = rawDoc.data[ 'x-gm-thrid' ]; 
				parsedMessage[ 'x-gm-msgid' ] = rawDoc.data[ 'x-gm-msgid' ];
				parsedMessage[ 'x-gm-labels' ] = rawDoc.data[ 'x-gm-labels' ];
				
				var msg = self.translateMessage( uri, accountUri, parsedMessage  );

				var data = {
						'uri': uri,
						'data': msg,
						'category': 'message'
					};
				if( msg.updated )
					data.time = msg.updated;
				else
					data.time = 0;
				
				process.nextTick( function() {
            		callback( error, data.data );
	        	});

			});

			for( var i in rawDoc.data.chunks )
{'fraction_non_alphanumeric': 0.17052980132450332, 'fraction_numerical': 0.0013245033112582781, 'mean_word_length': 2.8143939393939394, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6710749', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1018, 'n_words': 216}