<commit_before>var configuration = require('./configuration.js'),
	_ = require('lodash'),
	fs = require('fs'),
	request = require('request'),
	bunyan = require('bunyan');
var log = bunyan.createLogger({
	name: 'OutlookIndexer'
});


var OutlookIndexer = function OutlookIndexer(options) {
	this.options = _.assign({
		'url': configuration.url,
		'identifier': configuration.identifier,
		'username': configuration.username,
		'password': <PASSWORD>,
		'crawlfilename': configuration.crawlfilename,
		'norchindexer': configuration.norchindexer,
	}, options);

	var self = this;

	this._filterCallback = function filterCallback(existingDataObject, data) {
		console.log("Filtering " + data.length + " items...");
		if (existingDataObject.crawlDataExists) {
			var filteredData = _.filter(data, function(itemX) {
				if (itemX !== null) {
					var subFilter = _.filter(existingDataObject.crawlData.data, function(itemY) {
						if (itemY !== null && itemX !== null) {
							var identifier = self.options.identifier;
							return itemX[self.options.identifier] === itemY[self.options.identifier];
						}
					})
					console.log("SubFilter.length:" + subFilter.length)
					if (subFilter && subFilter.length === 0)
						return true;
				}
			});
			if (filteredData && filteredData.length > 0) {
				data = filteredData;
				console.log("Filtering done, result=" + data);
			}else{
				console.log("Filtering done, no delta was discovered");
			}
		} else
			console.log("No existing data to filter on")
		return data;
	}

	this._mappingCallback = function mappingCallback(data) {
		console.log("Mapping " + data.length + " items...");
		var mappedData = [];
		_.forEach(data, function(item) {
			if (item !== undefined) {
				var o = {
					Id: item.Id,
					Subject: item.Subject,
					BodyPreview: item.BodyPreview,
					CreatedOn: new Date(item.DateTimeCreated),
					From: item.From,
					ToRecipients: item.ToRecipients,
					CcRecipients: item.CcRecipients,
					Url: item.WebLink,
					ConversationId: item.ConversationId<commit_msg>implemented duplicate check<commit_after>var configuration = require('./configuration.js'),
	_ = require('lodash'),
	fs = require('fs'),
	request = require('request'),
	bunyan = require('bunyan');
var log = bunyan.createLogger({
	name: 'OutlookIndexer'
});


var OutlookIndexer = function OutlookIndexer(options) {
	this.options = _.assign({
		'url': configuration.url,
		'identifier': configuration.identifier,
		'username': configuration.username,
		'password': <PASSWORD>,
		'crawlfilename': configuration.crawlfilename,
		'norchindexer': configuration.norchindexer,
	}, options);

	var self = this;

	this._filterCallback = function filterCallback(existingDataObject, data) {
		if (!data)
			return data;

		console.log("Filtering " + data.length + " items...");
		if (existingDataObject.crawlDataExists) {
			/*var filteredData = _.filter(data, function(itemX) {
				if (itemX !== null) {
					var subFilter = _.filter(existingDataObject.crawlData.data, function(itemY) {
						if (itemY !== null && itemX !== null) {
							var identifier = self.options.identifier;
							return itemX[self.options.identifier] === itemY[self.options.identifier];
						}
					})
					console.log("SubFilter.length:" + subFilter.length)
					if (subFilter && subFilter.length === 0)
						return true;
				}
			});
			if (filteredData && filteredData.length > 0) {
				data = filteredData;
				console.log("Filtering done, result=" + data);
			} else {
				console.log("Filtering done, no delta was discovered");
			}
		}
		*/
			var duplicates = [];

			var foundDiff = false;
			var filteredData = data;

			for (var i = 0; i < existingDataObject.crawlData.data.length; i++) {
				var persistedObject = existingDataObject.crawlData.data[i];
				var foundDuplicateInCrawl = false;
				var duplicateObject;
				var duplicateObjectIdx;

				if (data) {
					for (var j = 0; j < data.length; j++) {
						var crawledObject = data[j];
						if (!crawledObject)
							continue;

						if (crawledObject[self.options.identifier] == persistedObject[self.options.identifier]) {
							duplicateObject = crawledObject;
							foundDuplicateInCrawl = true;
							duplicateObjectIdx = j;
							break;
						}

					}
				}

				if (foundDuplicateInCrawl && duplicateObject) {
					console.log("Found duplicate")
					data.splice(data[duplicateObjectIdx], 1);
					continue;
				}

			}

			console.log("DIFFDATA=" + data.length)

			return data;
		}

		console.log("No existing data to filter on")
		return data;
	}

	this._mappingCallback = function mappingCallback(data) {
		console.log("Mapping " + data.length + " items...");
		var mappedData = [];
		_.forEach(data, function(item) {
			if (item !== undefined) {
				var o = {
					Id: item.Id,
					Subject: item.Subject,
					BodyPreview: item.BodyPreview,
					CreatedOn: new Date(item.DateTimeCreated),
					From: item.From,
					ToRecipients: item.ToRecipients,
					CcRecipients: item.CcRecipients,
					Url: item.WebLink,
					ConversationId: item.ConversationId
{'fraction_non_alphanumeric': 0.13753468093539437, 'fraction_numerical': 0.0013872374157748713, 'mean_word_length': 3.4545454545454546, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '608307', 'n_tokens_mistral': 1955, 'n_tokens_neox': 1532, 'n_words': 338}