<commit_before>var request = require("request"),
    fs = require('fs'),
    express = require('express'),
	cookieParser = require('cookie-parser'),
	bodyParser = require('body-parser'),
	uuid = require('node-uuid'),
	_ = require('underscore'),
	Promise = require('promise'),
	base64 = require('base64encodedecode');


var getUtcTime = function()	{
	var now = new Date;
	var utc_timestamp = Date.UTC(now.getUTCFullYear(),now.getUTCMonth(), now.getUTCDate() , 
      now.getUTCHours(), now.getUTCMinutes(), now.getUTCSeconds(), now.getUTCMilliseconds());
	return utc_timestamp;
};

var crest = (function(request, _, getUtcTime)	{
	var cRestReqLimitPerSec = 20; //https://developers.eveonline.com/blog/article/a-note-on-crest-rate-limits
	var reqEveryXms = Math.ceil(1000 / cRestReqLimitPerSec);
	var reqQueue = [];
	var reqCount = 0;
	var startTime = getUtcTime();

	setInterval(function()	{
		if(reqQueue.length != 0)	{
			reqQueue.shift()();
			reqCount++;
		}
	}, reqEveryXms);

	var getStats = function()	{
		var runtimeInH = (getUtcTime() - startTime) / 3600000;
		return {
			totalRequests: reqCount,
			requestsPerHour: Math.round(reqCount / runtimeInH),
			runtimeInH: runtimeInH
		};
	};

	var pushReq = function(doRequest)	{
		reqQueue.push(doRequest);
	};
	var detectErrors = function(callback, errorCallback)	{
		return function(error, response, data)	{
			if(error) {
				errorCallback(error);
				return;
			}
			if(typeof data == "string")	{
				var tmp;
				try{
					tmp = JSON.parse(data);
				}
				catch(e){}
				if(tmp != null && tmp.exceptionType != null)	{
					errorCallback(tmp);
					return;
				}
			}
			if(data != null && data.exceptionType != null)	{
				errorCallback(data);
				return;
			}
			return callback.apply(this, arguments);
		};
	};
	var internalGet = function(token, url, callback, errorCallback)	{
		pushReq(function()	{
			request.get({
				url: url,
				json: true,
				headers: {
					Authorization: 'Bearer ' + token,
				}
			}, detectErrors(callback, errorCallback));
		});
	};
	var self = {
		get: function(token, url, callback, errorCallback, dataFromBefore)	{
			internalGet(token, url, function(error, response, data)	{
				if(data.items == null || !_.isArray(data.items))	{
					callback.apply(this, arguments);
					return;
				}
				var dataToPass = (dataFromBefore || []).concat(data.items);
				if(data.next != null) self.get(token, data.next.href, callback, errorCallback, dataToPass);
				else callback(error, response, dataToPass);
			}, errorCallback);
		},
		post: function(token, url, data, callback, errorCallback)	{
			pushReq(function()	{
				request.post({
					url: url,
					json: false,
					body:  JSON.stringify(data),
					headers: {
					    'content-type': 'application/json',
						Authorization: 'Bearer ' + token
					}
				}, detectErrors(callback, errorCallback));
			});
		},
		del: function(token, url, callback, errorCallback)	{
			pushReq(function()	{
				request.del({
					url: url,
					json: false,
					headers: {
					    'content-type': 'application/json',
						Authorization: 'Bearer ' + token
					}
				}, detectErrors(callback, errorCallback));
			});
		},
		stats: getStats
	};
	return self;
})(request, _, getUtcTime);

var eveapi = (function(request, crest, fs, _, base64)	{
	var config = {}; 
	fs.readFile('.apiKey', 'utf8', function(err, data)	{
		if(err) throw err;
		config.apiKey = data.trim();
	});
	fs.readFile('.clientId', 'utf8', function(err, data)	{
		if(err) throw err;
		config.clientId = data.trim();
	});

	var getContactData = function(contact)	{
		return {
			standing: contact.standing,
			contactType: contact.contactType,
			watched: contact.watched,
			contact: {<commit_msg>fixed standing data type for crest api<commit_after>var request = require("request"),
    fs = require('fs'),
    express = require('express'),
	cookieParser = require('cookie-parser'),
	bodyParser = require('body-parser'),
	uuid = require('node-uuid'),
	_ = require('underscore'),
	Promise = require('promise'),
	base64 = require('base64encodedecode');


var getUtcTime = function()	{
	var now = new Date;
	var utc_timestamp = Date.UTC(now.getUTCFullYear(),now.getUTCMonth(), now.getUTCDate() , 
      now.getUTCHours(), now.getUTCMinutes(), now.getUTCSeconds(), now.getUTCMilliseconds());
	return utc_timestamp;
};

var crest = (function(request, _, getUtcTime)	{
	var cRestReqLimitPerSec = 20; //https://developers.eveonline.com/blog/article/a-note-on-crest-rate-limits
	var reqEveryXms = Math.ceil(1000 / cRestReqLimitPerSec);
	var reqQueue = [];
	var reqCount = 0;
	var startTime = getUtcTime();

	setInterval(function()	{
		if(reqQueue.length != 0)	{
			reqQueue.shift()();
			reqCount++;
		}
	}, reqEveryXms);

	var getStats = function()	{
		var runtimeInH = (getUtcTime() - startTime) / 3600000;
		return {
			totalRequests: reqCount,
			requestsPerHour: Math.round(reqCount / runtimeInH),
			runtimeInH: runtimeInH
		};
	};

	var pushReq = function(doRequest)	{
		reqQueue.push(doRequest);
	};
	var detectErrors = function(callback, errorCallback)	{
		return function(error, response, data)	{
			if(error) {
				errorCallback(error);
				return;
			}
			if(typeof data == "string")	{
				var tmp;
				try{
					tmp = JSON.parse(data);
				}
				catch(e){}
				if(tmp != null && tmp.exceptionType != null)	{
					errorCallback(tmp);
					return;
				}
			}
			if(data != null && data.exceptionType != null)	{
				errorCallback(data);
				return;
			}
			return callback.apply(this, arguments);
		};
	};
	var internalGet = function(token, url, callback, errorCallback)	{
		pushReq(function()	{
			request.get({
				url: url,
				json: true,
				headers: {
					Authorization: 'Bearer ' + token,
				}
			}, detectErrors(callback, errorCallback));
		});
	};
	var self = {
		get: function(token, url, callback, errorCallback, dataFromBefore)	{
			internalGet(token, url, function(error, response, data)	{
				if(data.items == null || !_.isArray(data.items))	{
					callback.apply(this, arguments);
					return;
				}
				var dataToPass = (dataFromBefore || []).concat(data.items);
				if(data.next != null) self.get(token, data.next.href, callback, errorCallback, dataToPass);
				else callback(error, response, dataToPass);
			}, errorCallback);
		},
		post: function(token, url, data, callback, errorCallback)	{
			pushReq(function()	{
				request.post({
					url: url,
					json: false,
					body:  JSON.stringify(data),
					headers: {
					    'content-type': 'application/json',
						Authorization: 'Bearer ' + token
					}
				}, detectErrors(callback, errorCallback));
			});
		},
		del: function(token, url, callback, errorCallback)	{
			pushReq(function()	{
				request.del({
					url: url,
					json: false,
					headers: {
					    'content-type': 'application/json',
						Authorization: 'Bearer ' + token
					}
				}, detectErrors(callback, errorCallback));
			});
		},
		stats: getStats
	};
	return self;
})(request, _, getUtcTime);

var eveapi = (function(request, crest, fs, _, base64)	{
	var config = {}; 
	fs.readFile('.apiKey', 'utf8', function(err, data)	{
		if(err) throw err;
		config.apiKey = data.trim();
	});
	fs.readFile('.clientId', 'utf8', function(err, data)	{
		if(err) throw err;
		config.clientId = data.trim();
	});

	var getContactData = function(contact)	{
		return {
			standing: parseInt(contact.standing),
			contactType: contact.contactType,
			watched: contact.watched,
			contact: {
{'fraction_non_alphanumeric': 0.15591759795341323, 'fraction_numerical': 0.006193617880705534, 'mean_word_length': 3.548683404776485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7157597', 'n_tokens_mistral': 2978, 'n_tokens_neox': 2497, 'n_words': 512}