<commit_before>
const ASSERT = require("assert");
const WAITFOR = require("waitfor");
const CRYPTO = require("crypto");
const Q = require("q");
const UTILS = require("cifre/utils");
const AES = require("cifre/aes");
const MD5 = require("cifre/md5");
const FORGE_UTIL = require("cifre/forge/util")();
const SHA1 = require("cifre/forge/sha1")();
const SHA256 = require("cifre/forge/sha256")();

const DB_KEY_NS = "29";


var fullContacts = {};


function arrayToAscii(array) {
	var string = "";
	for (var i = 0, l = array.length; i < l; i++) {
		string += String.fromCharCode(array[i]);
	}
	return string;
}

function md5(data) {
	return UTILS.tohex(MD5(UTILS.stringToArray(data)));
}

function sha256(data) {
	var md = SHA256.create();
	md.start();
	md.update(data);
	return md.digest().toHex();
}

function encrypt(key, iv, data) {
	var state = UTILS.stringToArray(data);
	AES.cfb.encrypt(state, AES.keyExpansion(UTILS.fromhex(key)), UTILS.fromhex(iv));
	return UTILS.tohex(state);
}

function decrypt(key, iv, data) {
	var state = UTILS.fromhex(data);
	AES.cfb.decrypt(state, AES.keyExpansion(UTILS.fromhex(key)), UTILS.fromhex(iv));
	return arrayToAscii(state);
}

function decryptToken(token, tokenSecret, callback) {
	var tokenParts = token.split("-");
	var tokenData = decrypt(sha256(tokenSecret), tokenParts[0], tokenParts[1]);
	var credentials = null;
	try {
		credentials = JSON.parse(tokenData);
	} catch(err) {
		console.error("Error parsing token data:", tokenData, "from token:", token);
		console.error(err.stack);
		return callback(err);
	}
	return callback(null, credentials);
}

function encryptToken(tokenInfo, tokenSecret, callback) {
	return CRYPTO.randomBytes(32, function(err, buffer) {
		if (err) return callback(err);
		var iv = md5(buffer.toString("hex"));
		return callback(null, iv + "-" + encrypt(sha256(tokenSecret), iv, JSON.stringify(tokenInfo)));
	});
}


function Services(rolodex, options) {
	var self = this;
	self.rolodex = rolodex;
	self.options = options;

	if (rolodex.kickq) {
		rolodex.kickq.process([
			"fetch-for-service"
		], {
			concurrentJobs: 10
		}, function(jobItem, jobData, callback) {
			try {
				self.options.logger.debug("[rolodex][job:fetch-for-service] Running job:", jobItem.data);

				ASSERT.equal(typeof jobData.sessionID, "string");
				ASSERT.equal(typeof jobData.serviceId, "string");
				ASSERT.equal(typeof jobData.credentials, "object");

				return self.getForSessionID(jobData.sessionID, function(err, servicesSession) {
					if (err) throw err;
					return servicesSession._runFetchForService(jobData.serviceId, jobData.credentials, function(err) {
						if (err) {
							self.options.logger.error("[rolodex][job:fetch-for-service] Finished job '" + jobItem.id + "' with error:", err.stack);
							return callback(err.message);
						}
						self.options.logger.debug("[rolodex][job:fetch-for-service] Finished job '" + jobItem.id + "' successfully");
						return callback();
					});
				});

			} catch(err) {
				self.options.logger.error("[rolodex][job:fetch-for-service] Error initializing job:", err.stack);
				return callback(err.message);
			}
		});
	}
}

Services.prototype.getForSessionID = function(sessionID, callback) {
	var self = this;
	var session = new ServicesSession(self.rolodex, sessionID, self.options);
	return session.load(function(err) {
		if (err) return callback(err);
		try {
			return callback(null, session);
		} catch(err) {
			return callback(err);
		}
	});
}

Services.prototype.getForToken = function(sessionID, token, callback) {
	var self = this;

	return decryptToken(token, self.rolodex.config.api.tokenSecret, function(err, credentials) {
		if (err) return callback(err);

		try {

			var id = sessionID || UTILS.tohex(MD5(token.split("-")[0] + ":" + credentials.token));

			var session = new ServicesSession(self.rolodex, id, self.options);

			session._tokenData = credentials;

			self.rolodex.config.services.filter(function(service) {
				if (service.name === credentials.service) return true;
				return false;
			})[0].passport = {
				consumerKey: credentials.consumer_key,
				consumerSecret: credentials.consumer_secret
			};

			return session.load(function(err) {
				if (err) return callback(err);

				return session.loginService(credentials.service, {
					accessToken: credentials.token,
					accessTokenSecret: credentials.token_secret
				}, function(err) {
					if (err) return callback(err);
					return callback(null, session);
				});
			});

		} catch(err) {
			return callback(err);
		}
	});
}


function ServicesSession(rolodex, sessionID, options) {
	var self = this;
	self.rolodex = rolodex;
	self.options = options;
	self.sessionID = sessionID;
	self.services = {};
	self.servicesLoaded = null;
	function initService(serviceId) {
		var hidden = {
			contacts: {},
			credentials: {},
			load: function(callback) {
			},
			save: function(callback) {
			}
		};
		var service = Object.create(hidden);
		service.loggedin = false;
		service.logoutReason = false;
		service.fetching = false;
		service.lastFetchTime = false;
		service.fetchCount = 0;
		service.error = false;
		service.hCard = null;
		service.token = <PASSWORD>;
		service.contactsTotal = 0;
		service.contactsFetched = 0;
		// For some contacts there is no data available due to privary settings.
		// We record these just for informational purposes. They are not counted
		// in `service.contactsTotal` nor `service.contactsFetched`.
		// TODO: We may get rid of `service.contactsDropped` as linkedin is the only service using it so far.
		service.contactsDropped = 0;
		service.authURL = [
			(options.port === 443) ? "https" : "http",
			"://",
			options.hostname,
			(options.port && options.port !== 443 && options.port !== 80) ? ":" + options.port : "",
			self.rolodex.config.routes.auth,
			"/",
			serviceId
		].join("");
		service.logoutURL = [
			(options.port === 443) ? "https" : "http",
			"://",
			options.hostname,
			(options.port && options.port !== 443 && options.port !== 80) ? ":" + options.port : "",
			self.rolodex.config.routes.logout,
			"/",
			serviceId
		].join("");
		service.refetchURL = [
			(options.port === 443) ? "https" : "http",
			"://",
			options.hostname,
			(options.port && options.port !== 443 && options.port !== 80) ? ":" + options.port : "",
			self.rolodex.config.routes.refetch,
			"/",
			serviceId
		].join("");
		service.sid = self.sessionID;
		self.services[serviceId] = service;
	}
	for (var serviceId in self.rolodex.provisionedServices) {
		initService(serviceId);
	}
}

ServicesSession.prototype._getDBKey = function() {
	return [
		this.rolodex.config.db.redis.prefix,
		DB_KEY_NS,
		":",
		"services",
		":",
		this.sessionID
	].join("");
}

ServicesSession.prototype.load = function(callback) {
	var self = this;
    return self.rolodex.db.get(self._getDBKey(), function (err, reply) {
    	if (err) return callback(err);
    	if (reply) {
    		try {
    			self.servicesLoaded = reply;
    			var data = JSON.parse(reply);
    			for (var serviceId in data) {
    				if (self.services[serviceId]) {
		    			for (var key in data[serviceId]) {
		    				if (self.services[serviceId].hasOwnProperty(key)) {
			    				self.services[serviceId][key] = data[serviceId][key];
			    			} else {
			    				self.services[serviceId].__proto__[key] = data[serviceId][key];
			    			}
		    			}
	    			}
    			}
    		} catch(err) {
    			return callback(new Error("Error parsing redis response for key '" + self._getDBKey() + "'"));
    		}
    	}
		return callback(null);
    });
}

ServicesSession.prototype.save = function(changes, callback) {
	var self = this;
	if (typeof changes === "function" && typeof callback === "undefined") {
		callback = changes;
		changes = null;
	}
	function save(callback) {
		// TODO: Have records expire after some time for stale users (e.g. 1 month).
		var services = {};
		for (var serviceId in self.services) {
			services[serviceId] = {};
			for (var key in self.services[serviceId]) {
				if (key !== "authURL" && key !== "logoutURL" && key !== "refetchURL") {
					if (changes && changes[serviceId] && typeof changes[serviceId][key] !== "undefined") {
						services[serviceId][key] = changes[serviceId][key];
					} else {
						services[serviceId][key] = self.services[serviceId][key];
					}
				}
			}
			for (var key in self.services[serviceId].__proto__) {
				if (key !== "contacts") {
					if (changes && changes[serviceId] && typeof changes[serviceId][key] !== "undefined") {
						services[serviceId][key] = changes[serviceId][key];
					} else {
						services[serviceId][key] = self.services[serviceId].__proto__[key];
					}
				}
			}		
		}
		services = JSON.stringify(services);
		if (services === self.servicesLoaded) {
			return callback(null);		
		}
	    return self.rolodex.db.get(self._getDBKey(), function (err, reply) {
	    	if (err) return callback(err);
	    	if (reply && reply !== self.servicesLoaded) {
console.log("DB DATA ORIGINAL", JSON.stringify(JSON.parse(self.servicesLoaded), null, 4));
console.log("DB DATA NOW", JSON.stringify(JSON.parse(reply), null, 4));
console.log("DB DATA TO SAVE", JSON.stringify(JSON.parse(services), null, 4));
	    		return callback(new Error("Could not save to DB as DB value changed from what we originally loaded."));
	    	}
		    return self.rolodex.db.set(self._getDBKey(), services, function (err, reply) {
		    	if (err) return callback(err);
		    	self.servicesLoaded = services;
				return callback(null);
		    });
		});
	}
	if (changes) {
		return self.load(function(err) {
			if (err) return callback(err);
			return save(callback);
		});
	}
	return save(callback);
}

ServicesSession.prototype.getServices = function() {
	return this.services;
}

ServicesSession.prototype._getContactsKey = function(serviceId, time) {
	return [
		this.rolodex.config.db.redis.prefix,
		DB_KEY_NS,
		":",
		"contacts",
		":",
		time,
		":",
		this.services[serviceId].hCard.uid
	].join("");
}

ServicesSession.prototype._getPeerContactsKey = function(identity) {
	return [
		this.rolodex.config.db.redis.prefix,
		"identity",
		":",
		"peercontact",
		":",
		identity
	].join("");
}

ServicesSession.prototype.loadContactsForService = function(serviceId, callback) {
	var self = this;
	if (!self.services[serviceId].hCard) return callback(null);
    return self.rolodex.db.get(self._getContactsKey(serviceId, self.services[serviceId].lastFetchTime), function (err, reply) {
    	if (err) return callback(err);
    	if (!reply) {
			return callback(null);
    	}
		try {
			var data = JSON.parse(reply);
			var peerContactKeys = [];
			for (var contactId in data.fetched) {
    			self.services[serviceId].contacts[contactId] = data.fetched[contactId];
    			peerContactKeys.push(self._getPeerContactsKey(serviceId + ":" + contactId));
    		}
			self.services[serviceId].contactsFetched = Object.keys(self.services[serviceId].contacts).length;
			self.services[serviceId].contactsDropped = data.dropped;

		} catch(err) {
			return callback(new Error("Error parsing redis response for key '" + self._getContactsKey(serviceId, self.services[serviceId].lastFetchTime) + "'"));
		}
		if (peerContactKeys.length === 0) {
			return callback(null);
		}
	    return self.rolodex.db.mget(peerContactKeys, function (err, reply) {
	    	if (err) return callback(err);
	    	if (reply) {
	    		for (var i=0; i<peerContactKeys.length ; i++) {
	    			if (reply[i]) {
	    				var m = peerContactKeys[i].match(/:([^:]*):([^:]*)$/);
	    				self.services[m[1]].contacts[m[2]].peerContact = reply[i];
	    			}
	    		}
	    	}
			return callback(null);
	    });
    });
}

ServicesSession.prototype.getCntactsForServiceAndTimestamp = function(serviceId, timestamp, callback) {
	var self = this;
	if (!self.services[serviceId].hCard) return callback(null);
    return self.rolodex.db.get(self._getContactsKey(serviceId, timestamp), function (err, reply) {
    	if (err) return callback(err);
    	if (!reply) {
			return callback(null);
    	}
    	var data = null;
		try {
			data = JSON.parse(reply);
		} catch(err) {
			return callback(new Error("Error parsing redis response for key '" + self._getContactsKey(serviceId, timestamp) + "'"));
		}
		return callback(null, data);
    });
}

ServicesSession.prototype.resetPeerContactID = function(callback) {
	var self = this;
	var changedServices = [];
	var waitfor = WAITFOR.parallel(function(err) {
		if (err) return callback(err);
		if (changedServices.length === 0) return callback(null);
		return self.save(function(err) {
			if (err) return callback(err);
			self.rolodex.emit("service.updated", changedServices, self);
			return callback(null);
		});
	});
	for (var serviceId in self.services) {
		if (!self.services[serviceId].hCard || !self.services[serviceId].hCard.peerContact) continue;
		waitfor(serviceId, function(serviceId, done) {
			changedServices.push(serviceId);
			delete self.services[serviceId].hCard.peerContact;
		    return self.rolodex.db.del(self._getPeerContactsKey(self.services[serviceId].hCard.uid), function (err, reply) {
		    	if (err) return done(err);
				return done(null);
		    });
		});
	}
	waitfor();
}

ServicesSession.prototype.saveContactsForService = function(serviceId, callback) {
	var self = this;
	// TODO: Have records expire after some time for stale users (e.g. 1 month).
    return self.rolodex.db.set(self._getContactsKey(serviceId, self.services[serviceId].lastFetchTime), JSON.stringify({
    	dropped: self.services[serviceId].contactsDropped,
    	fetched: self.services[serviceId].contacts
    }), function (err, reply) {
    	if (err) return callback(err);
    	// TODO: Set expiry of contacts to 30 days so redis will remove.
		return callback(null);
    });
}

ServicesSession.prototype.getContacts = function(onlyServiceId, callback) {
	var self = this;
	var contacts = {};
	var waitfor = WAITFOR.parallel(function(err) {
		if (err) return callback(err);
		if (onlyServiceId) {
			return callback(null, contacts[onlyServiceId] || {});
		}
		return callback(null, contacts);
	});
	for (var serviceId in self.services) {
		waitfor(serviceId, function(serviceId, done) {
			return self.loadContactsForService(serviceId, function(err) {
				if (err) return done(err);
				contacts[serviceId] = self.services[serviceId].contacts;
				return done(null);
			});
		});
	}
	return waitfor();
}

ServicesSession.prototype.loginService = function(serviceId, credentials, callback) {
	var self = this;
	var service = self.services[serviceId];
	service.loggedin = true;
	service.logoutReason = false;
	service.__proto__.credentials = credentials;

	function setToken(callback) {
		/*
		<sharedSecret> = 52+ plain text characters
		<iv> = MD5 random hash (16 bytes)
		token = hex(<iv>) + "-" + hex(AES.encrypt(sha256(<sharedSecret>), <iv>, <credentials>))

		<credentials> = JSON.stringify({
		    service: <name (github|twitter|linkedin|facebook)>
		    consumer_key: <OAuth consumer/api key provided by service>,
		    consumer_secret: <OAuth consumer/api secret provided by service>,
		    token: <OAuth access token>,
		    token_secret: <OAuth access token secret>
		})
		*/
<commit_msg>Failed to authenticate: Cannot read property 'tokenSecret' of undefined (#24)<commit_after>
const ASSERT = require("assert");
const WAITFOR = require("waitfor");
const CRYPTO = require("crypto");
const Q = require("q");
const UTILS = require("cifre/utils");
const AES = require("cifre/aes");
const MD5 = require("cifre/md5");
const FORGE_UTIL = require("cifre/forge/util")();
const SHA1 = require("cifre/forge/sha1")();
const SHA256 = require("cifre/forge/sha256")();

const DB_KEY_NS = "29";


var fullContacts = {};


function arrayToAscii(array) {
	var string = "";
	for (var i = 0, l = array.length; i < l; i++) {
		string += String.fromCharCode(array[i]);
	}
	return string;
}

function md5(data) {
	return UTILS.tohex(MD5(UTILS.stringToArray(data)));
}

function sha256(data) {
	var md = SHA256.create();
	md.start();
	md.update(data);
	return md.digest().toHex();
}

function encrypt(key, iv, data) {
	var state = UTILS.stringToArray(data);
	AES.cfb.encrypt(state, AES.keyExpansion(UTILS.fromhex(key)), UTILS.fromhex(iv));
	return UTILS.tohex(state);
}

function decrypt(key, iv, data) {
	var state = UTILS.fromhex(data);
	AES.cfb.decrypt(state, AES.keyExpansion(UTILS.fromhex(key)), UTILS.fromhex(iv));
	return arrayToAscii(state);
}

function decryptToken(token, tokenSecret, callback) {
	var tokenParts = token.split("-");
	var tokenData = decrypt(sha256(tokenSecret), tokenParts[0], tokenParts[1]);
	var credentials = null;
	try {
		credentials = JSON.parse(tokenData);
	} catch(err) {
		console.error("Error parsing token data:", tokenData, "from token:", token);
		console.error(err.stack);
		return callback(err);
	}
	return callback(null, credentials);
}

function encryptToken(tokenInfo, tokenSecret, callback) {
	return CRYPTO.randomBytes(32, function(err, buffer) {
		if (err) return callback(err);
		var iv = md5(buffer.toString("hex"));
		return callback(null, iv + "-" + encrypt(sha256(tokenSecret), iv, JSON.stringify(tokenInfo)));
	});
}


function Services(rolodex, options) {
	var self = this;
	self.rolodex = rolodex;
	self.options = options;

	if (rolodex.kickq) {
		rolodex.kickq.process([
			"fetch-for-service"
		], {
			concurrentJobs: 10
		}, function(jobItem, jobData, callback) {
			try {
				self.options.logger.debug("[rolodex][job:fetch-for-service] Running job:", jobItem.data);

				ASSERT.equal(typeof jobData.sessionID, "string");
				ASSERT.equal(typeof jobData.serviceId, "string");
				ASSERT.equal(typeof jobData.credentials, "object");

				return self.getForSessionID(jobData.sessionID, function(err, servicesSession) {
					if (err) throw err;
					return servicesSession._runFetchForService(jobData.serviceId, jobData.credentials, function(err) {
						if (err) {
							self.options.logger.error("[rolodex][job:fetch-for-service] Finished job '" + jobItem.id + "' with error:", err.stack);
							return callback(err.message);
						}
						self.options.logger.debug("[rolodex][job:fetch-for-service] Finished job '" + jobItem.id + "' successfully");
						return callback();
					});
				});

			} catch(err) {
				self.options.logger.error("[rolodex][job:fetch-for-service] Error initializing job:", err.stack);
				return callback(err.message);
			}
		});
	}
}

Services.prototype.getForSessionID = function(sessionID, callback) {
	var self = this;
	var session = new ServicesSession(self.rolodex, sessionID, self.options);
	return session.load(function(err) {
		if (err) return callback(err);
		try {
			return callback(null, session);
		} catch(err) {
			return callback(err);
		}
	});
}

Services.prototype.getForToken = function(sessionID, token, callback) {
	var self = this;

	return decryptToken(token, self.rolodex.config.api.tokenSecret, function(err, credentials) {
		if (err) return callback(err);

		try {

			var id = sessionID || UTILS.tohex(MD5(token.split("-")[0] + ":" + credentials.token));

			var session = new ServicesSession(self.rolodex, id, self.options);

			session._tokenData = credentials;

			self.rolodex.config.services.filter(function(service) {
				if (service.name === credentials.service) return true;
				return false;
			})[0].passport = {
				consumerKey: credentials.consumer_key,
				consumerSecret: credentials.consumer_secret
			};

			return session.load(function(err) {
				if (err) return callback(err);

				return session.loginService(credentials.service, {
					accessToken: credentials.token,
					accessTokenSecret: credentials.token_secret
				}, function(err) {
					if (err) return callback(err);
					return callback(null, session);
				});
			});

		} catch(err) {
			return callback(err);
		}
	});
}


function ServicesSession(rolodex, sessionID, options) {
	var self = this;
	self.rolodex = rolodex;
	self.options = options;
	self.sessionID = sessionID;
	self.services = {};
	self.servicesLoaded = null;
	function initService(serviceId) {
		var hidden = {
			contacts: {},
			credentials: {},
			load: function(callback) {
			},
			save: function(callback) {
			}
		};
		var service = Object.create(hidden);
		service.loggedin = false;
		service.logoutReason = false;
		service.fetching = false;
		service.lastFetchTime = false;
		service.fetchCount = 0;
		service.error = false;
		service.hCard = null;
		service.token = null;
		service.contactsTotal = 0;
		service.contactsFetched = 0;
		// For some contacts there is no data available due to privary settings.
		// We record these just for informational purposes. They are not counted
		// in `service.contactsTotal` nor `service.contactsFetched`.
		// TODO: We may get rid of `service.contactsDropped` as linkedin is the only service using it so far.
		service.contactsDropped = 0;
		service.authURL = [
			(options.port === 443) ? "https" : "http",
			"://",
			options.hostname,
			(options.port && options.port !== 443 && options.port !== 80) ? ":" + options.port : "",
			self.rolodex.config.routes.auth,
			"/",
			serviceId
		].join("");
		service.logoutURL = [
			(options.port === 443) ? "https" : "http",
			"://",
			options.hostname,
			(options.port && options.port !== 443 && options.port !== 80) ? ":" + options.port : "",
			self.rolodex.config.routes.logout,
			"/",
			serviceId
		].join("");
		service.refetchURL = [
			(options.port === 443) ? "https" : "http",
			"://",
			options.hostname,
			(options.port && options.port !== 443 && options.port !== 80) ? ":" + options.port : "",
			self.rolodex.config.routes.refetch,
			"/",
			serviceId
		].join("");
		service.sid = self.sessionID;
		self.services[serviceId] = service;
	}
	for (var serviceId in self.rolodex.provisionedServices) {
		initService(serviceId);
	}
}

ServicesSession.prototype._getDBKey = function() {
	return [
		this.rolodex.config.db.redis.prefix,
		DB_KEY_NS,
		":",
		"services",
		":",
		this.sessionID
	].join("");
}

ServicesSession.prototype.load = function(callback) {
	var self = this;
    return self.rolodex.db.get(self._getDBKey(), function (err, reply) {
    	if (err) return callback(err);
    	if (reply) {
    		try {
    			self.servicesLoaded = reply;
    			var data = JSON.parse(reply);
    			for (var serviceId in data) {
    				if (self.services[serviceId]) {
		    			for (var key in data[serviceId]) {
		    				if (self.services[serviceId].hasOwnProperty(key)) {
			    				self.services[serviceId][key] = data[serviceId][key];
			    			} else {
			    				self.services[serviceId].__proto__[key] = data[serviceId][key];
			    			}
		    			}
	    			}
    			}
    		} catch(err) {
    			return callback(new Error("Error parsing redis response for key '" + self._getDBKey() + "'"));
    		}
    	}
		return callback(null);
    });
}

ServicesSession.prototype.save = function(changes, callback) {
	var self = this;
	if (typeof changes === "function" && typeof callback === "undefined") {
		callback = changes;
		changes = null;
	}
	function save(callback) {
		// TODO: Have records expire after some time for stale users (e.g. 1 month).
		var services = {};
		for (var serviceId in self.services) {
			services[serviceId] = {};
			for (var key in self.services[serviceId]) {
				if (key !== "authURL" && key !== "logoutURL" && key !== "refetchURL") {
					if (changes && changes[serviceId] && typeof changes[serviceId][key] !== "undefined") {
						services[serviceId][key] = changes[serviceId][key];
					} else {
						services[serviceId][key] = self.services[serviceId][key];
					}
				}
			}
			for (var key in self.services[serviceId].__proto__) {
				if (key !== "contacts") {
					if (changes && changes[serviceId] && typeof changes[serviceId][key] !== "undefined") {
						services[serviceId][key] = changes[serviceId][key];
					} else {
						services[serviceId][key] = self.services[serviceId].__proto__[key];
					}
				}
			}		
		}
		services = JSON.stringify(services);
		if (services === self.servicesLoaded) {
			return callback(null);		
		}
	    return self.rolodex.db.get(self._getDBKey(), function (err, reply) {
	    	if (err) return callback(err);
	    	if (reply && reply !== self.servicesLoaded) {
console.log("DB DATA ORIGINAL", JSON.stringify(JSON.parse(self.servicesLoaded), null, 4));
console.log("DB DATA NOW", JSON.stringify(JSON.parse(reply), null, 4));
console.log("DB DATA TO SAVE", JSON.stringify(JSON.parse(services), null, 4));
	    		return callback(new Error("Could not save to DB as DB value changed from what we originally loaded."));
	    	}
		    return self.rolodex.db.set(self._getDBKey(), services, function (err, reply) {
		    	if (err) return callback(err);
		    	self.servicesLoaded = services;
				return callback(null);
		    });
		});
	}
	if (changes) {
		return self.load(function(err) {
			if (err) return callback(err);
			return save(callback);
		});
	}
	return save(callback);
}

ServicesSession.prototype.getServices = function() {
	return this.services;
}

ServicesSession.prototype._getContactsKey = function(serviceId, time) {
	return [
		this.rolodex.config.db.redis.prefix,
		DB_KEY_NS,
		":",
		"contacts",
		":",
		time,
		":",
		this.services[serviceId].hCard.uid
	].join("");
}

ServicesSession.prototype._getPeerContactsKey = function(identity) {
	return [
		this.rolodex.config.db.redis.prefix,
		"identity",
		":",
		"peercontact",
		":",
		identity
	].join("");
}

ServicesSession.prototype.loadContactsForService = function(serviceId, callback) {
	var self = this;
	if (!self.services[serviceId].hCard) return callback(null);
    return self.rolodex.db.get(self._getContactsKey(serviceId, self.services[serviceId].lastFetchTime), function (err, reply) {
    	if (err) return callback(err);
    	if (!reply) {
			return callback(null);
    	}
		try {
			var data = JSON.parse(reply);
			var peerContactKeys = [];
			for (var contactId in data.fetched) {
    			self.services[serviceId].contacts[contactId] = data.fetched[contactId];
    			peerContactKeys.push(self._getPeerContactsKey(serviceId + ":" + contactId));
    		}
			self.services[serviceId].contactsFetched = Object.keys(self.services[serviceId].contacts).length;
			self.services[serviceId].contactsDropped = data.dropped;

		} catch(err) {
			return callback(new Error("Error parsing redis response for key '" + self._getContactsKey(serviceId, self.services[serviceId].lastFetchTime) + "'"));
		}
		if (peerContactKeys.length === 0) {
			return callback(null);
		}
	    return self.rolodex.db.mget(peerContactKeys, function (err, reply) {
	    	if (err) return callback(err);
	    	if (reply) {
	    		for (var i=0; i<peerContactKeys.length ; i++) {
	    			if (reply[i]) {
	    				var m = peerContactKeys[i].match(/:([^:]*):([^:]*)$/);
	    				self.services[m[1]].contacts[m[2]].peerContact = reply[i];
	    			}
	    		}
	    	}
			return callback(null);
	    });
    });
}

ServicesSession.prototype.getCntactsForServiceAndTimestamp = function(serviceId, timestamp, callback) {
	var self = this;
	if (!self.services[serviceId].hCard) return callback(null);
    return self.rolodex.db.get(self._getContactsKey(serviceId, timestamp), function (err, reply) {
    	if (err) return callback(err);
    	if (!reply) {
			return callback(null);
    	}
    	var data = null;
		try {
			data = JSON.parse(reply);
		} catch(err) {
			return callback(new Error("Error parsing redis response for key '" + self._getContactsKey(serviceId, timestamp) + "'"));
		}
		return callback(null, data);
    });
}

ServicesSession.prototype.resetPeerContactID = function(callback) {
	var self = this;
	var changedServices = [];
	var waitfor = WAITFOR.parallel(function(err) {
		if (err) return callback(err);
		if (changedServices.length === 0) return callback(null);
		return self.save(function(err) {
			if (err) return callback(err);
			self.rolodex.emit("service.updated", changedServices, self);
			return callback(null);
		});
	});
	for (var serviceId in self.services) {
		if (!self.services[serviceId].hCard || !self.services[serviceId].hCard.peerContact) continue;
		waitfor(serviceId, function(serviceId, done) {
			changedServices.push(serviceId);
			delete self.services[serviceId].hCard.peerContact;
		    return self.rolodex.db.del(self._getPeerContactsKey(self.services[serviceId].hCard.uid), function (err, reply) {
		    	if (err) return done(err);
				return done(null);
		    });
		});
	}
	waitfor();
}

ServicesSession.prototype.saveContactsForService = function(serviceId, callback) {
	var self = this;
	// TODO: Have records expire after some time for stale users (e.g. 1 month).
    return self.rolodex.db.set(self._getContactsKey(serviceId, self.services[serviceId].lastFetchTime), JSON.stringify({
    	dropped: self.services[serviceId].contactsDropped,
    	fetched: self.services[serviceId].contacts
    }), function (err, reply) {
    	if (err) return callback(err);
    	// TODO: Set expiry of contacts to 30 days so redis will remove.
		return callback(null);
    });
}

ServicesSession.prototype.getContacts = function(onlyServiceId, callback) {
	var self = this;
	var contacts = {};
	var waitfor = WAITFOR.parallel(function(err) {
		if (err) return callback(err);
		if (onlyServiceId) {
			return callback(null, contacts[onlyServiceId] || {});
		}
		return callback(null, contacts);
	});
	for (var serviceId in self.services) {
		waitfor(serviceId, function(serviceId, done) {
			return self.loadContactsForService(serviceId, function(err) {
				if (err) return done(err);
				contacts[serviceId] = self.services[serviceId].contacts;
				return done(null);
			});
		});
	}
	return waitfor();
}

ServicesSession.prototype.loginService = function(serviceId, credentials, callback) {
	var self = this;
	var service = self.services[serviceId];
	service.loggedin = true;
	service.logoutReason = false;
	service.__proto__.credentials = credentials;

	function setToken(callback) {
		if (!self.rolodex.config.api || !self.rolodex.config.api.tokenSecret) {
			return callback(null);
		}
		/*
		<sharedSecret> = 52+ plain text characters
		<iv> = MD5 random hash (16 bytes)
		token = hex(<iv>) + "-" + hex(AES.encrypt(sha256(<sharedSecret>), <iv>, <credentials>))

		<credentials> = JSON.stringify({
		    service: <name (github|twitter|linkedin|facebook)>
		    consumer_key: <OAuth consumer/api key provided by service>,
		    consumer_secret: <OAuth consumer/api secret provided by service>,
		    token: <OAuth access token>,
		    token_secret: <OAuth access token secret>
		})
		*/

{'fraction_non_alphanumeric': 0.15275042667716948, 'fraction_numerical': 0.005645267165550741, 'mean_word_length': 3.825625593918277, 'pattern_counts': {'":': 30, '<': 32, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1625840', 'n_tokens_mistral': 11290, 'n_tokens_neox': 10010, 'n_words': 2099}