<commit_before>/*
 * Copyright (C) 2012 Vanderbilt University, All rights reserved.
 * 
 * Author: <NAME>
 */

define([ "core/assert", "core/util", "core/config" ], function (ASSERT, UTIL, CONFIG) {
	"use strict";

	var Cache = function (storage, options) {
		ASSERT(storage !== null);

		options = UTIL.copyOptions(CONFIG.cache, options);
		
		var KEYNAME = storage.KEYNAME;
		var missing = {};
		var backup = {};
		var cache = {};
		var cacheSize = 0;

		var open = function (callback) {
			ASSERT(!storage.opened() && cacheSize === 0);

			storage.open(callback);
		};

		var close = function (callback) {
			ASSERT(storage.opened());

			storage.close(function () {
				for( var key in missing ) {
					var callbacks = missing[key];

					var cb;
					while( (cb = callbacks.pop()) ) {
						cb(new Error("storage closed"));
					}
				}

				missing = {};
				backup = {};
				cache = {};
				cacheSize = 0;

				callback();
			});
		};

		var cacheInsert = function (key, obj) {
			ASSERT(cache[key] === undefined && obj[KEYNAME] === key);

			cache[key] = obj;
			if( ++cacheSize >= options.maxSize ) {
				backup = cache;
				cache = {};
				cacheSize = 0;
			}
		};

		var load = function (key, callback) {
			ASSERT(typeof key === "string" && typeof callback === "function");

			if( key.charAt(0) === "#" ) {
				var obj = cache[key];
				if( obj === undefined ) {
					obj = backup[key];
					if( obj === undefined ) {
						obj = missing[key];
						if( obj === undefined ) {
							obj = [ callback ];
							missing[key] = obj;
							storage.load(key, function (err, obj2) {
								ASSERT(typeof obj2 === "object" || obj2 === undefined);

								if( obj.length !== 0 ) {
									ASSERT(missing[key] === obj);

									delete missing[key];
									if( !err && obj2 ) {
										cacheInsert(key, obj2);
									}

									var cb;
									while( (cb = obj.pop()) ) {
										cb(err, obj2);
									}
								}
							});
						}
						else {
							obj.push(callback);
						}
						return;
					}
					else {
						cacheInsert(key, obj);
					}
				}

				ASSERT(typeof obj === "object" && obj !== null && obj[KEYNAME] === key);
				UTIL.immediateCallback(callback, null, obj);
			}
			else {
				storage.load(key, callback);
			}
		};

		var save = function (obj, callback) {
			ASSERT(typeof obj === "object" && obj !== null && typeof callback === "function");

			var key = obj[KEYNAME];
			ASSERT(typeof key === "string");

			if( key.charAt(0) === "#" ) {
				if( cache[key] !== undefined ) {
					UTIL.immediateCallback(callback);
					return;
				}
				else {
					var item = backup[key];
					cacheInsert(key, obj);

					if( item !== undefined ) {
						UTIL.immediateCallback(callback);
						return;
					}
					else {
						item = missing[key];
						if( item !== undefined ) {
							delete missing[key];

							var cb;
							while( (cb = item.pop()) ) {
								cb(null, obj);
							}
						}
					}
				}
			}

			storage.save(obj, callback);
		};

		var remove = function (key, callback) {
			ASSERT(typeof key === "string" && typeof callback === "function");

			var item = cache[key];

			delete cache[key];
			delete backup[key];

			var callbacks = missing[key];
			if( callbacks ) {
				delete missing[key];

				var cb;
				while( (cb = callbacks.pop()) ) {
					cb(null, null);
				}
			}

			storage.remove(key, callback);
		};

		var removeAll = function (callback) {
			ASSERT(typeof callback === "function");
			
			for( var key in missing ) {
				var callbacks = missing[key];

				var cb;
				while( (cb = callbacks.pop()) ) {
					cb(null, null);
				}
			}

			missing = {};
			backup = {};
			cache = {};
			cacheSize = 0;

			storage.removeAll(callback);
		};

		return {
			open: open,
			opened: storage.opened,
			close: close,
			KEYNAME: KEYNAME,
			load: load,
			save: save,
			remove: remove,
			dumpAll: storage.dumpAll,
			removeAll: removeAll,
			searchId: storage.searchId,
			fsync: storage.fsync,
            requestPoll: storage.requestPoll
		};
	};

	return Cache;
});<commit_msg>find was missing from cache (now it only propagates the function)<commit_after>/*
 * Copyright (C) 2012 Vanderbilt University, All rights reserved.
 * 
 * Author: <NAME>
 */

define([ "core/assert", "core/util", "core/config" ], function (ASSERT, UTIL, CONFIG) {
	"use strict";

	var Cache = function (storage, options) {
		ASSERT(storage !== null);

		options = UTIL.copyOptions(CONFIG.cache, options);
		
		var KEYNAME = storage.KEYNAME;
		var missing = {};
		var backup = {};
		var cache = {};
		var cacheSize = 0;

		var open = function (callback) {
			ASSERT(!storage.opened() && cacheSize === 0);

			storage.open(callback);
		};

		var close = function (callback) {
			ASSERT(storage.opened());

			storage.close(function () {
				for( var key in missing ) {
					var callbacks = missing[key];

					var cb;
					while( (cb = callbacks.pop()) ) {
						cb(new Error("storage closed"));
					}
				}

				missing = {};
				backup = {};
				cache = {};
				cacheSize = 0;

				callback();
			});
		};

		var cacheInsert = function (key, obj) {
			ASSERT(cache[key] === undefined && obj[KEYNAME] === key);

			cache[key] = obj;
			if( ++cacheSize >= options.maxSize ) {
				backup = cache;
				cache = {};
				cacheSize = 0;
			}
		};

		var load = function (key, callback) {
			ASSERT(typeof key === "string" && typeof callback === "function");

			if( key.charAt(0) === "#" ) {
				var obj = cache[key];
				if( obj === undefined ) {
					obj = backup[key];
					if( obj === undefined ) {
						obj = missing[key];
						if( obj === undefined ) {
							obj = [ callback ];
							missing[key] = obj;
							storage.load(key, function (err, obj2) {
								ASSERT(typeof obj2 === "object" || obj2 === undefined);

								if( obj.length !== 0 ) {
									ASSERT(missing[key] === obj);

									delete missing[key];
									if( !err && obj2 ) {
										cacheInsert(key, obj2);
									}

									var cb;
									while( (cb = obj.pop()) ) {
										cb(err, obj2);
									}
								}
							});
						}
						else {
							obj.push(callback);
						}
						return;
					}
					else {
						cacheInsert(key, obj);
					}
				}

				ASSERT(typeof obj === "object" && obj !== null && obj[KEYNAME] === key);
				UTIL.immediateCallback(callback, null, obj);
			}
			else {
				storage.load(key, callback);
			}
		};

		var save = function (obj, callback) {
			ASSERT(typeof obj === "object" && obj !== null && typeof callback === "function");

			var key = obj[KEYNAME];
			ASSERT(typeof key === "string");

			if( key.charAt(0) === "#" ) {
				if( cache[key] !== undefined ) {
					UTIL.immediateCallback(callback);
					return;
				}
				else {
					var item = backup[key];
					cacheInsert(key, obj);

					if( item !== undefined ) {
						UTIL.immediateCallback(callback);
						return;
					}
					else {
						item = missing[key];
						if( item !== undefined ) {
							delete missing[key];

							var cb;
							while( (cb = item.pop()) ) {
								cb(null, obj);
							}
						}
					}
				}
			}

			storage.save(obj, callback);
		};

		var remove = function (key, callback) {
			ASSERT(typeof key === "string" && typeof callback === "function");

			var item = cache[key];

			delete cache[key];
			delete backup[key];

			var callbacks = missing[key];
			if( callbacks ) {
				delete missing[key];

				var cb;
				while( (cb = callbacks.pop()) ) {
					cb(null, null);
				}
			}

			storage.remove(key, callback);
		};

		var removeAll = function (callback) {
			ASSERT(typeof callback === "function");
			
			for( var key in missing ) {
				var callbacks = missing[key];

				var cb;
				while( (cb = callbacks.pop()) ) {
					cb(null, null);
				}
			}

			missing = {};
			backup = {};
			cache = {};
			cacheSize = 0;

			storage.removeAll(callback);
		};

		return {
			open: open,
			opened: storage.opened,
			close: close,
			KEYNAME: KEYNAME,
			load: load,
			save: save,
			remove: remove,
			dumpAll: storage.dumpAll,
			removeAll: removeAll,
			searchId: storage.searchId,
			fsync: storage.fsync,
            find: storage.find, //TODO: it should work more like load, but for now it is okay as it is
            requestPoll: storage.requestPoll
		};
	};

	return Cache;
});
{'fraction_non_alphanumeric': 0.16358360536815378, 'fraction_numerical': 0.004352557127312296, 'mean_word_length': 2.2186770428015565, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3996180', 'n_tokens_mistral': 3730, 'n_tokens_neox': 2620, 'n_words': 670}