<commit_before>'use strict';

var assign         = require('es5-ext/object/assign')
  , setPrototypeOf = require('es5-ext/object/set-prototype-of')
  , camelToHyphen  = require('es5-ext/string/#/camel-to-hyphen')
  , d              = require('d')
  , lazy           = require('d/lazy')
  , deferred       = require('deferred')
  , resolveKeyPath = require('dbjs/_setup/utils/resolve-key-path')
  , resolve        = require('path').resolve
  , mkdir          = require('fs2/mkdir')
  , rmdir          = require('fs2/rmdir')
  , Storage        = require('dbjs-persistence/storage')
  , resolveValue   = require('dbjs-persistence/lib/resolve-direct-value')

  , isArray = Array.isArray, create = Object.create, stringify = JSON.stringify, parse = JSON.parse
  , getOpts = { fillCache: false };

var LevelStorage = module.exports = function (driver, name/*, options*/) {
	if (!(this instanceof LevelStorage)) return new LevelStorage(driver, name, arguments[2]);
	Storage.call(this, driver, name, arguments[2]);
	this.dbPath = resolve(driver.dbPath, camelToHyphen.call(name));
};
setPrototypeOf(LevelStorage, Storage);

LevelStorage.prototype = Object.create(Storage.prototype, assign({
	constructor: d(LevelStorage),

	// Any data
	__getRaw: d(function (cat, ns, path) {
		if (cat === 'reduced') return this._getReduced_(ns + (path ? ('/' + path) : ''));
		if (cat === 'computed') return this._getComputed_(path, ns);
		return this._getDirect_(ns, path);
	}),
	__storeRaw: d(function (cat, ns, path, data) {
		if (cat === 'reduced') return this._storeReduced_(ns + (path ? ('/' + path) : ''), data);
		if (cat === 'computed') return this._storeComputed_(path, ns, data);
		return this._storeDirect_(ns, path, data);
	}),

	// Direct data
	__getObject: d(function (ownerId, objectPath, keyPaths) {
		var query = {
			gte: ownerId + (objectPath ? '/' + objectPath : ''),
			lte: ownerId + (objectPath ? '/' + objectPath : '') + '/\uffff'
		};
		return this._loadDirect_(query, keyPaths && function (ownerId, path) {
			if (!path) return true;
			return keyPaths.has(resolveKeyPath(ownerId + '/' + path));
		});
	}),
	__getAllObjectIds: d(function () {
		return this.directDb(function (db) {
			var def = deferred(), data = create(null);
			db.createReadStream().on('data', function (record) {
				var index = record.key.indexOf('/');
				if (index === -1) {
					index = record.value.indexOf('.');
					data[record.key] = {
						value: record.value.slice(index + 1),
						stamp: Number(record.value.slice(0, index))
					};
				}
			}).on('error', def.reject).on('close', function () { def.resolve(data); });
			return def.promise;
		});
	}),
	__getAll: d(function () { return this._loadDirect_(); }),

	// Reduced data
	__getReducedObject: d(function (ns, keyPaths) {
		return this.reducedDb(function (db) {
			var def, result;
			def = deferred();
			result = create(null);
			db.createReadStream({ gte: ns, lte: ns + '/\uffff' })
				.on('data', function (data) {
					var index, path;
					if (keyPaths) {
						index = data.key.indexOf('/');
						path = (index !== -1) ? data.key.slice(index + 1) : null;
						if (!keyPaths.has(path)) return; // filtered
					}
					index = data.value.indexOf('.');
					result[data.key] = {
						stamp: Number(data.value.slice(0, index)),
						value: data.value.slice(index + 1)
					};
				}).on('error', def.reject).on('close', function () { def.resolve(result); });
			return def.promise;
		});
	}),

	__search: d(function (keyPath, value, callback) {
		return this.directDb(function (db) {
			var def = deferred(), stream = db.createReadStream();
			stream.on('data', function (data) {
				var index, result, recordKeyPath = resolveKeyPath(data.key), recordValue
				  , resolvedValue, ownerId, path;
				if (keyPath !== undefined) {
					if (!keyPath) {
						if (recordKeyPath) return;
					} else if (keyPath !== recordKeyPath) {
						return;
					}
				}
				index = data.value.indexOf('.');
				recordValue = data.value.slice(index + 1);
				if (value != null) {
					ownerId = data.key.split('/', 1)[0];
					path = data.key.slice(ownerId.length + 1) || null;
					resolvedValue = resolveValue(ownerId, path, recordValue);
					if (value !== resolvedValue) return;
				}
				result = callback(data.key, {
					stamp: Number(data.value.slice(0, index)),
					value: recordValue
				});
				if (result) stream.destroy();
			}).on('error', def.reject).on('close', def.resolve);
			return def.promise;
		});
	}),
	__searchComputed: d(function (keyPath, callback) {
		return this.computedDb(function (db) {
			var def = deferred()
			  , stream = db.createReadStream({ gte: keyPath + ':', lte: keyPath + ':\uffff' });

			stream.on('data', function (data) {
				var index, value, ownerId = data.key.slice(data.key.lastIndexOf(':') + 1);
				index = data.value.indexOf('.');
				value = data.value.slice(index + 1);
				if (value[0] === '[') value = parse(value);
				if (callback(ownerId, { value: value, stamp: Number(data.value.slice(0, index)) })) {
					stream.destroy();
				}
			}).on('error', def.reject).on('close', def.resolve);
			return def.promise;
		});
	}),<commit_msg>Update __searchComputed up to changes in dbjs-persistence<commit_after>'use strict';

var assign              = require('es5-ext/object/assign')
  , setPrototypeOf      = require('es5-ext/object/set-prototype-of')
  , camelToHyphen       = require('es5-ext/string/#/camel-to-hyphen')
  , d                   = require('d')
  , lazy                = require('d/lazy')
  , deferred            = require('deferred')
  , resolveKeyPath      = require('dbjs/_setup/utils/resolve-key-path')
  , resolve             = require('path').resolve
  , mkdir               = require('fs2/mkdir')
  , rmdir               = require('fs2/rmdir')
  , Storage             = require('dbjs-persistence/storage')
  , resolveValue        = require('dbjs-persistence/lib/resolve-direct-value')
  , filterComputedValue = require('dbjs-persistence/lib/filter-computed-value')

  , isArray = Array.isArray, create = Object.create, stringify = JSON.stringify, parse = JSON.parse
  , getOpts = { fillCache: false };

var LevelStorage = module.exports = function (driver, name/*, options*/) {
	if (!(this instanceof LevelStorage)) return new LevelStorage(driver, name, arguments[2]);
	Storage.call(this, driver, name, arguments[2]);
	this.dbPath = resolve(driver.dbPath, camelToHyphen.call(name));
};
setPrototypeOf(LevelStorage, Storage);

LevelStorage.prototype = Object.create(Storage.prototype, assign({
	constructor: d(LevelStorage),

	// Any data
	__getRaw: d(function (cat, ns, path) {
		if (cat === 'reduced') return this._getReduced_(ns + (path ? ('/' + path) : ''));
		if (cat === 'computed') return this._getComputed_(path, ns);
		return this._getDirect_(ns, path);
	}),
	__storeRaw: d(function (cat, ns, path, data) {
		if (cat === 'reduced') return this._storeReduced_(ns + (path ? ('/' + path) : ''), data);
		if (cat === 'computed') return this._storeComputed_(path, ns, data);
		return this._storeDirect_(ns, path, data);
	}),

	// Direct data
	__getObject: d(function (ownerId, objectPath, keyPaths) {
		var query = {
			gte: ownerId + (objectPath ? '/' + objectPath : ''),
			lte: ownerId + (objectPath ? '/' + objectPath : '') + '/\uffff'
		};
		return this._loadDirect_(query, keyPaths && function (ownerId, path) {
			if (!path) return true;
			return keyPaths.has(resolveKeyPath(ownerId + '/' + path));
		});
	}),
	__getAllObjectIds: d(function () {
		return this.directDb(function (db) {
			var def = deferred(), data = create(null);
			db.createReadStream().on('data', function (record) {
				var index = record.key.indexOf('/');
				if (index === -1) {
					index = record.value.indexOf('.');
					data[record.key] = {
						value: record.value.slice(index + 1),
						stamp: Number(record.value.slice(0, index))
					};
				}
			}).on('error', def.reject).on('close', function () { def.resolve(data); });
			return def.promise;
		});
	}),
	__getAll: d(function () { return this._loadDirect_(); }),

	// Reduced data
	__getReducedObject: d(function (ns, keyPaths) {
		return this.reducedDb(function (db) {
			var def, result;
			def = deferred();
			result = create(null);
			db.createReadStream({ gte: ns, lte: ns + '/\uffff' })
				.on('data', function (data) {
					var index, path;
					if (keyPaths) {
						index = data.key.indexOf('/');
						path = (index !== -1) ? data.key.slice(index + 1) : null;
						if (!keyPaths.has(path)) return; // filtered
					}
					index = data.value.indexOf('.');
					result[data.key] = {
						stamp: Number(data.value.slice(0, index)),
						value: data.value.slice(index + 1)
					};
				}).on('error', def.reject).on('close', function () { def.resolve(result); });
			return def.promise;
		});
	}),

	__search: d(function (keyPath, value, callback) {
		return this.directDb(function (db) {
			var def = deferred(), stream = db.createReadStream();
			stream.on('data', function (data) {
				var index, result, recordKeyPath = resolveKeyPath(data.key), recordValue
				  , resolvedValue, ownerId, path;
				if (keyPath !== undefined) {
					if (!keyPath) {
						if (recordKeyPath) return;
					} else if (keyPath !== recordKeyPath) {
						return;
					}
				}
				index = data.value.indexOf('.');
				recordValue = data.value.slice(index + 1);
				if (value != null) {
					ownerId = data.key.split('/', 1)[0];
					path = data.key.slice(ownerId.length + 1) || null;
					resolvedValue = resolveValue(ownerId, path, recordValue);
					if (value !== resolvedValue) return;
				}
				result = callback(data.key, {
					stamp: Number(data.value.slice(0, index)),
					value: recordValue
				});
				if (result) stream.destroy();
			}).on('error', def.reject).on('close', def.resolve);
			return def.promise;
		});
	}),
	__searchComputed: d(function (keyPath, value, callback) {
		return this.computedDb(function (db) {
			var def = deferred(), stream, query;
			if (keyPath) query = { gte: keyPath + ':', lte: keyPath + ':\uffff' };
			stream = db.createReadStream(query);

			stream.on('data', function (data) {
				var index, recordValue, ownerId = data.key.slice(data.key.lastIndexOf(':') + 1);
				index = data.value.indexOf('.');
				recordValue = data.value.slice(index + 1);
				if (recordValue[0] === '[') recordValue = parse(recordValue);
				if ((value != null) && !filterComputedValue(value, recordValue)) return;
				if (callback(ownerId, { value: recordValue, stamp: Number(data.value.slice(0, index)) })) {
					stream.destroy();
				}
			}).on('error', def.reject).on('close', def.resolve);
			return def.promise;
		});
	}),
{'fraction_non_alphanumeric': 0.18010372465818011, 'fraction_numerical': 0.004337576614804338, 'mean_word_length': 3.2956662616443904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4569120', 'n_tokens_mistral': 4014, 'n_tokens_neox': 3471, 'n_words': 752}