<commit_before>////////////////////////////////////////////////////////////////////////////////
// Media Data
//
// Based on Flock (https://github.com/wwidd/flock).
// Until the server stores data in a NoSQL way, the client
// has to build the NoSQL cache in this class. This makes page initialization
// on high media and tag count a bit slow (2-3s on netbooks, tablets).
// Once SQLite is ditched on the server, the JSON can come unaltered and
// there would be no need to use Flock for building the cache.
////////////////////////////////////////////////////////////////////////////////
/*global jOrder, flock, escape */
var app = app || {};

app.data = function (data, jOrder, flock, services) {
	// initializing cache
	data.cache = data.cache || flock();
	
	// checks whether an object has any properties
	function isEmpty(obj) {
		var key;
		for (key in obj) {
			if (obj.hasOwnProperty(key)) {
				return false;
			}
		}
		return true;
	}
	
	// extracts filename from path
	function splitPath(path) {
		var bits = path.split('/');
		return {
			file: bits.pop()
		};
	}
	
	// preprocesses video metadata
	function preprocess(json) {
		var i, row, fileInfo;
		for (i = 0; i < json.length; i++) {
			row = json[i];
			fileInfo = splitPath(row.path);
			row.file = fileInfo.file;
			row.lfile = fileInfo.file.toLowerCase();
		}
		return json;
	}
	
	data.media = function () {
		var
		
		// FIFO stack search results
		stack = [{
			term: '',
			data: {}
		}],
		
		self = {
			stack: function () {
				return stack;
			},

			// initializes data object: calls service, populates cache
			init: function (handler) {
				handler = handler || function () {};
				
				// calling service
				services.media.get('', function (json) {
					json = preprocess(json.data);
					
					var cache = data.cache,
							i, row,
							j, tag,
							tags,
							table;
					
					// setting up datastore roots
					cache.set('tag', {});
					cache.set('media', {});
					cache.set('keyword', {});
					cache.set('kind', {});
					cache.set('name', {});
					cache.set('rating', {});
					cache.set('field', {});
					cache.set('search', {});
					
					// loading media data into cache
					for (i = 0; i < json.length; i++) {
						row = json[i];		// quick reference
						tags = row.tags;	// backup of flat tags
						row.tags = {};		// tags field will be rewritten
						
						// storing media node in cache
						cache.set(['media', row.mediaid], row);
						
						// setting properties
						self.setRating(row);
						for (j = 0; j < tags.length; j++) {
							self.addTag(row, tags[j]);
						}
					}
					
					// setting up jOrder table
					// required for paging
					table = jOrder(preprocess(json))
						.index('pager', ['lfile'], {ordered: true, grouped: true, type: jOrder.string});
						
					// initializing stack
					stack = [{
						term: '',
						data: {
							cache: cache,
							table: table
						}
					}];
					
					handler();
				});
				return self;
			},

			// filters current state further
			// - expression: filter expression, comma-separated
			// returns a flag indicating if the result set changed
			filter: function (expression) {
				if (!expression.length) {
					stack.splice(0, stack.length - 1);
					return true;
				}
				
				var terms = (',' + expression).split(','), term,
						tstack = jOrder.shallow(stack).reverse(),
						tags, matches, hits,
						i,
						result = false;

				// stack can't be longer than terms
				if (stack.length > terms.length) {
					stack.splice(0, stack.length - terms.length);
					result = true;
				}
				
				// skipping already processed search terms
				skipper:
				for (i = 1; i < terms.length; i++) {
					if (i >= stack.length) {
						// stack is shorter than terms, continuing below
						result = true;
						break skipper;
					} else if (terms[i] !== tstack[i].term) {
						// cutting stack at first non-matching term
						stack.splice(0, stack.length - i);
						result = true;
						break skipper;
					}
				}
				
				for (; i < terms.length; i++) {
					// taking next term
					term = terms[i];
					
					// tags matching the entered string
					tags = data.cache.multiget(['search'].concat(term.toLowerCase().split('')).concat(['', 'tag', 'tag']));
					
					// ids of media entries having one or more of the matched tags
					matches = data.cache.multiget(['tag', tags, 'media', '*'], {mode: flock.keys});
					
					// intersection of tag matches and previous hits
					hits = stack[0].data.cache.multiget(['media', matches], {mode: flock.both});
	
					// adding search hits to stack
					stack.unshift({
						term: term,
						data: {
							cache: flock({
								media: hits
							}),
							table: jOrder(jOrder.values(hits))
								.index('pager', ['lfile'], {ordered: true, grouped: true, type: jOrder.string})
						}
					});
				}
				
				return result;
			},
			
			// retrieves a reference to the data associated with a media entry
			getRow: function (mediaid) {
				return data.cache.get(['media', mediaid]) || {};
			},
			
			// adds a tag to media entry
			addTag: function (row, tag) {
				var cache = data.cache,
						mediaid = row.mediaid,
						ref, tmp,
						path_media_tag = ['media', mediaid, 'tags', tag],
						path_tag = ['tag', tag];

				if (cache.get(path_media_tag) !== tag) {<commit_msg>performance opt. for first search term<commit_after>////////////////////////////////////////////////////////////////////////////////
// Media Data
//
// Based on Flock (https://github.com/wwidd/flock).
// Until the server stores data in a NoSQL way, the client
// has to build the NoSQL cache in this class. This makes page initialization
// on high media and tag count a bit slow (2-3s on netbooks, tablets).
// Once SQLite is ditched on the server, the JSON can come unaltered and
// there would be no need to use Flock for building the cache.
////////////////////////////////////////////////////////////////////////////////
/*global jOrder, flock, escape */
var app = app || {};

app.data = function (data, jOrder, flock, services) {
	// initializing cache
	data.cache = data.cache || flock();
	
	// checks whether an object has any properties
	function isEmpty(obj) {
		var key;
		for (key in obj) {
			if (obj.hasOwnProperty(key)) {
				return false;
			}
		}
		return true;
	}
	
	// extracts filename from path
	function splitPath(path) {
		var bits = path.split('/');
		return {
			file: bits.pop()
		};
	}
	
	// preprocesses video metadata
	function preprocess(json) {
		var i, row, fileInfo;
		for (i = 0; i < json.length; i++) {
			row = json[i];
			fileInfo = splitPath(row.path);
			row.file = fileInfo.file;
			row.lfile = fileInfo.file.toLowerCase();
		}
		return json;
	}
	
	data.media = function () {
		var
		
		// FIFO stack search results
		stack = [{
			term: '',
			data: {}
		}],
		
		self = {
			stack: function () {
				return stack;
			},

			// initializes data object: calls service, populates cache
			init: function (handler) {
				handler = handler || function () {};
				
				// calling service
				services.media.get('', function (json) {
					json = preprocess(json.data);
					
					var cache = data.cache,
							i, row,
							j, tag,
							tags,
							table;
					
					// setting up datastore roots
					cache.set('tag', {});
					cache.set('media', {});
					cache.set('keyword', {});
					cache.set('kind', {});
					cache.set('name', {});
					cache.set('rating', {});
					cache.set('field', {});
					cache.set('search', {});
					
					// loading media data into cache
					for (i = 0; i < json.length; i++) {
						row = json[i];		// quick reference
						tags = row.tags;	// backup of flat tags
						row.tags = {};		// tags field will be rewritten
						
						// storing media node in cache
						cache.set(['media', row.mediaid], row);
						
						// setting properties
						self.setRating(row);
						for (j = 0; j < tags.length; j++) {
							self.addTag(row, tags[j]);
						}
					}
					
					// setting up jOrder table
					// required for paging
					table = jOrder(preprocess(json))
						.index('pager', ['lfile'], {ordered: true, grouped: true, type: jOrder.string});
						
					// initializing stack
					stack = [{
						term: '',
						data: {
							cache: cache,
							table: table
						}
					}];
					
					handler();
				});
				return self;
			},

			// filters current state further
			// - expression: filter expression, comma-separated
			// returns a flag indicating if the result set changed
			filter: function (expression) {
				if (!expression.length) {
					stack.splice(0, stack.length - 1);
					return true;
				}
				
				var terms = (',' + expression).split(','), term,
						tstack = jOrder.shallow(stack).reverse(),
						tags, matches, hits,
						i,
						result = false;

				// stack can't be longer than terms
				if (stack.length > terms.length) {
					stack.splice(0, stack.length - terms.length);
					result = true;
				}
				
				// skipping already processed search terms
				skipper:
				for (i = 1; i < terms.length; i++) {
					if (i >= stack.length) {
						// stack is shorter than terms, continuing below
						result = true;
						break skipper;
					} else if (terms[i] !== tstack[i].term) {
						// cutting stack at first non-matching term
						stack.splice(0, stack.length - i);
						result = true;
						break skipper;
					}
				}
				
				// processing terms that differ from what's already on the stack
				for (; i < terms.length; i++) {
					// taking next term
					term = terms[i];
					
					// acquiring tags matching the entered string
					tags = data.cache.multiget(['search'].concat(term.toLowerCase().split('')).concat(['', 'tag', 'tag']));
					
					// acquiring search hits
					if (stack.length === 1) {
						// performing direct query on first search term
						hits = data.cache.multiget(['tag', tags, 'media', '*'], {mode: flock.both});
					} else {
						// performing two-stage query on subsequent terms
						// first, taking _all_ media ids where tags match, then
						// matching them against previous search results
						matches = data.cache.multiget(['tag', tags, 'media', '*'], {mode: flock.keys});
						hits = stack[0].data.cache.multiget(['media', matches], {mode: flock.both});
					}
	
					// adding search hits to stack
					stack.unshift({
						term: term,
						data: {
							cache: flock({
								media: hits
							}),
							table: jOrder(jOrder.values(hits))
								.index('pager', ['lfile'], {ordered: true, grouped: true, type: jOrder.string})
						}
					});
				}
				
				return result;
			},
			
			// retrieves a reference to the data associated with a media entry
			getRow: function (mediaid) {
				return data.cache.get(['media', mediaid]) || {};
			},
			
			// adds a tag to media entry
			addTag: function (row, tag) {
				var cache = data.cache,
						mediaid = row.mediaid,
						ref, tmp,
						path_media_tag = ['media', mediaid, 'tags', tag],
						path_tag = ['tag', tag];

				if (cache.get(path_media_tag) !== tag) {
{'fraction_non_alphanumeric': 0.17017035157665822, 'fraction_numerical': 0.002084088437839797, 'mean_word_length': 1.9910569105691056, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2231553', 'n_tokens_mistral': 4588, 'n_tokens_neox': 3261, 'n_words': 1018}