<commit_before>'use strict';

var markovDictionaryBuilder = (function() {

	function buildDict(wordSet, chainSize) {

		console.log("building dictionary from " + wordSet.length + " words with a chain size of " + chainSize);

		var map = [];
		var dict = [];
		for (var i = 0, len = wordSet.length - chainSize; i < len; i++) {
			
			var end = i + parseFloat(chainSize); 
			var workingSet = wordSet.slice(i, end+1);
			var k = workingSet.slice(0, workingSet.length-1);
			var n = workingSet.slice(-1);
			
			var dictItem = {
				'key' : k.join('/').toLowerCase(),
				'words' : k,
				'next' : n
			};
			
			var mi = map[dictItem.key];
			if (typeof(mi) === 'undefined') {
				map[dictItem.key] = dict.length;
				dict.push(dictItem);
			} else {
				dict[mi].next.push(dictItem.next[0]);
			}
		}
		return dict;
	}

	return { 
		buildDict : buildDict
	};

})();<commit_msg>HACK: Prevent js keywords in the map from being parsed as native code instead of strings (affects the map array when using chainSize of 1). That is, the word "every", when added to the map and then retrieved from the map, is treated as the function "every()"<commit_after>'use strict';

var markovDictionaryBuilder = (function() {

	function buildDict(wordSet, chainSize) {

		console.log("building dictionary from " + wordSet.length + " words with a chain size of " + chainSize);

		var map = [];
		var dict = [];
		for (var i = 0, len = wordSet.length - chainSize; i < len; i++) {
			
			var end = i + parseFloat(chainSize); 
			var workingSet = wordSet.slice(i, end+1);
			var k = workingSet.slice(0, workingSet.length-1);
			var n = workingSet.slice(-1);
			
			var dictItem = {
				'key' : k.join('/').toLowerCase(),
				'words' : k,
				'next' : n
			};
			
			//TODO: cheap hack to prevent keywords (e.g. "every") from being parsed as native code (e.g. function)
			// revisit this issue, because this is bloody stupid.
			var mapKey = "*" + dictItem.key;
			
			var mi = map[mapKey];
			if (typeof(mi) === 'undefined') {
				var dictIndex = dict.length;
				dict.push(dictItem);
				map[mapKey] = dictIndex;
			} else {
				dict[mi].next.push(dictItem.next[0]);
			}
		}
		return dict;
	}

	return { 
		buildDict : buildDict
	};

})();
{'fraction_non_alphanumeric': 0.15006759801712483, 'fraction_numerical': 0.005858494817485353, 'mean_word_length': 2.978494623655914, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5811202', 'n_tokens_mistral': 876, 'n_tokens_neox': 742, 'n_words': 220}