<commit_before>'use strict';

var markovGenerator = (function() {

	var EOS = [ "." , "?", "!" ];
	var min_sentence_words = 5;
	var max_sentence_words = 30;

	function getDictItemByKey(dict, key) {
		for (var i = 0, len = dict.length; i < len; i++) {
			if (dict[i].key === key) return dict[i];
		}
		return null;
	}
	
	function getSeed(dict, chainSize) {
		// find a seed that starts with a capital letter
		var seed = [];
		var seedAttempt = 0;
		while (true) {
			seedAttempt++;
			var seed = dict[Math.floor(Math.random() * dict.length)].words;

			var firstChar = seed[0].charAt(0);
			if (firstChar === firstChar.toUpperCase()) {
				var generatedWords = seed;
				break;
			}

			// too many loops, just use this one and move on.
			if (seedAttempt > 20) {
				var generatedWords = seed;
				break;
			}
		}
		return seed;
	}
	
	function getWords(words, dict, chainSize) {
		while (true) {
			var last_words = words.slice(-1 * chainSize);
			var match = getDictItemByKey(dict, last_words.join('/'));

			if (match === null) break;

			var next_opts = match.next;
			var rand_next = next_opts[Math.floor(Math.random() * next_opts.length)];

			if (typeof(rand_next) === 'undefined') break;

			words.push(rand_next);
			if (isFinalCharEOS(rand_next)) break;
		}
		return words;
	}

	function generateSentence(dict, chainSize) {
		console.log("generating sentence");

		var seed = getSeed(dict, chainSize);
		var generatedWords = getWords(seed, dict, chainSize);

		var sentence = generatedWords.join(' ');

		var wordCount= generatedWords.length;
		if (wordCount < min_sentence_words || wordCount > max_sentence_words) {
			sentence = generateSentence(dict, chainSize);
		}

		if (!isFinalCharEOS(sentence)) {
			sentence = sentence + "."; // append a . to finish it
		}
		return sentence;
	}

	function isFinalCharEOS(str) {
		var final_char = str.charAt(str.length - 1); 
		if (EOS.indexOf(final_char) > -1) {
			return true;
		}
		return false;
	}

	return { generateSentence : generateSentence }

})();
<commit_msg>Remove min/max words option from generateSentence (might add back later, but it needs more cleanup first)<commit_after>'use strict';

var markovGenerator = (function() {

	var EOS = [ "." , "?", "!" ];

	function getDictItemByKey(dict, key) {
		for (var i = 0, len = dict.length; i < len; i++) {
			if (dict[i].key === key) return dict[i];
		}
		return null;
	}
	
	function getSeed(dict, chainSize) {
		// find a seed that starts with a capital letter
		var seed = [];
		var seedAttempt = 0;
		while (true) {
			seedAttempt++;
			var seed = dict[Math.floor(Math.random() * dict.length)].words;

			var firstChar = seed[0].charAt(0);
			if (firstChar === firstChar.toUpperCase()) {
				var generatedWords = seed;
				break;
			}

			// too many loops, just use this one and move on.
			if (seedAttempt > 20) {
				var generatedWords = seed;
				break;
			}
		}
		return seed;
	}
	
	function getWords(words, dict, chainSize) {
		while (true) {
			var last_words = words.slice(-1 * chainSize);
			var match = getDictItemByKey(dict, last_words.join('/'));
			if (match === null) break;

			var next_opts = match.next;
			var rand_next = next_opts[Math.floor(Math.random() * next_opts.length)];

			if (typeof(rand_next) === 'undefined') break;

			words.push(rand_next);
			if (isFinalCharEOS(rand_next)) break;
		}
		return words;
	}

	function generateSentence(dict, chainSize) {
		console.log("generating sentence");

		var seed = getSeed(dict, chainSize);
		var generatedWords = getWords(seed, dict, chainSize);

		var sentence = generatedWords.join(' ');

		if (!isFinalCharEOS(sentence)) {
			sentence = sentence + "."; // append a . to finish it
		}
		return sentence;
	}

	function isFinalCharEOS(str) {
		var final_char = str.charAt(str.length - 1); 
		if (EOS.indexOf(final_char) > -1) {
			return true;
		}
		return false;
	}

	return { generateSentence : generateSentence }

})();

{'fraction_non_alphanumeric': 0.14256515074092999, 'fraction_numerical': 0.005365355135411344, 'mean_word_length': 3.4087837837837838, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '485018', 'n_tokens_mistral': 1556, 'n_tokens_neox': 1311, 'n_words': 348}