<commit_before>
var reserved = ["break","default","function","return","var","case","delete","if","switch","void","catch","do","in","this","while","const","else","instanceof","throw","with","continue","finally","let","try","debugger","for","new","typeof"];//reserved words cannot be used for property names

//Make a dictionary from text
function popDictJS(txt){
	//Takes a String, builds dictionary of all words in the file mapped to their counts. Returns dictionary.
	var user = {};
	var words = txt.split(" ");
	//remove any reserved words (I could remove the frequent words here too)
	var finalWords = [];
	for (i=0; i<words.length; i++){
		var safeWord = true; 
		for (j=0; j<reserved.length; j++){
			if (words[i]==reserved[j]){
				safeWord = false;
				break;
			}
		}
		if (safeWord){
			finalWords.push(words[i]);
		}
	}
	//remove . , ? ! : ; ) (  and  eol  and  make lowercase 
	for (i=0; i<finalWords.length; i++){
		var word = finalWords[i];
		console.log(word);
		word = word.replace(/(\r\n|\n|\S|\r)/gm,"");
		
		if (word.length>1){
			if (word.charAt(word.length-1) == "." || word.charAt(word.length-1) == "," || word.charAt(word.length-1) == "!" || word.charAt(word.length-1) == "?" || word.charAt(word.length-1) == ":" || word.charAt(word.length-1) == ";" || word.charAt(word.length-1) == ")"){
				//console.log("Found . , ? ! : ; in "+word);
				finalWords[i] = word.substr(0,word.length-1);
			}
			if(word.charAt(0) == "("){
				finalWords[i] = word.substr(1,word.length);
			}
		}
		finalWords[i] = finalWords[i].toLowerCase();
	}
	
	//put counts in dict
	for (i=0; i<finalWords.length; i++){
		var hf = false;<commit_msg>Add table borders<commit_after>
var reserved = ["break","default","function","return","var","case","delete","if","switch","void","catch","do","in","this","while","const","else","instanceof","throw","with","continue","finally","let","try","debugger","for","new","typeof"];//reserved words cannot be used for property names

//Make a dictionary from text
function popDictJS(txt){
	//Takes a String, builds dictionary of all words in the file mapped to their counts. Returns dictionary.
	var user = {};
	var words = txt.split(" ");
	//remove any reserved words (I could remove the frequent words here too)
	var finalWords = [];
	for (i=0; i<words.length; i++){
		var safeWord = true; 
		for (j=0; j<reserved.length; j++){
			if (words[i]==reserved[j]){
				safeWord = false;
				break;
			}
		}
		if (safeWord){
			finalWords.push(words[i]);
		}
	}
	//remove . , ? ! : ; ) (  and  eol  and  make lowercase 
	for (i=0; i<finalWords.length; i++){
		var word = finalWords[i];
		
		word = word.replace(/(\r\n|\n|\S|\r)/gm,"");
		console.log(word);
		if (word.length>1){
			if (word.charAt(word.length-1) == "." || word.charAt(word.length-1) == "," || word.charAt(word.length-1) == "!" || word.charAt(word.length-1) == "?" || word.charAt(word.length-1) == ":" || word.charAt(word.length-1) == ";" || word.charAt(word.length-1) == ")"){
				//console.log("Found . , ? ! : ; in "+word);
				finalWords[i] = word.substr(0,word.length-1);
			}
			if(word.charAt(0) == "("){
				finalWords[i] = word.substr(1,word.length);
			}
		}
		finalWords[i] = finalWords[i].toLowerCase();
	}
	
	//put counts in dict
	for (i=0; i<finalWords.length; i++){
		var hf = false;
{'fraction_non_alphanumeric': 0.22964763061968407, 'fraction_numerical': 0.009720534629404616, 'mean_word_length': 3.95187969924812, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2480150', 'n_tokens_mistral': 1291, 'n_tokens_neox': 1169, 'n_words': 243}