<commit_before>/*
TODO:
 fix:
	' sin(x)' => ∘s∘i∘n∘(x)


*/

var msg={
	"latexParse":"Unable to parse LaTeX string",
	"parenMismatch":"There are mismatched parentheses"
};

var types = {
	"number": 1,
	"operator": 2,
	"paren": 3,
	"variable": 4,
	"function": 5
};

var parse = (function (language) {
	O(1, "parse - build");
	//Begin parse building space.
	//This context will be accessible to parse()
	
	
	var names = ["none","num","op","paren","var"];

	

	//Operator characters
	//TODO: calculate programmatically
	
	var ochars=":>-.+~!^%/*<=&|?,;±∘'∫∑∫√¬_$";
	
	//TODO: Allow 1e+2 format
	var nummustbe="1234567890.";
	var parenmustbe="([{}\"])";
	var varcannotbe=ochars+parenmustbe+nummustbe;
	var match=[0,
		function(e){
			return !isNaN(e);
		},
		function(e){
			if(operators[e]){
				return true;
			}
			return false;
			//return ochars.indexOf(e)!==-1;
		},
		function(e){
			return e.length === 1 && parenmustbe.indexOf(e)!=-1;<commit_msg>fix bug: TypeError: 'undefined' is not an object (evaluating 'e.length')<commit_after>/*
TODO:
 fix:
	' sin(x)' => ∘s∘i∘n∘(x)


*/

var msg={
	"latexParse":"Unable to parse LaTeX string",
	"parenMismatch":"There are mismatched parentheses"
};

var types = {
	"number": 1,
	"operator": 2,
	"paren": 3,
	"variable": 4,
	"function": 5
};

var parse = (function (language) {
	O(1, "parse - build");
	//Begin parse building space.
	//This context will be accessible to parse()
	
	
	var names = ["none","num","op","paren","var"];

	

	//Operator characters
	//TODO: calculate programmatically
	
	var ochars=":>-.+~!^%/*<=&|?,;±∘'∫∑∫√¬_$";
	
	//TODO: Allow 1e+2 format
	var nummustbe="1234567890.";
	var parenmustbe="([{}\"])";
	var varcannotbe=ochars+parenmustbe+nummustbe;
	var match=[0,
		function(e){
			return !isNaN(e);
		},
		function(e){
			if(operators[e]){
				return true;
			}
			return false;
			//return ochars.indexOf(e)!==-1;
		},
		function(e){
			return e && e.length === 1 && parenmustbe.indexOf(e)!=-1;
{'fraction_non_alphanumeric': 0.21719226856561547, 'fraction_numerical': 0.022380467955239063, 'mean_word_length': 3.880893300248139, 'pattern_counts': {'":': 16, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3325602', 'n_tokens_mistral': 899, 'n_tokens_neox': 787, 'n_words': 177}