<commit_before>//parser file for the royalscript language

//table of transpiling functions

var funcTable = {
	"+":function(args){
		for (var i = args.length - 1; i >= 0; i--) {
			args[i] = args[i].call();
		};
		return args.join(" + ");
	},
	"-":true,
	"$":true,
	"*":true,
	"/":true,
	"%":true
};

//quick way of checking if a string is a primtive value.
var primTable = {
	"true":true,
	"false":true,
	"null":true,
	"0":true,
	"1":true,
	"2":true
};


//creates an abstract node for the AST
function AbsNode(sym){
	return {sym:sym,
		    children:[],
		    fn:FuncTable[sym],
		    call:function(){ return this.fn(this.children) }
	};
}

//basic node type for primitive values
function ValNode(value){
	return {
		val:value,
		call:function(){ return this.val }
	};
}

//parenthesis counter object to facilitate top down parsing.
function ParenCounter(){
	this["("] = 0;
	this[")"] = 0;
    this.put = function(token){
    	if(token in this) this[token] += 1;
    };
    this.offset = function(){
    	return this["("] - this[")"];
    };
}


var AST = (function(){
	//determines if string is a primitive value
	function isPrim(string){
		return /^"[^"]*"$|^[0-9]+$|true|false|null/.test(string);
	}

	function isFunc(string){
		//allows dot notation in function names
		//checks for opers too
		return /^[\+\-\*\%\/><=!?&^@#$~_]+$|^[a-zA-Z.]+$/.test(string);
	}

	function isOp(string){
		return /^[\+\-\*\%\/><=!?&^@#$~_]+$/.test(string);
	}

	function AST(tokens){
		this.name = null;
		this.children = [];
		var counter = new ParenCounter();
		//args for parsing current child node
		var args = [];
		for (var i = 0; i < tokens.length-1; i++) {
			counter.put(tokens[i]);
			//console.log(counter.offset());
			switch(counter.offset()) {
				case -1:
				    throw "INVALID Parenthsis";
				    break;
				case 0:
				    if(tokens[i+1] === "("){
				    	this.name = tokens[i];
				    	counter.put("(");
				    	i += 1; //skips parenthesis token
				    }
				    //else if(i === tokens.length-1 && tokens[i] === ")")
				    else {
				    	throw "Error token not starting function";
				    }
				    break;
				case 1:
				    if(tokens[i+1] === "("){
				    	args.push(tokens[i]);
				    	args.push("(");
				    	counter.put("(");
				    	i += 1; //skips parenthesis token
				    }
				    else {
				    	this.children.push(tokens[i]);
				    }
				    break;
				case 2:
				    if(tokens[i+1] === ")"){
				    	//skips closing parenthesis and closes child AST node.
				    	args.push(tokens[i]);
				    	args.push(")");
				    	counter.put(")");
				    	if(args[0] === ")") args.splice(0, 1);
				    	this.children.push(new AST(args.slice()));
				    	args = [];
				    	i += 1;
				    }
				default:
				    args.push(tokens[i]);
				    break;
			}
		};
	}
	return AST;
})();

exports.AST = AST;<commit_msg>fixed parsing bug<commit_after>//parser file for the royalscript language

//table of transpiling functions

var funcTable = {
	"+":function(args){
		for (var i = args.length - 1; i >= 0; i--) {
			args[i] = args[i].call();
		};
		return args.join(" + ");
	},
	"-":true,
	"$":true,
	"*":true,
	"/":true,
	"%":true
};

//quick way of checking if a string is a primtive value.
var primTable = {
	"true":true,
	"false":true,
	"null":true,
	"0":true,
	"1":true,
	"2":true
};


//creates an abstract node for the AST
function AbsNode(sym){
	return {sym:sym,
		    children:[],
		    fn:FuncTable[sym],
		    call:function(){ return this.fn(this.children) }
	};
}

//basic node type for primitive values
function ValNode(value){
	return {
		val:value,
		call:function(){ return this.val }
	};
}

//parenthesis counter object to facilitate top down parsing.
function ParenCounter(){
	this["("] = 0;
	this[")"] = 0;
    this.put = function(token){
    	if(token in this) this[token] += 1;
    };
    this.offset = function(){
    	return this["("] - this[")"];
    };
}


var AST = (function(){
	//determines if string is a primitive value
	function isPrim(string){
		return /^"[^"]*"$|^[0-9]+$|true|false|null/.test(string);
	}

	function isFunc(string){
		//allows dot notation in function names
		//checks for opers too
		return /^[\+\-\*\%\/><=!?&^@#$~_]+$|^[a-zA-Z.]+$/.test(string);
	}

	function isOp(string){
		return /^[\+\-\*\%\/><=!?&^@#$~_]+$/.test(string);
	}

	function AST(tokens){
		this.name = null;
		this.children = [];
		var counter = new ParenCounter();
		//args for parsing current child node
		var args = [];
		for (var i = 0; i < tokens.length-1; i++) {
			counter.put(tokens[i]);
			//console.log(counter.offset());
			switch(counter.offset()) {
				case -1:
				    throw "INVALID Parenthsis";
				    break;
				case 0:
				    if(tokens[i+1] === "("){
				    	this.name = tokens[i];
				    	counter.put("(");
				    	i += 1; //skips parenthesis token
				    }
				    else {
				    	throw "Error token not starting function";
				    }
				    break;
				case 1:
				    if(tokens[i+1] === "("){
				    	args.push(tokens[i]);
				    	args.push("(");
				    	counter.put("(");
				    	i += 1; //skips parenthesis token
				    	if(tokens[i+1] === ")"){
				    		//parses empty call functions f()
				    		args.push(")");
				    		counter.put(")");
				    		if(args[0] === ")") args.splice(0, 1);
				    		this.children.push(new AST(args.slice()));
				    		args = [];
				    		i += 1;
				    	}
				    }
				    else {
				    	this.children.push(tokens[i]);
				    }
				    break;
				case 2:
				    if(tokens[i+1] === ")"){
				    	//skips closing parenthesis and closes child AST node.
				    	args.push(tokens[i]);
				    	args.push(")");
				    	counter.put(")");
				    	if(args[0] === ")") args.splice(0, 1);
				    	this.children.push(new AST(args.slice()));
				    	args = [];
				    	i += 1;
				    }
				default:
				    args.push(tokens[i]);
				    break;
			}
		};
	}
	return AST;
})();

exports.AST = AST;
{'fraction_non_alphanumeric': 0.2029769959404601, 'fraction_numerical': 0.009472259810554804, 'mean_word_length': 2.2650469353948095, 'pattern_counts': {'":': 24, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4841597', 'n_tokens_mistral': 2569, 'n_tokens_neox': 2107, 'n_words': 483}