<commit_before>//
//  XPINSCompiler.cpp
//  XPINS
//
//  Created by <NAME> on 9/7/13.
//  Copyright (c) 2013 <NAME>. All rights reserved.
//

#include "XPINSCompiler.h"
using namespace std;

const int kMajor=0;
const int kMinor=1;

namespace XPINSCompileUtil {
	string strRepresentationOfInt(int);
	bool stringsMatch(int,string,string);
	int readVarIndex(string,int *,char);
}

//Compile Script
bool XPINSCompiler::compileScript(string* input){
	string scriptText=*input;
	cout<<endl<<"Validating Version..."<<endl;
	checkVersion(&scriptText);
	cout<<"Version Compatilbe!";
	cout<<endl<<"About To Compile Script:\n"<<scriptText<<endl;
	if(!removeComments(&scriptText))return false;
	cout<<endl<<"Comments and ';'s Removed:\n"<<scriptText<<endl;
	if(!renameFunctions(&scriptText))return false;
	cout<<endl<<"Functions Renamed:\n"<<scriptText<<endl;
	if(!renameTypes(&scriptText))return false;
	cout<<endl<<"Variable Types Renamed:\n"<<scriptText<<endl;
	if(!renameVars(&scriptText))return false;
	cout<<endl<<"Variables Renamed:\n"<<scriptText<<endl;
	if(!cleanUp(&scriptText))return false;
	cout<<endl<<"Cleaned Up:\n"<<scriptText;
	*input=scriptText;
	return true;
}

//Compiler Steps
bool XPINSCompiler::checkVersion(string* script){
	string input=*script;
	for(int i=0;i<input.length();i++){
		if(input[i]=='@'&&input[i+1]=='C'&&input[i+2]=='O'&&input[i+3]=='M'&&input[i+4]=='P'&&input[i+5]=='I'&&input[i+6]=='L'&&input[i+7]=='E'&&input[i+8]=='R'){
			while(i<input.length()&&input[i]!='[')i++;
			if(i==input.length()){
				cout<<"SCRIPT MISSING VERSION. EXITING";
				return false;;
			}
			int MAJOR=XPINSCompileUtil::readVarIndex(input, &i, '.');
			int MINOR=XPINSCompileUtil::readVarIndex(input, &i, ']');
			if(MAJOR!=kMajor){
				cout<<"INCOMPATIBLE VERSION. EXITING";
				return false;
			}
			if(MINOR<kMinor){
				cout<<"INCOMPATIBLE VERSION. EXITING";
				return false;
			}
			while(i<input.length()&&input[i]!='\n')i++;
			i++;
			string output="";
			while (i<input.length()) {
				output+=input[i++];
			}
			*script=output;
			return true;
		}
	}
	cout<<"SCRIPT MISSING VERSION. EXITING";
	return false;
}
bool XPINSCompiler::removeComments(string* text){
	string input=*text;
	string output="";
	char ch;
	for(int i=0;i<input.length();i++){
		ch=input[i];
		if(ch=='/'){// all '/'s will be removed, any meaningful ones denote a comment
			i++;
			ch=input[i];
			if(ch=='*'){//multi line comments
				while (input[i]!='*'||input[i+1]!='/')i++;
				i++;
			}
			else if(ch=='/'){// single line comments
				while (input[i+1]!='\n'||i+1==input.length())i++;
			}
		}
		else if(ch!=';') output+=ch;//No semicolons :(
	}
	bool hitFirstEND=false;
	string intermediate1=""+output;
	output="";
	//Double Check Ending (strip after 2nd @END)
	for(int j=0;j<intermediate1.length();j++){
		if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
			if(hitFirstEND){
				output+="@END\n";
				break;
			}
			else hitFirstEND=true;
		}
		output+=intermediate1[j];
	}
	*text=output;
	return true;
}

bool XPINSCompiler::renameFunctions(string *text){
	string input=*text;
	string output="";
	char ch;
	int i=0;
	//Locate Function block
	while(input[i]!='@'||input[i+1]!='F'){
		i++;
	}
	if(input[i+1]!='F'||input[i+2]!='U'||input[i+3]!='N'||input[i+4]!='C'){
		cerr<<"Invalid Script: Missing @FUNC.\nEXITING";
		return false;
	}
	i+=5;
	string intermediate1="";
	string intermediate2="";
	//Initialize first intermediate
	int j=0;
	while (input[j]!='@'||input[j+1]!='C') {
		j++;
	}
	for(;j<input.length();j++){
		ch=input[j];
		intermediate1+=ch;
	}
	//Do the actual work
	int x=1;
	while(input[i+1]!='@'){
		if(i==input.length()){
			cerr<<"Invalid Script: Missing @END.\nEXITING";
			return false;;
		}
		while (input[i]!=' '){
			i++;
		}
		i++;
		//read function name
		string functionName;
		for(j=i;j<input.length()&&input[j]!='(';j++){
			functionName+=input[j];
		}
		//Replace function name
		intermediate2="";
		j=0;
		while(true){
			//Get to next line/END
			while(j<intermediate1.length()&&intermediate1[j]!='\n'){
				intermediate2+=intermediate1[j];
				j++;
			}
			//IF END CLEAN UP
			if(j>=intermediate1.length()){
				intermediate2+="@END";
				break;
			}
			intermediate2+=intermediate1[j];
			j++;
			//CHECK FOR @END
			if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
				intermediate2+="@END";
				break;
			}
			//Find start of function
			while(intermediate1[j]!='#'){
				intermediate2+=intermediate1[j];
				j++;
			}
			intermediate2+=intermediate1[j];
			j++;
			//Check for match
			if(XPINSCompileUtil::stringsMatch(j, intermediate1, functionName)){
				intermediate2+='F';
				intermediate2+=XPINSCompileUtil::strRepresentationOfInt(x);
				while(intermediate1[j]!='('){//Find '('
					j++;
				}
			}
		}
		//Get ready for next loop
		intermediate1=""+intermediate2;
		x++;
		while (input[i]!='\n')i++;
	}
	//Double Check Ending (strip after @END)
	for(j=0;j<intermediate1.length();j++){
		if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
			output+="@END";
			break;
		}
		output+=intermediate1[j];
	}
	//output=intermediate1;
	*text=output;
	return true;
}
bool XPINSCompiler::renameTypes(string *text){
	string input=*text;
	string output="";
	string intermediate1="";
	//Rename Types
	for(int i=0;i<input.length();i++){
		if((input[i]=='B'||input[i]=='I'||input[i]=='F'||input[i]=='V'||input[i]=='*')&&input[i-1]=='\n'){
			intermediate1+=input[i];
			while (input[i]!=' ')i++;
		}
		intermediate1+=input[i];
	}
	//Double Check Ending (strip after @END)
	for(int i=0;i<intermediate1.length();i++){
		if(intermediate1[i]=='@'&&intermediate1[i+1]=='E'&&intermediate1[i+2]=='N'&&intermediate1[i+3]=='D'){
			output+="@END\n";
			break;
		}
		output+=intermediate1[i];
	}
	*text=output;
	return true;
}
bool XPINSCompiler::renameVars(string *text){
	string input=*text;
	string output="";
	char ch;
	int i=0;
	//Locate Code block
	while(input[i]!='@'||input[i+1]!='C'){
		i++;
	}
	if(input[i+1]!='C'||input[i+2]!='O'||input[i+3]!='D'||input[i+4]!='E'){
		cerr<<"Invalid Script: Missing @CODE.\nEXITING";
		return false;;
	}
	i+=5;
	//Initialize intermediates
	string intermediate1=""+input;
	string intermediate2="";
	int j=0;
	//Do the actual work
	int xb=-1;
	int xi=-1;
	int xf=-1;
	int xv=-1;
	int xp=-1;
	char varType='P';
	while(i<input.length()&&(input[i]!='@'||input[i+1]!='E')){
		if(input[i-1]=='\n'&&(input[i]=='B'||input[i]=='I'||input[i]=='F'||input[i]=='V'||input[i]=='*'))
		{
			//determine new variable name
			varType=input[i]=='*'?'P':input[i];
			int varNum=0;
			switch (varType) {
				case 'B':
					varNum=++xb;
					break;
				case 'I':
					varNum=++xi;
					break;
				case 'F':
					varNum=++xf;
					break;
				case 'V':
					varNum=++xv;
					break;
				case 'P':
					varNum=++xp;
					break;
				default: varType='P';varNum=++xp;
			}
			//read Var name
			string varName;
			for(i+=3;i<input.length()&&input[i]!='=';i++){
				varName+=input[i];
			}
			//Replace Var name
			intermediate2="";
			j=0;
			int blockCount=0;
			while(j<intermediate1.length()){
				//CHECK FOR @END
				if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
					intermediate2+="@END";
					break;
				}
				//Check for block end
				if(intermediate1[j]=='{')blockCount++;
				else if(intermediate1[j]=='}'){
					if(blockCount==0){
						while(!(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D')&&j+3<intermediate1.length())intermediate2+=intermediate1[j];
						intermediate2+="@END";
						break;
					}
					else blockCount--;
					
				}
				//Find start of Var
				if(intermediate1[j]=='$'){
					intermediate2+='$';
					j++;
					//Check for match
					if(XPINSCompileUtil::stringsMatch(j, intermediate1, varName)){
						intermediate2+=varType;
						intermediate2+=XPINSCompileUtil::strRepresentationOfInt(varNum);
						while(intermediate1[j]!=')'&&intermediate1[j]!='='&&intermediate1[j]!=','&&intermediate1[j]!=']'){//Find '('
							j++;
						}
					}
				}
				else{
					intermediate2+=intermediate1[j];
					j++;
				}
			}
			intermediate1=""+intermediate2;
		}
		i++;
	}
	//Double Check Ending (strip after @END)
	for(j=0;j<intermediate1.length();j++){
		if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
			output+="@END\n";
			break;
		}
		output+=intermediate1[j];
	}
	*text=output;
	return true;
}
bool XPINSCompiler::cleanUp(string *text){
	string input=*text;
	string output="";
	string intermediate1="";
	//Remove excess whitespace
	for(int i=0;i<input.length();i++){
		if(input[i]!='\n'||input[i+1]!='\n')intermediate1+=input[i];
	}
	//Double Check Ending (strip after @END)
	for(int i=0;i<intermediate1.length();i++){
		if(intermediate1[i]=='@'&&intermediate1[i+1]=='E'&&intermediate1[i+2]=='N'&&intermediate1[i+3]=='D'){
			output+="@END\n";
			break;
		}
		output+=intermediate1[i];
	}
	*text=output;
	return true;
}
//UTILTY FUNCTIONS<commit_msg>fixed whitespace issue in compiler<commit_after>//
//  XPINSCompiler.cpp
//  XPINS
//
//  Created by <NAME> on 9/7/13.
//  Copyright (c) 2013 <NAME>. All rights reserved.
//

#include "XPINSCompiler.h"
using namespace std;

const int kMajor=0;
const int kMinor=1;

namespace XPINSCompileUtil {
	string strRepresentationOfInt(int);
	bool stringsMatch(int,string,string);
	int readVarIndex(string,int *,char);
}

//Compile Script
bool XPINSCompiler::compileScript(string* input){
	string scriptText=*input;
	cout<<endl<<"Validating Version..."<<endl;
	checkVersion(&scriptText);
	cout<<"Version Compatilbe!";
	cout<<endl<<"About To Compile Script:\n"<<scriptText<<endl;
	if(!removeComments(&scriptText))return false;
	cout<<endl<<"Comments and ';'s Removed:\n"<<scriptText<<endl;
	if(!renameFunctions(&scriptText))return false;
	cout<<endl<<"Functions Renamed:\n"<<scriptText<<endl;
	if(!renameTypes(&scriptText))return false;
	cout<<endl<<"Variable Types Renamed:\n"<<scriptText<<endl;
	if(!renameVars(&scriptText))return false;
	cout<<endl<<"Variables Renamed:\n"<<scriptText<<endl;
	if(!cleanUp(&scriptText))return false;
	cout<<endl<<"Cleaned Up:\n"<<scriptText;
	*input=scriptText;
	return true;
}

//Compiler Steps
bool XPINSCompiler::checkVersion(string* script){
	string input=*script;
	for(int i=0;i<input.length();i++){
		if(input[i]=='@'&&input[i+1]=='C'&&input[i+2]=='O'&&input[i+3]=='M'&&input[i+4]=='P'&&input[i+5]=='I'&&input[i+6]=='L'&&input[i+7]=='E'&&input[i+8]=='R'){
			while(i<input.length()&&input[i]!='[')i++;
			if(i==input.length()){
				cout<<"SCRIPT MISSING VERSION. EXITING";
				return false;;
			}
			int MAJOR=XPINSCompileUtil::readVarIndex(input, &i, '.');
			int MINOR=XPINSCompileUtil::readVarIndex(input, &i, ']');
			if(MAJOR!=kMajor){
				cout<<"INCOMPATIBLE VERSION. EXITING";
				return false;
			}
			if(MINOR<kMinor){
				cout<<"INCOMPATIBLE VERSION. EXITING";
				return false;
			}
			while(i<input.length()&&input[i]!='\n')i++;
			i++;
			string output="";
			while (i<input.length()) {
				output+=input[i++];
			}
			*script=output;
			return true;
		}
	}
	cout<<"SCRIPT MISSING VERSION. EXITING";
	return false;
}
bool XPINSCompiler::removeComments(string* text){
	string input=*text;
	string output="";
	char ch;
	for(int i=0;i<input.length();i++){
		ch=input[i];
		if(ch=='/'){// all '/'s will be removed, any meaningful ones denote a comment
			i++;
			ch=input[i];
			if(ch=='*'){//multi line comments
				while (input[i]!='*'||input[i+1]!='/')i++;
				i++;
			}
			else if(ch=='/'){// single line comments
				while (input[i+1]!='\n'||i+1==input.length())i++;
			}
		}
		else if(ch!=';') output+=ch;//No semicolons :(
	}
	bool hitFirstEND=false;
	string intermediate1=""+output;
	output="";
	//Double Check Ending (strip after 2nd @END)
	for(int j=0;j<intermediate1.length();j++){
		if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
			if(hitFirstEND){
				output+="@END\n";
				break;
			}
			else hitFirstEND=true;
		}
		output+=intermediate1[j];
	}
	*text=output;
	return true;
}

bool XPINSCompiler::renameFunctions(string *text){
	string input=*text;
	string output="";
	char ch;
	int i=0;
	//Locate Function block
	while(input[i]!='@'||input[i+1]!='F'){
		i++;
	}
	if(input[i+1]!='F'||input[i+2]!='U'||input[i+3]!='N'||input[i+4]!='C'){
		cerr<<"Invalid Script: Missing @FUNC.\nEXITING";
		return false;
	}
	i+=5;
	string intermediate1="";
	string intermediate2="";
	//Initialize first intermediate
	int j=0;
	while (input[j]!='@'||input[j+1]!='C') {
		j++;
	}
	for(;j<input.length();j++){
		ch=input[j];
		intermediate1+=ch;
	}
	//Do the actual work
	int x=1;
	while(input[i+1]!='@'){
		if(i==input.length()){
			cerr<<"Invalid Script: Missing @END.\nEXITING";
			return false;;
		}
		while (input[i]!=' '){
			i++;
		}
		i++;
		//read function name
		string functionName;
		for(j=i;j<input.length()&&input[j]!='(';j++){
			functionName+=input[j];
		}
		//Replace function name
		intermediate2="";
		j=0;
		while(true){
			//Get to next line/END
			while(j<intermediate1.length()&&intermediate1[j]!='\n'){
				intermediate2+=intermediate1[j];
				j++;
			}
			//IF END CLEAN UP
			if(j>=intermediate1.length()){
				intermediate2+="@END";
				break;
			}
			intermediate2+=intermediate1[j];
			j++;
			//CHECK FOR @END
			if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
				intermediate2+="@END";
				break;
			}
			//Find start of function
			while(intermediate1[j]!='#'){
				intermediate2+=intermediate1[j];
				j++;
			}
			intermediate2+=intermediate1[j];
			j++;
			//Check for match
			if(XPINSCompileUtil::stringsMatch(j, intermediate1, functionName)){
				intermediate2+='F';
				intermediate2+=XPINSCompileUtil::strRepresentationOfInt(x);
				while(intermediate1[j]!='('){//Find '('
					j++;
				}
			}
		}
		//Get ready for next loop
		intermediate1=""+intermediate2;
		x++;
		while (input[i]!='\n')i++;
	}
	//Double Check Ending (strip after @END)
	for(j=0;j<intermediate1.length();j++){
		if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
			output+="@END";
			break;
		}
		output+=intermediate1[j];
	}
	//output=intermediate1;
	*text=output;
	return true;
}
bool XPINSCompiler::renameTypes(string *text){
	string input=*text;
	string output="";
	string intermediate1="";
	//Rename Types
	for(int i=0;i<input.length();i++){
		if((input[i]=='B'||input[i]=='I'||input[i]=='F'||input[i]=='V'||input[i]=='*')&&input[i-1]=='\n'){
			intermediate1+=input[i];
			while (input[i]!=' ')i++;
		}
		intermediate1+=input[i];
	}
	//Double Check Ending (strip after @END)
	for(int i=0;i<intermediate1.length();i++){
		if(intermediate1[i]=='@'&&intermediate1[i+1]=='E'&&intermediate1[i+2]=='N'&&intermediate1[i+3]=='D'){
			output+="@END\n";
			break;
		}
		output+=intermediate1[i];
	}
	*text=output;
	return true;
}
bool XPINSCompiler::renameVars(string *text){
	string input=*text;
	string output="";
	char ch;
	int i=0;
	//Locate Code block
	while(input[i]!='@'||input[i+1]!='C'){
		i++;
	}
	if(input[i+1]!='C'||input[i+2]!='O'||input[i+3]!='D'||input[i+4]!='E'){
		cerr<<"Invalid Script: Missing @CODE.\nEXITING";
		return false;;
	}
	i+=5;
	//Initialize intermediates
	string intermediate1=""+input;
	string intermediate2="";
	int j=0;
	//Do the actual work
	int xb=-1;
	int xi=-1;
	int xf=-1;
	int xv=-1;
	int xp=-1;
	char varType='P';
	while(i<input.length()&&(input[i]!='@'||input[i+1]!='E')){
		if(input[i-1]=='\n'&&(input[i]=='B'||input[i]=='I'||input[i]=='F'||input[i]=='V'||input[i]=='*'))
		{
			//determine new variable name
			varType=input[i]=='*'?'P':input[i];
			int varNum=0;
			switch (varType) {
				case 'B':
					varNum=++xb;
					break;
				case 'I':
					varNum=++xi;
					break;
				case 'F':
					varNum=++xf;
					break;
				case 'V':
					varNum=++xv;
					break;
				case 'P':
					varNum=++xp;
					break;
				default: varType='P';varNum=++xp;
			}
			//read Var name
			string varName;
			for(i+=3;i<input.length()&&input[i]!='=';i++){
				varName+=input[i];
			}
			//Replace Var name
			intermediate2="";
			j=0;
			int blockCount=0;
			while(j<intermediate1.length()){
				//CHECK FOR @END
				if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
					intermediate2+="@END";
					break;
				}
				//Check for block end
				if(intermediate1[j]=='{')blockCount++;
				else if(intermediate1[j]=='}'){
					if(blockCount==0){
						while(!(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D')&&j+3<intermediate1.length())intermediate2+=intermediate1[j];
						intermediate2+="@END";
						break;
					}
					else blockCount--;
					
				}
				//Find start of Var
				if(intermediate1[j]=='$'){
					intermediate2+='$';
					j++;
					//Check for match
					if(XPINSCompileUtil::stringsMatch(j, intermediate1, varName)){
						intermediate2+=varType;
						intermediate2+=XPINSCompileUtil::strRepresentationOfInt(varNum);
						while(intermediate1[j]!=')'&&intermediate1[j]!='='&&intermediate1[j]!=','&&intermediate1[j]!=']'){//Find '('
							j++;
						}
					}
				}
				else{
					intermediate2+=intermediate1[j];
					j++;
				}
			}
			intermediate1=""+intermediate2;
		}
		i++;
	}
	//Double Check Ending (strip after @END)
	for(j=0;j<intermediate1.length();j++){
		if(intermediate1[j]=='@'&&intermediate1[j+1]=='E'&&intermediate1[j+2]=='N'&&intermediate1[j+3]=='D'){
			output+="@END\n";
			break;
		}
		output+=intermediate1[j];
	}
	*text=output;
	return true;
}
bool XPINSCompiler::cleanUp(string *text){
	string input=*text;
	string output="";
	string intermediate1="";
	//Remove excess whitespace
	for(int i=0;i<input.length();i++){
		if((input[i]!='\n'||input[i+1]!='\n')&&input[i]!='\t')intermediate1+=input[i];
	}
	//Double Check Ending (strip after @END)
	for(int i=0;i<intermediate1.length();i++){
		if(intermediate1[i]=='@'&&intermediate1[i+1]=='E'&&intermediate1[i+2]=='N'&&intermediate1[i+3]=='D'){
			output+="@END\n";
			break;
		}
		output+=intermediate1[i];
	}
	*text=output;
	return true;
}
//UTILTY FUNCTIONS
{'fraction_non_alphanumeric': 0.23605572248019666, 'fraction_numerical': 0.020431576072111445, 'mean_word_length': 4.734962406015038, 'pattern_counts': {'":': 0, '<': 183, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '341170', 'n_tokens_mistral': 8556, 'n_tokens_neox': 7425, 'n_words': 1099}