<commit_before>// parses LLVM IR code

var fs = require('fs');

var regexs = {
	define: /^define ([^ ]+) ([^\(]+)\(([^\)]*)\)([^{]+){/,
	declare: /^declare ([^ ]+) ([^\(]+)([^\)]+)\)/,

	call: /^\s*call ([^ ]+) ([^\(]+)\((.+)/,
	ret: /^\s*ret (.+)/,

	alloca: /^\s*alloca (.+)/,
	store: /^\s+store ([^ ]+) ([^,]+), ([^ ]+) ([^,]+)/,

	load: /^load ([^ ]+) (.+)/,
	add: /^add ([^ ]+) ([^,]+), (.+)/,
	sub: /^sub ([^ ]+) ([^,]+), (.+)/,
	mul: /^mul ([^ ]+) ([^,]+), (.+)/,
	div: /^div ([^ ]+) ([^,]+), (.+)/,
	icmp: /^icmp ([^ ]+) ([^ ]+) ([^,]+), (.+)/,
	sext: /^sext i\d+ ([^ ]+) to i\d+/,

	localSet: /^\s+%([^ ]+) = (.+)/,

	label: /; <label>:(\d+)/,
	absoluteBranch: /\s+br label (.+)/,
	conditionalBranch: /\s+br i1 ([^,]+), label ([^,]+), label (.+)/,

	globalVar: /@([^ ]+) = (private )?(unnamed_addr )?(constant )?((\[([^\]])+\])|([^ ]+))(.+)/,

	inlineInstruction: /([a-zA-Z ]+)\(([^\(]+)\)/
}

function parse(file, ffi) {
	// strip out some really useless fields
	// if this turns out to be useful in the future
	// (aka I just broke something)
	// poke me

	file = file.replace(/zeroext /g, "");
	file = file.replace(/ zeroext/g, "");
	file = file.replace(/signext /g, "");
	file = file.replace(/ signext/g, "");
	file = file.replace(/, align \d/g, "");
	file = file.replace(/ nsw/g, "");
	file = file.replace(/ nuw/g, "");

	var lines = file.split('\n');

	var mod = {
		functions: [],
		globals: []
	};

	var inFunctionBlock = false;
	var functionBlock = {};

	function gotoComplex(initBranch) {
		if(!functionBlock.inGotoComplex) {
			functionBlock.inGotoComplex = true;
			functionBlock.code.push({
				type: "gotoComplex"
			});

		}
	}

	for(var i = 0; i < lines.length; ++i) {
		if(!inFunctionBlock) {
			if(regexs.define.test(lines[i])) {
				var m = lines[i].match(regexs.define);

				var returnType = m[1];
				var funcName = m[2];
				var paramList = m[3];
				var modifiers = m[4];

				functionBlock = {
					returnType: returnType,
					paramList: extractParamList(paramList),
					funcName: funcName,
					code: [],
					labels: {},
					inGotoComplex: false
				};
				inFunctionBlock = true;
			} else if(regexs.declare.test(lines[i])) {
				var m = lines[i].match(regexs.declare);

				var returnType = m[1];
				var funcName = m[2];
				var paramList = extractParamList(m[3]);

				codeBlock = [];
				hasFFI = false;

				if(ffi.indexOf(funcName) > -1) {
					codeBlock = [{
						type: "ffi",
						ffiBlock: funcName
					}];
					hasFFI = true;
				}

				mod.functions.push({
					returnType: returnType,
					paramList: paramList,
					funcName: funcName,
					code: codeBlock,
					hasFFI: hasFFI
				})

				console.log(m);
			} else if(regexs.globalVar.test(lines[i])) {
				var m = lines[i].match(regexs.globalVar);

				var name = m[1].trim();
				var type = m[5].trim();

				var val = null;
				
				if(m[9]) {
					val = formatValue(type, m[9].trim());
				}

				mod.globals.push({
					name: name,
					type: type,
					val: val
				});
			}
		} else {
			if(lines[i] == "}") {
				mod.functions.push(functionBlock);
				inFunctionBlock = false;
			} else if(regexs.label.test(lines[i])) {
				var m = lines[i].match(regexs.label);
				functionBlock.labels[m[1]] = functionBlock.code.length;

				// synthetic label block for enabling the backend to function properly
				// without the need of very messy hacks (iterating through labels, etc.)

				functionBlock.code.push({
					type: "label",
					label: m[1]
				});
			} else if(regexs.localSet.test(lines[i])) {
				var m = lines[i].match(regexs.localSet);
				console.log(m);

				var block = {
					type: "set",
					name: "%"+m[1],
					val: 0,
					computation: []
				};

				if(regexs.call.test(m[2])) {
					block.val = {
						type: "return value"
					}
					block.computation = callBlock(m[2].match(regexs.call));

					functionBlock.code.push(block);
				} else if(regexs.alloca.test(m[2])) {
					// no computation work here, but it still needs a spot on the stack for now
					// todo: optimize alloca calls out
					
					functionBlock.code.push(block);
				} else if(regexs.load.test(m[2])) {
					block.val = {
						type: "variable",
						name: m[2].match(regexs.load)[2]
					}

					functionBlock.code.push(block);
				} else if(regexs.add.test(m[2])) {
					var m = m[2].match(regexs.add);
					block.val = {
						type: "arithmetic",
						operation: "+",
						operand1: m[2],
						operand2: m[3]
					};

					functionBlock.code.push(block);
				} else if(regexs.sub.test(m[2])) {
					var m = m[2].match(regexs.add);
					block.val = {<commit_msg>Add type to load instruction<commit_after>// parses LLVM IR code

var fs = require('fs');

var regexs = {
	define: /^define ([^ ]+) ([^\(]+)\(([^\)]*)\)([^{]+){/,
	declare: /^declare ([^ ]+) ([^\(]+)([^\)]+)\)/,

	call: /^\s*call ([^ ]+) ([^\(]+)\((.+)/,
	ret: /^\s*ret (.+)/,

	alloca: /^\s*alloca (.+)/,
	store: /^\s+store ([^ ]+) ([^,]+), ([^ ]+) ([^,]+)/,

	load: /^load ([^ ]+) (.+)/,
	add: /^add ([^ ]+) ([^,]+), (.+)/,
	sub: /^sub ([^ ]+) ([^,]+), (.+)/,
	mul: /^mul ([^ ]+) ([^,]+), (.+)/,
	div: /^div ([^ ]+) ([^,]+), (.+)/,
	icmp: /^icmp ([^ ]+) ([^ ]+) ([^,]+), (.+)/,
	sext: /^sext i\d+ ([^ ]+) to i\d+/,

	localSet: /^\s+%([^ ]+) = (.+)/,

	label: /; <label>:(\d+)/,
	absoluteBranch: /\s+br label (.+)/,
	conditionalBranch: /\s+br i1 ([^,]+), label ([^,]+), label (.+)/,

	globalVar: /@([^ ]+) = (private )?(unnamed_addr )?(constant )?((\[([^\]])+\])|([^ ]+))(.+)/,

	inlineInstruction: /([a-zA-Z ]+)\(([^\(]+)\)/
}

function parse(file, ffi) {
	// strip out some really useless fields
	// if this turns out to be useful in the future
	// (aka I just broke something)
	// poke me

	file = file.replace(/zeroext /g, "");
	file = file.replace(/ zeroext/g, "");
	file = file.replace(/signext /g, "");
	file = file.replace(/ signext/g, "");
	file = file.replace(/, align \d/g, "");
	file = file.replace(/ nsw/g, "");
	file = file.replace(/ nuw/g, "");

	var lines = file.split('\n');

	var mod = {
		functions: [],
		globals: []
	};

	var inFunctionBlock = false;
	var functionBlock = {};

	function gotoComplex(initBranch) {
		if(!functionBlock.inGotoComplex) {
			functionBlock.inGotoComplex = true;
			functionBlock.code.push({
				type: "gotoComplex"
			});

		}
	}

	for(var i = 0; i < lines.length; ++i) {
		if(!inFunctionBlock) {
			if(regexs.define.test(lines[i])) {
				var m = lines[i].match(regexs.define);

				var returnType = m[1];
				var funcName = m[2];
				var paramList = m[3];
				var modifiers = m[4];

				functionBlock = {
					returnType: returnType,
					paramList: extractParamList(paramList),
					funcName: funcName,
					code: [],
					labels: {},
					inGotoComplex: false
				};
				inFunctionBlock = true;
			} else if(regexs.declare.test(lines[i])) {
				var m = lines[i].match(regexs.declare);

				var returnType = m[1];
				var funcName = m[2];
				var paramList = extractParamList(m[3]);

				codeBlock = [];
				hasFFI = false;

				if(ffi.indexOf(funcName) > -1) {
					codeBlock = [{
						type: "ffi",
						ffiBlock: funcName
					}];
					hasFFI = true;
				}

				mod.functions.push({
					returnType: returnType,
					paramList: paramList,
					funcName: funcName,
					code: codeBlock,
					hasFFI: hasFFI
				})

				console.log(m);
			} else if(regexs.globalVar.test(lines[i])) {
				var m = lines[i].match(regexs.globalVar);

				var name = m[1].trim();
				var type = m[5].trim();

				var val = null;
				
				if(m[9]) {
					val = formatValue(type, m[9].trim());
				}

				mod.globals.push({
					name: name,
					type: type,
					val: val
				});
			}
		} else {
			if(lines[i] == "}") {
				mod.functions.push(functionBlock);
				inFunctionBlock = false;
			} else if(regexs.label.test(lines[i])) {
				var m = lines[i].match(regexs.label);
				functionBlock.labels[m[1]] = functionBlock.code.length;

				// synthetic label block for enabling the backend to function properly
				// without the need of very messy hacks (iterating through labels, etc.)

				functionBlock.code.push({
					type: "label",
					label: m[1]
				});
			} else if(regexs.localSet.test(lines[i])) {
				var m = lines[i].match(regexs.localSet);
				console.log(m);

				var block = {
					type: "set",
					name: "%"+m[1],
					val: 0,
					computation: []
				};

				if(regexs.call.test(m[2])) {
					block.val = {
						type: "return value"
					}
					block.computation = callBlock(m[2].match(regexs.call));

					functionBlock.code.push(block);
				} else if(regexs.alloca.test(m[2])) {
					// no computation work here, but it still needs a spot on the stack for now
					// todo: optimize alloca calls out
					
					functionBlock.code.push(block);
				} else if(regexs.load.test(m[2])) {
					var m = m[2].match(regexs.load);

					block.val = {
						type: "variable",
						name: m[2],
						type: m[1]
					}

					functionBlock.code.push(block);
				} else if(regexs.add.test(m[2])) {
					var m = m[2].match(regexs.add);
					block.val = {
						type: "arithmetic",
						operation: "+",
						operand1: m[2],
						operand2: m[3]
					};

					functionBlock.code.push(block);
				} else if(regexs.sub.test(m[2])) {
					var m = m[2].match(regexs.add);
					block.val = {
{'fraction_non_alphanumeric': 0.23108695652173913, 'fraction_numerical': 0.007065217391304348, 'mean_word_length': 2.5759813447337736, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5634803', 'n_tokens_mistral': 4438, 'n_tokens_neox': 3497, 'n_words': 681}