<commit_before>/*************************************
CONVERT B9 PORCELAIN TO BINARY MODULE 
**************************************/

// syntax is <in> <out>

var esprima = require('esprima');
var fs = require('fs');

function outputUInt32(out, value) {

	var buf = Buffer.allocUnsafe(4);
	buf.writeUInt32LE(value, 0);
	fs.writeSync(out, buf);
}

function outputString(out, string) {
	outputUInt32(out, string.length);
	fs.writeSync(out, string);
}

function primitiveCode(primitiveName) {
	var primitives = { "print_string": 0, "print_number": 1 };
	return primitives[primitiveName];
}

function operatorCode(operator) {
	switch (operator) {
		case "END_SECTION":
			return 0;
		case "FUNCTION_CALL":
			return 1;
		case "FUNCTION_RETURN":
			return 2;
		case "PRIMITIVE_CALL":
			return 3;
		case "JMP":
			return 4;
		case "DUPLICATE":
			return 5;
		case "DROP":
			return 6;
		case "PUSH_FROM_VAR":
			return 7;
		case "POP_INTO_VAR":
			return 8;
		case "INT_ADD":
			return 9;
		case "INT_SUB":
			return 10;
		case "INT_PUSH_CONSTANT":
			return 13;
		case "INT_NOT":
			return 14;
		case "INT_JMP_EQ":
			return 15;
		case "INT_JMP_NEQ":
			return 16;
		case "INT_JMP_GT":
			return 17;
		case "INT_JMP_GE":
			return 18;
		case "INT_JMP_LT":
			return 19;
		case "INT_JMP_LE":
			return 20;
		case "STR_PUSH_CONSTANT":
			return 21;
		case "STR_JMP_EQ":
			return 22;
		case "STR_JMP_NEQ":
			return 23;
		default:
			return -1;
	}
};

/// operator - name of operator (NOT CODE)
/// operand - numeric operand, if applicable.
var Instruction = function (operator, operand) {

	this.output = function (out) {
		var encoded = (operatorCode(this.operator) << 24);

		if (this.operand) {
			encoded |= this.operand & 0x00FFFFFF;
		}

		encoded &= 0xFFFFFFFF;

		console.log(this);
		console.log("&&&&" + encoded.toString(16));
		outputUInt32(out, encoded);
	}

	this.operator = operator;
	this.operand = operand;
};

var SymbolTable = function () {

	this.next = 0;
	this.map = {};

	/// Get without interning
	this.lookup = function (symbol) {
		return this.map[symbol];
	}

	/// Lookup and intern if not found.
	this.get = function (symbol) {
		var id = this.lookup(symbol);

		if (id == undefined) {
			
			console.log(this.map);
			id = this.next;
			this.next += 1;
			console.log("interning " + symbol + "with id " + id);
			this.map[symbol] = id;
		}

		return id;
	}

	/// callback(name, id)
	this.forEach = function (callback) {
		var me = this;
		for (symbol in this.map) {
			callback(symbol, me.map[symbol]);
		}
	}
}

var LabelTable = function () {

	this.table = [];

	this.create = function () {
		return table.push(undefined);
	}

	this.place = function (label, offset) {
		this.table[label] = offset;
	}

	this.createAndPlace = function (offset) {
		id = this.make();
		this.place(id, offset);
		return id;
	}

	this.instructionIndex = function (label) {
		return this.table[label];
	}

};

var LexicalContext = function (outer) {
	new SymbolTable();
	this.outer = outer;
};

/// A section of code. CodeBody has information about args and registers, but no information about indexes or it's name.
/// The name-to-index mapping is managed externally, by the module's FunctionTable. Eventually, the args and registers
/// might move to a lexical environment, and this will become a simple bytecode array.
function FunctionDefinition(outer) {
	this.args = new SymbolTable();
	this.regs = new SymbolTable();
	this.labels = new LabelTable();
	this.instructions = [];

	this.resolve = function (module) {
		for (instruction in this.instructions) {
			// TODO: Resolve any undefined function-id's or labels in the body.
			// 	switch(instruction.operator) {
			// 	case 
			// 	}

			// this.deltaForLabel = function (labelName) {
			// 	var gen = this;
			// 	return function (fromInstruction) {
			// 		// delay computing offset until forward labels are found
			// 		// output is done at the end, all labels will be defined
			// 		var fromIndex = fromInstruction.instructionIndex;
			// 		var toIndex = gen.labels[labelName];
			// 		return toIndex - fromIndex - 1;
			// 	}
			// };
		}
	}

	this.output = function (out) {
		// note that name and index are output by the module.
		outputUInt32(out, this.args.length);
		outputUInt32(out, this.regs.length);
		this.instructions.forEach(function (instruction) {
			instruction.output(out);
		})
	};

	this.pushInstruction = function (instruction) {
		this.instructions.push(instruction);
	};

	this.localIndex = function (name) {
		var index = this.regs.lookup(name);
		if (index) {
			return index + this.args.next;
		}
		return this.args.lookup(name);<commit_msg>Output the nargs and nregs correctly<commit_after>/*************************************
CONVERT B9 PORCELAIN TO BINARY MODULE 
**************************************/

// syntax is <in> <out>

var esprima = require('esprima');
var fs = require('fs');

function outputUInt32(out, value) {

	var buf = Buffer.allocUnsafe(4);
	buf.writeUInt32LE(value, 0);
	fs.writeSync(out, buf);
}

function outputString(out, string) {
	outputUInt32(out, string.length);
	fs.writeSync(out, string);
}

function primitiveCode(primitiveName) {
	var primitives = { "print_string": 0, "print_number": 1 };
	return primitives[primitiveName];
}

function operatorCode(operator) {
	switch (operator) {
		case "END_SECTION":
			return 0;
		case "FUNCTION_CALL":
			return 1;
		case "FUNCTION_RETURN":
			return 2;
		case "PRIMITIVE_CALL":
			return 3;
		case "JMP":
			return 4;
		case "DUPLICATE":
			return 5;
		case "DROP":
			return 6;
		case "PUSH_FROM_VAR":
			return 7;
		case "POP_INTO_VAR":
			return 8;
		case "INT_ADD":
			return 9;
		case "INT_SUB":
			return 10;
		case "INT_PUSH_CONSTANT":
			return 13;
		case "INT_NOT":
			return 14;
		case "INT_JMP_EQ":
			return 15;
		case "INT_JMP_NEQ":
			return 16;
		case "INT_JMP_GT":
			return 17;
		case "INT_JMP_GE":
			return 18;
		case "INT_JMP_LT":
			return 19;
		case "INT_JMP_LE":
			return 20;
		case "STR_PUSH_CONSTANT":
			return 21;
		case "STR_JMP_EQ":
			return 22;
		case "STR_JMP_NEQ":
			return 23;
		default:
			return -1;
	}
};

/// operator - name of operator (NOT CODE)
/// operand - numeric operand, if applicable.
var Instruction = function (operator, operand) {

	this.output = function (out) {
		var encoded = (operatorCode(this.operator) << 24);

		if (this.operand) {
			encoded |= this.operand & 0x00FFFFFF;
		}

		encoded &= 0xFFFFFFFF;

		console.log(this);
		console.log("&&&&" + encoded.toString(16));
		outputUInt32(out, encoded);
	}

	this.operator = operator;
	this.operand = operand;
};

var SymbolTable = function () {

	this.next = 0;
	this.map = {};

	/// Get without interning
	this.lookup = function (symbol) {
		return this.map[symbol];
	}

	/// Lookup and intern if not found.
	this.get = function (symbol) {
		var id = this.lookup(symbol);

		if (id == undefined) {
			
			console.log(this.map);
			id = this.next;
			this.next += 1;
			console.log("interning " + symbol + "with id " + id);
			this.map[symbol] = id;
		}

		return id;
	}

	/// callback(name, id)
	this.forEach = function (callback) {
		var me = this;
		for (symbol in this.map) {
			callback(symbol, me.map[symbol]);
		}
	}
}

var LabelTable = function () {

	this.table = [];

	this.create = function () {
		return table.push(undefined);
	}

	this.place = function (label, offset) {
		this.table[label] = offset;
	}

	this.createAndPlace = function (offset) {
		id = this.make();
		this.place(id, offset);
		return id;
	}

	this.instructionIndex = function (label) {
		return this.table[label];
	}

};

var LexicalContext = function (outer) {
	new SymbolTable();
	this.outer = outer;
};

/// A section of code. CodeBody has information about args and registers, but no information about indexes or it's name.
/// The name-to-index mapping is managed externally, by the module's FunctionTable. Eventually, the args and registers
/// might move to a lexical environment, and this will become a simple bytecode array.
function FunctionDefinition(outer) {
	this.args = new SymbolTable();
	this.regs = new SymbolTable();
	this.labels = new LabelTable();
	this.instructions = [];

	this.resolve = function (module) {
		for (instruction in this.instructions) {
			// TODO: Resolve any undefined function-id's or labels in the body.
			// 	switch(instruction.operator) {
			// 	case 
			// 	}

			// this.deltaForLabel = function (labelName) {
			// 	var gen = this;
			// 	return function (fromInstruction) {
			// 		// delay computing offset until forward labels are found
			// 		// output is done at the end, all labels will be defined
			// 		var fromIndex = fromInstruction.instructionIndex;
			// 		var toIndex = gen.labels[labelName];
			// 		return toIndex - fromIndex - 1;
			// 	}
			// };
		}
	}

	this.output = function (out) {
		// note that name and index are output by the module.
		outputUInt32(out, this.args.next);
		outputUInt32(out, this.regs.next);
		this.instructions.forEach(function (instruction) {
			instruction.output(out);
		})
	};

	this.pushInstruction = function (instruction) {
		this.instructions.push(instruction);
	};

	this.localIndex = function (name) {
		var index = this.regs.lookup(name);
		if (index) {
			return index + this.args.next;
		}
		return this.args.lookup(name);
{'fraction_non_alphanumeric': 0.14884221863220248, 'fraction_numerical': 0.0135702746365105, 'mean_word_length': 3.303058387395737, 'pattern_counts': {'":': 48, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675182', 'n_tokens_mistral': 3642, 'n_tokens_neox': 3155, 'n_words': 854}