<commit_before>var generics;

var inverseTable = {
	'AdminStatus':              'AdminStatusArray',
	'AdminUpdateType':          'AdminUpdateTypeArray',
	'AdminUpdateFrequency':     'AdminUpdateFrequencyArray',
	'AdminCompanyRemoveReason': 'AdminCompanyRemoveReasonArray',
	'Colour':                   'ColourArray',
	'DestType':                 'DestTypeArray',
	'Landscape':                'LandscapeArray',
	'NetworkAction':            'NetworkActionArray',
	'NetworkErrorCode':         'NetworkErrorCodeArray',
	'NetworkLanguage':          'NetworkLanguageArray',
	'VehicleType':              'VehicleTypeArray'
};

module.exports = generics = {};

// Protocol data

// Swaps keys with values
generics.getInversedObject = function(object) {
	var result = {};

	for (var i in object) {
		result[object[i]] = i;
	}

	return result;
};

generics.spec = require("../data/protocol")
for (var i in inverseTable) {
	generics.spec[inverseTable[i]] = generics.getInversedObject(generics.spec[i]);
}

// Date handling

generics.isLeapYear = function(year) {
	return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
};

generics.daysInMonth = function(month, year) {
	return [31, this.isLeapYear(year) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month];
};

generics.getQuarter = function(month) {
	return Math.floor(month / 3);
};

// Year calculation based on
// http://dev.openttdcoop.org/projects/joan/repository/entry/src/main/java/org/openttd/GameDate.java
generics.gameDateToDate = function(date) {
	var rem, year, month = -1, days = 0;

	// There are 97 leap years in 400 years
	year = 400 * Math.floor(date / (365 * 400 + 97));
	rem  = date % (365 * 400 + 97);

	// There are 24 leap years in 100 years
	year += 100 * Math.floor(rem / (365 * 100 + 24));
	rem  %= 365 * 100 + 24;

	// There is 1 leap year every 4 years
	year += 4 * Math.floor(rem / (365 * 4 + 1));
	rem  %= 365 * 4 + 1;

	// Finish year calculation
	while (rem >= (this.isLeapYear(year) ? 366 : 365)) {
		rem -= this.isLeapYear(year) ? 366 : 365;
		year++;
	}

	// Get month
	do {
		month++;
		rem -=  days;
		days = this.daysInMonth(month, year);
	} while (rem > days);

	// Rem should be day in the month
	return {year: year, month: month, day: rem + 1, quarter: this.getQuarter(month)};
};

// Network stuff

generics.typeTableSend = {
	"uint8":    function(input) {var buffer=new Buffer(1);buffer.writeUInt8   (input, 0); return buffer;},
	"int8":     function(input) {var buffer=new Buffer(1);buffer.writeInt8    (input, 0); return buffer;},
	"uint16be": function(input) {var buffer=new Buffer(2);buffer.writeUInt16BE(input, 0); return buffer;},
	"uint16le": function(input) {var buffer=new Buffer(2);buffer.writeUInt16LE(input, 0); return buffer;},
	"int16be":  function(input) {var buffer=new Buffer(2);buffer.writeInt16BE (input, 0); return buffer;},
	"int16le":  function(input) {var buffer=new Buffer(2);buffer.writeInt16LE (input, 0); return buffer;},
	"uint32be": function(input) {var buffer=new Buffer(4);buffer.writeUInt32BE(input, 0); return buffer;},
	"uint32le": function(input) {var buffer=new Buffer(4);buffer.writeUInt32LE(input, 0); return buffer;},
	"int32be":  function(input) {var buffer=new Buffer(4);buffer.writeInt32BE (input, 0); return buffer;},
	"int32le":  function(input) {var buffer=new Buffer(4);buffer.writeInt32LE (input, 0); return buffer;},
	"floatbe":  function(input) {var buffer=new Buffer(4);buffer.writeFloatBE (input, 0); return buffer;},
	"floatle":  function(input) {var buffer=new Buffer(4);buffer.writeFloatLE (input, 0); return buffer;},
	"doublebe": function(input) {var buffer=new Buffer(8);buffer.writeDoubleBE(input, 0); return buffer;},
	"doublele": function(input) {var buffer=new Buffer(8);buffer.writeDoubleLE(input, 0); return buffer;},
	"bool":     function(input) {var buffer=new Buffer(1);buffer.writeUInt8(input ? 1 : 0, 0); return buffer;},
	"utf8":     function(input) {return new Buffer(input.replace("\u0000", "") + "\u0000", 'utf8');}
};

generics.typeTableReceive = {
	"bool":     function(buffer, offset) {return [buffer.readUInt8(offset) === 1, 1];},
	"uint8":    function(buffer, offset) {return [buffer.readUInt8(offset),       1];},
	"int8":     function(buffer, offset) {return [buffer.readInt8(offset),        1];},
	"uint16be": function(buffer, offset) {return [buffer.readUInt16BE(offset),    2];},
	"uint16le": function(buffer, offset) {return [buffer.readUInt16LE(offset),    2];},
	"int16be":  function(buffer, offset) {return [buffer.readInt16BE(offset),     2];},
	"int16le":  function(buffer, offset) {return [buffer.readInt16LE(offset),     2];},
	"uint32be": function(buffer, offset) {return [buffer.readUInt32BE(offset),    4];},
	"uint32le": function(buffer, offset) {return [buffer.readUInt32LE(offset),    4];},
	"int32be":  function(buffer, offset) {return [buffer.readInt32BE(offset),     4];},
	"int32le":  function(buffer, offset) {return [buffer.readInt32LE(offset),     4];},<commit_msg>Fix utf8 filtering in typeTableSend<commit_after>var generics;

var inverseTable = {
	'AdminStatus':              'AdminStatusArray',
	'AdminUpdateType':          'AdminUpdateTypeArray',
	'AdminUpdateFrequency':     'AdminUpdateFrequencyArray',
	'AdminCompanyRemoveReason': 'AdminCompanyRemoveReasonArray',
	'Colour':                   'ColourArray',
	'DestType':                 'DestTypeArray',
	'Landscape':                'LandscapeArray',
	'NetworkAction':            'NetworkActionArray',
	'NetworkErrorCode':         'NetworkErrorCodeArray',
	'NetworkLanguage':          'NetworkLanguageArray',
	'VehicleType':              'VehicleTypeArray'
};

module.exports = generics = {};

// Protocol data

// Swaps keys with values
generics.getInversedObject = function(object) {
	var result = {};

	for (var i in object) {
		result[object[i]] = i;
	}

	return result;
};

generics.spec = require("../data/protocol")
for (var i in inverseTable) {
	generics.spec[inverseTable[i]] = generics.getInversedObject(generics.spec[i]);
}

// Date handling

generics.isLeapYear = function(year) {
	return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
};

generics.daysInMonth = function(month, year) {
	return [31, this.isLeapYear(year) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month];
};

generics.getQuarter = function(month) {
	return Math.floor(month / 3);
};

// Year calculation based on
// http://dev.openttdcoop.org/projects/joan/repository/entry/src/main/java/org/openttd/GameDate.java
generics.gameDateToDate = function(date) {
	var rem, year, month = -1, days = 0;

	// There are 97 leap years in 400 years
	year = 400 * Math.floor(date / (365 * 400 + 97));
	rem  = date % (365 * 400 + 97);

	// There are 24 leap years in 100 years
	year += 100 * Math.floor(rem / (365 * 100 + 24));
	rem  %= 365 * 100 + 24;

	// There is 1 leap year every 4 years
	year += 4 * Math.floor(rem / (365 * 4 + 1));
	rem  %= 365 * 4 + 1;

	// Finish year calculation
	while (rem >= (this.isLeapYear(year) ? 366 : 365)) {
		rem -= this.isLeapYear(year) ? 366 : 365;
		year++;
	}

	// Get month
	do {
		month++;
		rem -=  days;
		days = this.daysInMonth(month, year);
	} while (rem > days);

	// Rem should be day in the month
	return {year: year, month: month, day: rem + 1, quarter: this.getQuarter(month)};
};

// Network stuff

generics.typeTableSend = {
	"uint8":    function(input) {var buffer=new Buffer(1);buffer.writeUInt8   (input, 0); return buffer;},
	"int8":     function(input) {var buffer=new Buffer(1);buffer.writeInt8    (input, 0); return buffer;},
	"uint16be": function(input) {var buffer=new Buffer(2);buffer.writeUInt16BE(input, 0); return buffer;},
	"uint16le": function(input) {var buffer=new Buffer(2);buffer.writeUInt16LE(input, 0); return buffer;},
	"int16be":  function(input) {var buffer=new Buffer(2);buffer.writeInt16BE (input, 0); return buffer;},
	"int16le":  function(input) {var buffer=new Buffer(2);buffer.writeInt16LE (input, 0); return buffer;},
	"uint32be": function(input) {var buffer=new Buffer(4);buffer.writeUInt32BE(input, 0); return buffer;},
	"uint32le": function(input) {var buffer=new Buffer(4);buffer.writeUInt32LE(input, 0); return buffer;},
	"int32be":  function(input) {var buffer=new Buffer(4);buffer.writeInt32BE (input, 0); return buffer;},
	"int32le":  function(input) {var buffer=new Buffer(4);buffer.writeInt32LE (input, 0); return buffer;},
	"floatbe":  function(input) {var buffer=new Buffer(4);buffer.writeFloatBE (input, 0); return buffer;},
	"floatle":  function(input) {var buffer=new Buffer(4);buffer.writeFloatLE (input, 0); return buffer;},
	"doublebe": function(input) {var buffer=new Buffer(8);buffer.writeDoubleBE(input, 0); return buffer;},
	"doublele": function(input) {var buffer=new Buffer(8);buffer.writeDoubleLE(input, 0); return buffer;},
	"bool":     function(input) {var buffer=new Buffer(1);buffer.writeUInt8(input ? 1 : 0, 0); return buffer;},
	"utf8":     function(input) {return new Buffer(input.replace(/\u0000/g, "") + "\u0000", 'utf8');}
};

generics.typeTableReceive = {
	"bool":     function(buffer, offset) {return [buffer.readUInt8(offset) === 1, 1];},
	"uint8":    function(buffer, offset) {return [buffer.readUInt8(offset),       1];},
	"int8":     function(buffer, offset) {return [buffer.readInt8(offset),        1];},
	"uint16be": function(buffer, offset) {return [buffer.readUInt16BE(offset),    2];},
	"uint16le": function(buffer, offset) {return [buffer.readUInt16LE(offset),    2];},
	"int16be":  function(buffer, offset) {return [buffer.readInt16BE(offset),     2];},
	"int16le":  function(buffer, offset) {return [buffer.readInt16LE(offset),     2];},
	"uint32be": function(buffer, offset) {return [buffer.readUInt32BE(offset),    4];},
	"uint32le": function(buffer, offset) {return [buffer.readUInt32LE(offset),    4];},
	"int32be":  function(buffer, offset) {return [buffer.readInt32BE(offset),     4];},
	"int32le":  function(buffer, offset) {return [buffer.readInt32LE(offset),     4];},
{'fraction_non_alphanumeric': 0.16912724716280003, 'fraction_numerical': 0.04850858692377222, 'mean_word_length': 4.426702997275204, 'pattern_counts': {'":': 54, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5062743', 'n_tokens_mistral': 3724, 'n_tokens_neox': 3354, 'n_words': 827}