<commit_before>global_encodingNames = [];
global_encodings = [];
// dictionary of name: {
//     type: 'typename',
//     encode: function(codepoints/str),
//     decode: function(codeUnits/str)
// }

function escapeHtml(string) {
	return he.encode(string);
}

function displayCodepoint(codepoint) {
	if (typeof codepoint == 'undefined')
		return;
	if (codepoint < 0x20)
		codepoint += 0x2400;
	if (codepoint == 0x7F)
		codepoint = 0x2421;
	codepoints = [codepoint];
	if (graphemeBreakValueForCodepoint(codepoint) == 'Extend')
		codepoints = [0x25CC, codepoint];
	return escapeHtml(ctos(codepoints));
}

function ctos(codepoints) {
	return punycode.ucs2.encode(codepoints);
}

function stoc(string) {
	return punycode.ucs2.decode(string);
}

function ctou8(codepoints) {
	var u8str = utf8.encode(ctos(codepoints));
	var res = [];
	for (var i = 0; i < u8str.length; ++i)
		res.push(u8str.charCodeAt(i));
	return res;
}

function u8toc(bytes) {
	var u8str = '';
	for (var i = 0; i < bytes.length; ++i)
		u8str += String.fromCharCode(bytes[i]);
	return stoc(utf8.decode(u8str));
}

function itos(int, base, padding) {
	var res = int.toString(base).toUpperCase();
	if (padding) {
		while (res.length < padding) {
			res = '0' + res;
		}
	}
	return res;
}

function initializeMappings(completion) {
	requestAsync('data/encodings.txt', function() {
		totalCount = 0;
		count = 0;
	}, function(line) {
		var parts = line.split('\t');
		++totalCount;
		var type = parts[0];
		var name = parts[1];
		var url = parts[2];
		global_encodingNames.push(name);
		window.setTimeout(function() {
			loadEncodingFromURL(type, name, url, function() {
				++count;
				if (count == totalCount) {
					$.each(global_encodingNames, function(i, encodingName) {
						if (global_encodings[encodingName].type == '7-bit mapping'
							|| global_encodings[encodingName].type == '8-bit mapping') {
							$('#codepageEncoding')
								.append($('<option' 
									+ (encodingName == 'ISO-8859-1 (Latin-1 Western European)' ? ' selected' : '') 
									+ '></option>')
								.text(encodingName));
						}
						$('#outputEncoding')
							.append($('<option></option>')
							.text(encodingName));
					});
					completion();
				}
			});
		}, 0);
	});
}

function loadEncodingFromURL(type, name, url, completion) {
	var encoding = {};
	encoding.type = type;
	requestAsync(url, function(lines) {
		if (type.includes('function')) {
			encoding = eval(lines.join('\n'));
		} else {
			encoding.encode = function(codepoints) {
				var bytes = [];
				for (var i = 0; i < codepoints.length; ++i) {
					var codepoint = parseInt(codepoints[i]);
					var number = encoding.table[codepoint];
					if (typeof number == 'undefined') {
						// if (codepoint <= 0xFF)
						// 	bytes.push(codepoint);
						// else
						return codepoint;
					}
					if (number <= 0xFF) {
						bytes.push(number);
					} else {
						bytes.push(number >> 8);
						bytes.push(number & 0xFF);
					}
				}
				return bytes;
			};
			encoding.decode = function(bytes) {
				var table = encoding.table;
				var codepointForByteUsingMapping = function(byte) {
					byte = parseInt(byte);
					for (var codepoint in table) {
						if (byte == table[codepoint])
							return parseInt(codepoint);
					}
				};
				var codepoints = [];
				for (var i = 0; i < bytes.length; ++i) {
					var cp = codepointForByteUsingMapping(bytes[i]);
					if (typeof cp != 'undefined') {
						codepoints.push(cp);
						continue;
					}
					cp = codepointForByteUsingMapping((bytes[i] << 8) + bytes[i+1]);
					if (typeof cp == 'undefined') {
						return;
					}
					codepoints.push(cp);
					++i;
				}
				return codepoints;
			};
		}
	}, function(line) {
		if (type.includes('mapping')) {
			if (line.length == 1 && line.charCodeAt(0) == 26) // weird format found in CP857 (and others)
				return;
			var components = line.split('\t');
			if (components[1].trim() == '')
				return;
			if (isNaN(parseInt(components[0])) || isNaN(parseInt(components[1])))
				throw new Error('Invalid line detected in ' + url + ' (' + i + ')');
			if (!encoding.table)
				encoding.table = [];
			encoding.table[parseInt(components[1])] = parseInt(components[0]);
		}
	}, function() {
		global_encodings[name] = encoding;
		completion();
	});
}

function codepointsToEncoding(encoding, codepoints) {
	return global_encodings[encoding].encode(codepoints);
}

function codeUnitsToCodepoints(encoding, codeUnits) {
	return global_encodings[encoding].decode(codeUnits);
}

function bytesToText(format, bytes, hexadecimalPadding) {<commit_msg>Fix 'undefined' in error message when encoding file contains invalid lines<commit_after>global_encodingNames = [];
global_encodings = [];
// dictionary of name: {
//     type: 'typename',
//     encode: function(codepoints/str),
//     decode: function(codeUnits/str)
// }

function escapeHtml(string) {
	return he.encode(string);
}

function displayCodepoint(codepoint) {
	if (typeof codepoint == 'undefined')
		return;
	if (codepoint < 0x20)
		codepoint += 0x2400;
	if (codepoint == 0x7F)
		codepoint = 0x2421;
	codepoints = [codepoint];
	if (graphemeBreakValueForCodepoint(codepoint) == 'Extend')
		codepoints = [0x25CC, codepoint];
	return escapeHtml(ctos(codepoints));
}

function ctos(codepoints) {
	return punycode.ucs2.encode(codepoints);
}

function stoc(string) {
	return punycode.ucs2.decode(string);
}

function ctou8(codepoints) {
	var u8str = utf8.encode(ctos(codepoints));
	var res = [];
	for (var i = 0; i < u8str.length; ++i)
		res.push(u8str.charCodeAt(i));
	return res;
}

function u8toc(bytes) {
	var u8str = '';
	for (var i = 0; i < bytes.length; ++i)
		u8str += String.fromCharCode(bytes[i]);
	return stoc(utf8.decode(u8str));
}

function itos(int, base, padding) {
	var res = int.toString(base).toUpperCase();
	if (padding) {
		while (res.length < padding) {
			res = '0' + res;
		}
	}
	return res;
}

function initializeMappings(completion) {
	requestAsync('data/encodings.txt', function() {
		totalCount = 0;
		count = 0;
	}, function(line) {
		var parts = line.split('\t');
		++totalCount;
		var type = parts[0];
		var name = parts[1];
		var url = parts[2];
		global_encodingNames.push(name);
		window.setTimeout(function() {
			loadEncodingFromURL(type, name, url, function() {
				++count;
				if (count == totalCount) {
					$.each(global_encodingNames, function(i, encodingName) {
						if (global_encodings[encodingName].type == '7-bit mapping'
							|| global_encodings[encodingName].type == '8-bit mapping') {
							$('#codepageEncoding')
								.append($('<option' 
									+ (encodingName == 'ISO-8859-1 (Latin-1 Western European)' ? ' selected' : '') 
									+ '></option>')
								.text(encodingName));
						}
						$('#outputEncoding')
							.append($('<option></option>')
							.text(encodingName));
					});
					completion();
				}
			});
		}, 0);
	});
}

function loadEncodingFromURL(type, name, url, completion) {
	var encoding = {};
	encoding.type = type;
	requestAsync(url, function(lines) {
		if (type.includes('function')) {
			encoding = eval(lines.join('\n'));
		} else {
			encoding.encode = function(codepoints) {
				var bytes = [];
				for (var i = 0; i < codepoints.length; ++i) {
					var codepoint = parseInt(codepoints[i]);
					var number = encoding.table[codepoint];
					if (typeof number == 'undefined') {
						// if (codepoint <= 0xFF)
						// 	bytes.push(codepoint);
						// else
						return codepoint;
					}
					if (number <= 0xFF) {
						bytes.push(number);
					} else {
						bytes.push(number >> 8);
						bytes.push(number & 0xFF);
					}
				}
				return bytes;
			};
			encoding.decode = function(bytes) {
				var table = encoding.table;
				var codepointForByteUsingMapping = function(byte) {
					byte = parseInt(byte);
					for (var codepoint in table) {
						if (byte == table[codepoint])
							return parseInt(codepoint);
					}
				};
				var codepoints = [];
				for (var i = 0; i < bytes.length; ++i) {
					var cp = codepointForByteUsingMapping(bytes[i]);
					if (typeof cp != 'undefined') {
						codepoints.push(cp);
						continue;
					}
					cp = codepointForByteUsingMapping((bytes[i] << 8) + bytes[i+1]);
					if (typeof cp == 'undefined') {
						return;
					}
					codepoints.push(cp);
					++i;
				}
				return codepoints;
			};
		}
	}, function(line) {
		if (type.includes('mapping')) {
			if (line.length == 1 && line.charCodeAt(0) == 26) // weird format found in CP857 (and others)
				return;
			var components = line.split('\t');
			if (components[1].trim() == '')
				return;
			if (isNaN(parseInt(components[0])) || isNaN(parseInt(components[1])))
				throw new Error('Invalid line detected in ' + url);
			if (!encoding.table)
				encoding.table = [];
			encoding.table[parseInt(components[1])] = parseInt(components[0]);
		}
	}, function() {
		global_encodings[name] = encoding;
		completion();
	});
}

function codepointsToEncoding(encoding, codepoints) {
	return global_encodings[encoding].encode(codepoints);
}

function codeUnitsToCodepoints(encoding, codeUnits) {
	return global_encodings[encoding].decode(codeUnits);
}

function bytesToText(format, bytes, hexadecimalPadding) {
{'fraction_non_alphanumeric': 0.16248363160192056, 'fraction_numerical': 0.014622435617634221, 'mean_word_length': 3.32311320754717, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5058684', 'n_tokens_mistral': 3780, 'n_tokens_neox': 3225, 'n_words': 645}