<commit_before>"use strict";

module ("Tokenizer");

///*
test ("Ending backslash removal and line number", function() {
	var source = "f\nop \\\nba\\z";
	var t = new Tokenizer(null, source);

	var result = "f\nop ba\\z";
	var lines = [1, 1, 2, 2, 2, 2, 3, 3, 3]; // [5] should in reality be 3, but I'm not picky
	var columns = [1, 2, 1, 2, 3, 4, 2, 3, 4]; // [5] -> 1

	var ch, cursor = 0, line = 1, column = 1;
	while ((ch = t.nextch())) {
		equal (String.fromCharCode(ch), result.charAt(cursor), "Test character @ "+cursor);
		equal (line, lines[cursor], "Test line number @ "+cursor);
		equal (column, columns[cursor], "Test column number @ "+cursor);

		cursor++;
		line = t.line;
		column = t.column;
	}
});
//*/

///*
test ("Greedy matching 'a+++++a;'", function() {
	var t = new Tokenizer(null, "a+++++a;");
	var result = [
		"a",
		Token.PUNC_INCREMENT.value,
		Token.PUNC_INCREMENT.value,
		Token.PUNC_PLUS.value,
		"a",
		Token.PUNC_SEMICOLON.value
	];
	
	var tok, i = 0;
	while (tok = t.consume ()) {
		equal (tok.value, result[i++], "Test returned token");
	}
});
//*/

///*
test ("UTF-16 surrogates are converted", function() {
	var t = new Tokenizer(null, "\uD800\uDC00\uD800\uDFFF\uDBFF\uDFFF");
	var result = [0x10000,0x103FF,0x10FFFF];

	for (var i = 0; i < result.length; i++) {
		equal (t.nextch (), result[i], "UTF-16 converted 0xD800 0xDFFF -> 0x103FF");
	}
});
//*/
/*
test ("Random test", function() {
	var t = new Tokenizer(null, "#include <stdio.h>\nint main(int argc, char *argv[]) {\n\twhile (argc--) {\n\t\tputs (argv[argc]);\n\t}\n\treturn EXIT_SUCCESS;\n}\n"), tok;
	var result = ["#", "include", " ", "<", "stdio", ".", "h", ">", "\n", "int", " ", "main", "(", "int", " ", "argc", ",", " ", "char", " ", "*", "argv", "[", "]", ")", " ", "{", "\n\t", "while", " ", "(", "argc", "--", ")", " ", "{", "\n\t\t", "puts", " ", "(", "argv", "[", "argc", "]", ")", ";", "\n\t", "}", "\n\t", "return", " ", "EXIT_SUCCESS", ";", "\n", "}", "\n"];
	var i = 0;<commit_msg>Fix UTF-16 surrogate test messages.<commit_after>"use strict";

module ("Tokenizer");

///*
test ("Ending backslash removal and line number", function() {
	var source = "f\nop \\\nba\\z";
	var t = new Tokenizer(null, source);

	var result = "f\nop ba\\z";
	var lines = [1, 1, 2, 2, 2, 2, 3, 3, 3]; // [5] should in reality be 3, but I'm not picky
	var columns = [1, 2, 1, 2, 3, 4, 2, 3, 4]; // [5] -> 1

	var ch, cursor = 0, line = 1, column = 1;
	while ((ch = t.nextch())) {
		equal (String.fromCharCode(ch), result.charAt(cursor), "Test character @ "+cursor);
		equal (line, lines[cursor], "Test line number @ "+cursor);
		equal (column, columns[cursor], "Test column number @ "+cursor);

		cursor++;
		line = t.line;
		column = t.column;
	}
});
//*/

///*
test ("Greedy matching 'a+++++a;'", function() {
	var t = new Tokenizer(null, "a+++++a;");
	var result = [
		"a",
		Token.PUNC_INCREMENT.value,
		Token.PUNC_INCREMENT.value,
		Token.PUNC_PLUS.value,
		"a",
		Token.PUNC_SEMICOLON.value
	];
	
	var tok, i = 0;
	while (tok = t.consume ()) {
		equal (tok.value, result[i++], "Test returned token");
	}
});
//*/

///*
test ("UTF-16 surrogates are converted", function() {
	var t = new Tokenizer(null, "\uD800\uDC00\uD800\uDFFF\uDBFF\uDFFF");
	var result = [0x10000,0x103FF,0x10FFFF];

	for (var i = 0; i < result.length; i++) {
		equal (t.nextch (), result[i], "Read UTF-16 surrogate #" + (i+1) + " as 0x" + result[i].toString(16));
	}
});
//*/
/*
test ("Random test", function() {
	var t = new Tokenizer(null, "#include <stdio.h>\nint main(int argc, char *argv[]) {\n\twhile (argc--) {\n\t\tputs (argv[argc]);\n\t}\n\treturn EXIT_SUCCESS;\n}\n"), tok;
	var result = ["#", "include", " ", "<", "stdio", ".", "h", ">", "\n", "int", " ", "main", "(", "int", " ", "argc", ",", " ", "char", " ", "*", "argv", "[", "]", ")", " ", "{", "\n\t", "while", " ", "(", "argc", "--", ")", " ", "{", "\n\t\t", "puts", " ", "(", "argv", "[", "argc", "]", ")", ";", "\n\t", "}", "\n\t", "return", " ", "EXIT_SUCCESS", ";", "\n", "}", "\n"];
	var i = 0;
{'fraction_non_alphanumeric': 0.28075396825396826, 'fraction_numerical': 0.030009920634920636, 'mean_word_length': 3.456353591160221, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5183721', 'n_tokens_mistral': 1774, 'n_tokens_neox': 1626, 'n_words': 411}