<commit_before>import {LexDefinition, Language, GrammarDefinition} from "../def/language";
import {ParsingOperation, ParsingTable} from "../def/parsingtable";
import {SYMBOL_EOF, Token} from "../def/token";
import {ParserFactory} from "../parser/factory";
import {Parser} from "../parser/parser";
import {ParserGenerator} from "../parsergenerator/parsergenerator";

const lex: LexDefinition = {
	rules: [
		{token: "EXCLAMATION", pattern: "!"},
		{token: "VBAR", pattern: "|"},
		{token: "DOLLAR", pattern: "$"},
		{token: "COLON", pattern: ":"},
		{token: "SEMICOLON", pattern: ";"},
		{token: "LT", pattern: "<"},
		{token: "GT", pattern: ">"},
		{token: "COMMA", pattern: ","},
		{token: "LEX_BEGIN", pattern: "#lex_begin"},
		{token: "LEX_END", pattern: "#lex_end"},
		{token: "LEX_DEFAULT", pattern: "#lex_default"},
		{token: "DEFAULT", pattern: "#default"},
		{token: "LABEL", pattern: /[a-zA-Z_][a-zA-Z0-9_]*/},
		{
			token: "REGEXP", pattern: /\/.*\/[gimuy]*/,
			callback: (v) => {
				const tmp = v.split("/");
				const flags = tmp[tmp.length - 1];
				const p = v.slice(1, -1 - flags.length);
				return ["REGEXP", new RegExp(p, flags)];
			}
		},
		{token: "STRING", pattern: /".*"/, callback: (v) => ["STRING", v.slice(1, -1)]},
		{token: "STRING", pattern: /'.*'/, callback: (v) => ["STRING", v.slice(1, -1)]},
		{
			token: "START_BLOCK", pattern: /%*{+/,
			callback: (value, token, lex) => {
				const match = /(%*)({+)/.exec(value)!;
				const end_delimiter = "}".repeat(match[2].length) + match[1]!;
				lex.callState("callback");
				lex.addRule("body_block", {token: "BODY_BLOCK", pattern: new RegExp(`(?:.|\\s)*?(?<!})(?=${end_delimiter})(?!${end_delimiter}%+)`), states: ["callback"]});
				lex.addRule("end_block", {
					token: "END_BLOCK", pattern: end_delimiter, states: ["callback"],
					callback: (value, token, lex) => {
						lex.returnState();
						lex.removeRule("body_block");
						lex.removeRule("end_block");
					}
				});
			}
		},
		{token: null, pattern: /(\r\n|\r|\n)+/},
		{token: null, pattern: /[ \f\t\v\u00a0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]+/},
		{token: "INVALID", pattern: /./}
	]
};

const grammar: GrammarDefinition = {
	rules: [
		{
			ltoken: "LANGUAGE",
			pattern: ["OPTIONAL_LEX_EX_CALLBACKS", "LEX", "DEFAULT_CALLBACK", "GRAMMAR"],
			callback: (c) => {
				let start_symbol = c[3].start_symbol;
				// 開始記号の指定がない場合、最初の規則に設定]
				if (start_symbol === null) {
					if (c[3].sect.length > 0) start_symbol = c[3].sect[0].ltoken;
					else start_symbol = "";
				}
				return {lex: {rules: c[1]}, grammar: {rules: c[3].grammar, start_symbol: start_symbol}};
			}
		},
		{
			ltoken: "LANGUAGE",
			pattern: ["OPTIONAL_LEX_EX_CALLBACKS", "LEX", "GRAMMAR"],
			callback: (c) => {
				let start_symbol = c[2].start_symbol;
				// 開始記号の指定がない場合、最初の規則に設定]
				if (start_symbol === null) {
					if (c[2].sect.length > 0) start_symbol = c[2].sect[0].ltoken;
					else start_symbol = "";
				}
				return {lex: {rules: c[1]}, grammar: {rules: c[2].grammar, start_symbol: start_symbol}};
			}
		},
		{
			ltoken: "OPTIONAL_LEX_EX_CALLBACKS",
			pattern: ["LEX_EX_CALLBACKS"]
		},
		{
			ltoken: "OPTIONAL_LEX_EX_CALLBACKS",
			pattern: []
		},
		{
			ltoken: "LEX_EX_CALLBACKS",
			pattern: ["LEX_EX_CALLBACKS", "LEX_EX_CALLBACK"]
		},
		{
			ltoken: "LEX_EX_CALLBACKS",
			pattern: ["LEX_EX_CALLBACK"]
		},
		{
			ltoken: "LEX_EX_CALLBACK",
			pattern: ["LEX_EX_CALLBACK_BEGIN"]
		},
		{
			ltoken: "LEX_EX_CALLBACK",
			pattern: ["LEX_EX_CALLBACK_END"]
		},
		{
			ltoken: "LEX_EX_CALLBACK",
			pattern: ["LEX_EX_CALLBACK_DEFAULT"]
		},
		{
			ltoken: "LEX_EX_CALLBACK_BEGIN",
			pattern: ["LEX_BEGIN", "BLOCK"]
		},
		{
			ltoken: "LEX_EX_CALLBACK_END",
			pattern: ["LEX_END", "BLOCK"]
		},
		{
			ltoken: "LEX_EX_CALLBACK_DEFAULT",
			pattern: ["LEX_DEFAULT", "BLOCK"]
		},
		{
			ltoken: "LEX",
			pattern: ["LEX", "LEXSECT"],
			callback: (c) => c[0].concat([c[1]])
		},
		{
			ltoken: "LEX",
			pattern: ["LEXSECT"],
			callback: (c) => [c[0]]
		},
		{
			ltoken: "LEXSECT",
			pattern: ["MULTIPLE_LEXSTATE", "LEXLABEL", "LEXDEF", "LEXCALLBACK"],
			callback: (c) => ({token: c[1], pattern: c[2], states: c[0]})
		},
		{
			ltoken: "LEXSECT",
			pattern: ["LEXLABEL", "LEXDEF", "LEXCALLBACK"],
			callback: (c) => ({token: c[0], pattern: c[1]})
		},
		{
			ltoken: "LEXLABEL",
			pattern: ["LABEL"]
		},
		{
			ltoken: "LEXLABEL",
			pattern: ["EXCLAMATION"],
			callback: () => null
		},
		{
			ltoken: "LEXLABEL",
			pattern: ["EXCLAMATION", "LABEL"],
			callback: () => null
		},
		{
			ltoken: "LEXDEF",
			pattern: ["STRING"]
		},
		{
			ltoken: "LEXDEF",
			pattern: ["REGEXP"]
		},
		{
			ltoken: "MULTIPLE_LEXSTATE",
			pattern: ["LT", "LEXSTATE_LIST", "GT"],
			callback: (c) => c[1]
		},
		{
			ltoken: "LEXSTATE_LIST",
			pattern: ["LABEL", "COMMA", "LEXSTATE_LIST"],
			callback: (c) => [c[0], ...c[2]]
		},
		{
			ltoken: "LEXSTATE_LIST",
			pattern: ["LABEL"],
			callback: (c) => [c[0]]
		},
		{
			ltoken: "LEXSTATE",
			pattern: ["LT", "LABEL", "GT"],
			callback: (c) => [c[1]]
		},
		{
			ltoken: "LEXCALLBACK",
			pattern: ["BLOCK"],
			callback: (c) => c[0]
		},
		{
			ltoken: "LEXCALLBACK",
			pattern: []
		},
		{
			ltoken: "DEFAULT_CALLBACK",
			pattern: ["DEFAULT", "BLOCK"]
		},
		{
			ltoken: "GRAMMAR",
			pattern: ["RULES"]
		},
		{
			ltoken: "RULES",
			pattern: ["SECT", "RULES"],
			callback: (c) => {
				let start_symbol = c[1].start_symbol;
				if (c[0].start_symbol !== null) {
					start_symbol = c[0].start_symbol;
				}
				return {
					start_symbol,
					grammar: c[0].sect.concat(c[1].grammar)
				};
			}
		},
		{
			ltoken: "RULES",
			pattern: ["SECT"],
			callback: (c) => {
				let start_symbol = null;
				if (c[0].start_symbol !== null) {
					start_symbol = c[0].start_symbol;
				}
				return {
					start_symbol,
					grammar: c[0].sect
				};
			}
		},
		{
			ltoken: "SECT",
			pattern: ["SECTLABEL", "COLON", "DEF", "SEMICOLON"],
			callback: (c) => {
				const result = [];
				for (const pt of c[2]) {
					result.push({ltoken: c[0].label, pattern: pt});
				}
				return {start_symbol: c[0].start_symbol, sect: result};
			}
		},
		{
			ltoken: "SECTLABEL",
			pattern: ["LABEL"],
			callback: (c) => ({start_symbol: null, label: c[0]})
		},
		{
			ltoken: "SECTLABEL",
			pattern: ["DOLLAR", "LABEL"],
			callback: (c) => ({start_symbol: c[1], label: c[1]})
		},
		{
			ltoken: "DEF",
			pattern: ["PATTERN", "CALLBACK", "VBAR", "DEF"],
			callback: (c) => [c[0]].concat(c[3])
		},
		{
			ltoken: "DEF",
			pattern: ["PATTERN", "CALLBACK"],
			callback: (c) => [c[0]]
		},
		{
			ltoken: "PATTERN",
			pattern: ["SYMBOLLIST"]
		},
		{
			ltoken: "PATTERN",
			pattern: [],
			callback: () => []
		},
		{
			ltoken: "SYMBOLLIST",
			pattern: ["LABEL", "SYMBOLLIST"],
			callback: (c) => [c[0]].concat(c[1])
		},
		{
			ltoken: "SYMBOLLIST",
			pattern: ["LABEL"],
			callback: (c) => [c[0]]
		},
		{
			ltoken: "CALLBACK",
			pattern: ["BLOCK"],
			callback: (c) => c[0]
		},
		{
			ltoken: "CALLBACK",
			pattern: []
		},
		{
			ltoken: "BLOCK",
			pattern: ["START_BLOCK", "BODY_BLOCK", "END_BLOCK"],
			callback: (c) => [c[1]]
		}
	], start_symbol: "LANGUAGE"
};

/**
 * 言語定義文法の言語定義
 * @type Language
 */
export const language_language: Language = {lex: lex, grammar: grammar};

// 予めParsingTableを用意しておくことで高速化
/**
 * 言語定義文法の言語定義、の構文解析表
 * @type ParsingTable
 */
export const language_parsing_table: ParsingTable = [
	new Map<Token, ParsingOperation>([
		["LANGUAGE", {type: "goto", to: 1}],
		["LEX", {type: "goto", to: 2}],<commit_msg>read lex-callbacks and grammar-callbacks from language files<commit_after>import {LexDefinition, Language, GrammarDefinition} from "../def/language";
import {ParsingOperation, ParsingTable} from "../def/parsingtable";
import {SYMBOL_EOF, Token} from "../def/token";
import {Parser} from "../parser/parser";
import {ParserGenerator} from "../parsergenerator/parsergenerator";

const lex: LexDefinition = {
	rules: [
		{token: "EXCLAMATION", pattern: "!"},
		{token: "VBAR", pattern: "|"},
		{token: "DOLLAR", pattern: "$"},
		{token: "COLON", pattern: ":"},
		{token: "SEMICOLON", pattern: ";"},
		{token: "LT", pattern: "<"},
		{token: "GT", pattern: ">"},
		{token: "COMMA", pattern: ","},
		{token: "LEX_BEGIN", pattern: "#lex_begin"},
		{token: "LEX_END", pattern: "#lex_end"},
		{token: "LEX_DEFAULT", pattern: "#lex_default"},
		{token: "DEFAULT", pattern: "#default"},
		{token: "LABEL", pattern: /[a-zA-Z_][a-zA-Z0-9_]*/},
		{
			token: "REGEXP", pattern: /\/.*\/[gimuy]*/,
			callback: (v) => {
				const tmp = v.split("/");
				const flags = tmp[tmp.length - 1];
				const p = v.slice(1, -1 - flags.length);
				return ["REGEXP", new RegExp(p, flags)];
			}
		},
		{token: "STRING", pattern: /".*"/, callback: (v) => ["STRING", v.slice(1, -1)]},
		{token: "STRING", pattern: /'.*'/, callback: (v) => ["STRING", v.slice(1, -1)]},
		{
			token: "START_BLOCK", pattern: /%*{+/,
			callback: (value, token, lex) => {
				const match = /(%*)({+)/.exec(value)!;
				const end_delimiter = "}".repeat(match[2].length) + match[1]!;
				lex.callState("callback");
				lex.addRule("body_block", {token: "BODY_BLOCK", pattern: new RegExp(`(?:.|\\s)*?(?<!})(?=${end_delimiter})(?!${end_delimiter}%+)`), states: ["callback"]});
				lex.addRule("end_block", {
					token: "END_BLOCK", pattern: end_delimiter, states: ["callback"],
					callback: (value, token, lex) => {
						lex.returnState();
						lex.removeRule("body_block");
						lex.removeRule("end_block");
					}
				});
			}
		},
		{token: null, pattern: /(\r\n|\r|\n)+/},
		{token: null, pattern: /[ \f\t\v\u00a0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]+/},
		{token: "INVALID", pattern: /./}
	]
};

const grammar: GrammarDefinition = {
	rules: [
		{
			ltoken: "LANGUAGE",
			pattern: ["OPTIONAL_LEX_EX_CALLBACKS", "LEX", "DEFAULT_CALLBACK", "GRAMMAR"],
			callback: (c) => {
				let start_symbol = c[3].start_symbol;
				// 開始記号の指定がない場合、最初の規則に設定]
				if (start_symbol === null) {
					if (c[3].sect.length > 0) start_symbol = c[3].sect[0].ltoken;
					else start_symbol = "";
				}
				return {lex: {rules: c[1]}, grammar: {rules: c[3].grammar, start_symbol: start_symbol}};
			}
		},
		{
			ltoken: "LANGUAGE",
			pattern: ["OPTIONAL_LEX_EX_CALLBACKS", "LEX", "GRAMMAR"],
			callback: (c) => {
				let start_symbol = c[2].start_symbol;
				// 開始記号の指定がない場合、最初の規則に設定]
				if (start_symbol === null) {
					if (c[2].sect.length > 0) start_symbol = c[2].sect[0].ltoken;
					else start_symbol = "";
				}
				return {lex: {rules: c[1]}, grammar: {rules: c[2].grammar, start_symbol: start_symbol}};
			}
		},
		{
			ltoken: "OPTIONAL_LEX_EX_CALLBACKS",
			pattern: ["LEX_EX_CALLBACKS"]
		},
		{
			ltoken: "OPTIONAL_LEX_EX_CALLBACKS",
			pattern: []
		},
		{
			ltoken: "LEX_EX_CALLBACKS",
			pattern: ["LEX_EX_CALLBACKS", "LEX_EX_CALLBACK"]
		},
		{
			ltoken: "LEX_EX_CALLBACKS",
			pattern: ["LEX_EX_CALLBACK"]
		},
		{
			ltoken: "LEX_EX_CALLBACK",
			pattern: ["LEX_EX_CALLBACK_BEGIN"]
		},
		{
			ltoken: "LEX_EX_CALLBACK",
			pattern: ["LEX_EX_CALLBACK_END"]
		},
		{
			ltoken: "LEX_EX_CALLBACK",
			pattern: ["LEX_EX_CALLBACK_DEFAULT"]
		},
		{
			ltoken: "LEX_EX_CALLBACK_BEGIN",
			pattern: ["LEX_BEGIN", "BLOCK"]
		},
		{
			ltoken: "LEX_EX_CALLBACK_END",
			pattern: ["LEX_END", "BLOCK"]
		},
		{
			ltoken: "LEX_EX_CALLBACK_DEFAULT",
			pattern: ["LEX_DEFAULT", "BLOCK"]
		},
		{
			ltoken: "LEX",
			pattern: ["LEX", "LEXSECT"],
			callback: (c) => c[0].concat([c[1]])
		},
		{
			ltoken: "LEX",
			pattern: ["LEXSECT"],
			callback: (c) => [c[0]]
		},
		{
			ltoken: "LEXSECT",
			pattern: ["MULTIPLE_LEXSTATE", "LEXLABEL", "LEXDEF", "LEXCALLBACK"],
			callback: (c) => (c[3] === null ? {token: c[1], pattern: c[2], states: c[0]} : {token: c[1], pattern: c[2], states: c[0], callback: [3]})
		},
		{
			ltoken: "LEXSECT",
			pattern: ["LEXLABEL", "LEXDEF", "LEXCALLBACK"],
			callback: (c) => (c[2] === null ? {token: c[0], pattern: c[1]} : {token: c[0], pattern: c[1], callback: c[2]})
		},
		{
			ltoken: "LEXLABEL",
			pattern: ["LABEL"]
		},
		{
			ltoken: "LEXLABEL",
			pattern: ["EXCLAMATION"],
			callback: () => null
		},
		{
			ltoken: "LEXLABEL",
			pattern: ["EXCLAMATION", "LABEL"],
			callback: () => null
		},
		{
			ltoken: "LEXDEF",
			pattern: ["STRING"]
		},
		{
			ltoken: "LEXDEF",
			pattern: ["REGEXP"]
		},
		{
			ltoken: "MULTIPLE_LEXSTATE",
			pattern: ["LT", "LEXSTATE_LIST", "GT"],
			callback: (c) => c[1]
		},
		{
			ltoken: "LEXSTATE_LIST",
			pattern: ["LABEL", "COMMA", "LEXSTATE_LIST"],
			callback: (c) => [c[0], ...c[2]]
		},
		{
			ltoken: "LEXSTATE_LIST",
			pattern: ["LABEL"],
			callback: (c) => [c[0]]
		},
		{
			ltoken: "LEXSTATE",
			pattern: ["LT", "LABEL", "GT"],
			callback: (c) => [c[1]]
		},
		{
			ltoken: "LEXCALLBACK",
			pattern: ["BLOCK"]
		},
		{
			ltoken: "LEXCALLBACK",
			pattern: [],
			callback: () => null
		},
		{
			ltoken: "DEFAULT_CALLBACK",
			pattern: ["DEFAULT", "BLOCK"]
		},
		{
			ltoken: "GRAMMAR",
			pattern: ["RULES"]
		},
		{
			ltoken: "RULES",
			pattern: ["SECT", "RULES"],
			callback: (c) => {
				let start_symbol = c[1].start_symbol;
				if (c[0].start_symbol !== null) {
					start_symbol = c[0].start_symbol;
				}
				return {
					start_symbol,
					grammar: c[0].sect.concat(c[1].grammar)
				};
			}
		},
		{
			ltoken: "RULES",
			pattern: ["SECT"],
			callback: (c) => {
				let start_symbol = null;
				if (c[0].start_symbol !== null) {
					start_symbol = c[0].start_symbol;
				}
				return {
					start_symbol,
					grammar: c[0].sect
				};
			}
		},
		{
			ltoken: "SECT",
			pattern: ["SECTLABEL", "COLON", "DEF", "SEMICOLON"],
			callback: (c) => {
				const result = [];
				for (const def of c[2]) {
					result.push({ltoken: c[0].label, ...def});
				}
				return {start_symbol: c[0].start_symbol, sect: result};
			}
		},
		{
			ltoken: "SECTLABEL",
			pattern: ["LABEL"],
			callback: (c) => ({start_symbol: null, label: c[0]})
		},
		{
			ltoken: "SECTLABEL",
			pattern: ["DOLLAR", "LABEL"],
			callback: (c) => ({start_symbol: c[1], label: c[1]})
		},
		{
			ltoken: "DEF",
			pattern: ["PATTERN", "CALLBACK", "VBAR", "DEF"],
			callback: (c) => [c[1] === null ? {pattern: c[0]} : {pattern: c[0], callback: c[1]}].concat(c[3])
		},
		{
			ltoken: "DEF",
			pattern: ["PATTERN", "CALLBACK"],
			callback: (c) => [c[1] === null ? {pattern: c[0]} : {pattern: c[0], callback: c[1]}]
		},
		{
			ltoken: "PATTERN",
			pattern: ["SYMBOLLIST"]
		},
		{
			ltoken: "PATTERN",
			pattern: [],
			callback: () => []
		},
		{
			ltoken: "SYMBOLLIST",
			pattern: ["LABEL", "SYMBOLLIST"],
			callback: (c) => [c[0]].concat(c[1])
		},
		{
			ltoken: "SYMBOLLIST",
			pattern: ["LABEL"],
			callback: (c) => [c[0]]
		},
		{
			ltoken: "CALLBACK",
			pattern: ["BLOCK"]
		},
		{
			ltoken: "CALLBACK",
			pattern: [],
			callback: () => null
		},
		{
			ltoken: "BLOCK",
			pattern: ["START_BLOCK", "BODY_BLOCK", "END_BLOCK"],
			callback: (c) => c[1]
		}
	], start_symbol: "LANGUAGE"
};

/**
 * 言語定義文法の言語定義
 * @type Language
 */
export const language_language: Language = {lex: lex, grammar: grammar};

// 予めParsingTableを用意しておくことで高速化
/**
 * 言語定義文法の言語定義、の構文解析表
 * @type ParsingTable
 */
export const language_parsing_table: ParsingTable = [
	new Map<Token, ParsingOperation>([
		["LANGUAGE", {type: "goto", to: 1}],
		["LEX", {type: "goto", to: 2}],
{'fraction_non_alphanumeric': 0.22699227122166657, 'fraction_numerical': 0.012534909397934663, 'mean_word_length': 3.326496206799663, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4772268', 'n_tokens_mistral': 7511, 'n_tokens_neox': 6316, 'n_words': 1172}