<commit_before>export type Token = string|symbol;
export const SYMBOL_EOF = Symbol("EOF");
export interface LexDefinitionSection{
	token: Token|null;
	pattern: string|RegExp;
}
export type LexDefinitions = Array<LexDefinitionSection>;
export var def:LexDefinitions = [
	{token:null, pattern:" "},
	{token:null, pattern:/\t|\r|\n/},
	{token:"LPAREN", pattern:"("},
	{token:"RPAREN", pattern:")"},
	{token:"PLUS", pattern:"+"},
	{token:"ASTERISK", pattern:"*"},
	{token:"DIGITS", pattern:/[1-9][0-9]*/},
{token:"INVALID", pattern:/./}
];
export type TokenList = Array<{token_type:Token, value:string}>;
export class Lexer{
	constructor(public def: LexDefinitions){
		// 正しいトークン定義が与えられているかチェック
		for(var i=0; i<this.def.length; i++){
			var token_pattern = this.def[i].pattern;
			if(typeof token_pattern == "string"){
				continue;
			}
			else if(token_pattern instanceof RegExp){
				continue;
			}
			throw new Error("invalid token definition: neither string nor RegExp object");
		}
	}
	exec(str: string):TokenList{
		var result:TokenList = [];
		while(true){
			if(str.length == 0) break;
			for(var i=0; i<this.def.length; i++){
				var token_type:Token|null = this.def[i].token;
				var token_pattern = this.def[i].pattern;
				var match:string;
				if(typeof token_pattern == "string"){
					if(str.substring(0,token_pattern.length) != token_pattern) continue;
					match = token_pattern;
				}
				else{
					// token_pattern: RegExp
					if(str.search(token_pattern) != 0) continue;
					match = token_pattern.exec(str)![0]; // str.searchの結果が0なのでexecは必ずnull以外が返る
				}
				// token_typeがnullなら処理を飛ばします
				if(token_type != null) {
					console.log(token_type," : "+match);
					result.push({token_type:token_type, value:match});
				}
				str = str.substring(match.length);
				break;
			}
		}
		// 最後にEOFトークンを付与
		result.push({token_type:SYMBOL_EOF, value:""});
		return result;
	}
}
<commit_msg>use sticky regexp to work properly with ^ and $ pattern<commit_after>export type Token = string|symbol;
export const SYMBOL_EOF = Symbol("EOF");
export interface LexDefinitionSection{
	token: Token|null;
	pattern: string|RegExp;
}
export type LexDefinitions = Array<LexDefinitionSection>;
export var def:LexDefinitions = [
	{token:null, pattern:" "},
	{token:null, pattern:/\t|\r|\n/y},
	{token:"LPAREN", pattern:"("},
	{token:"RPAREN", pattern:")"},
	{token:"PLUS", pattern:"+"},
	{token:"ASTERISK", pattern:"*"},
	{token:"DIGITS", pattern:/[1-9][0-9]*/y},
	{token:"INVALID", pattern:/./y}
];
export var lexerlexdef:LexDefinitions = [
	{token:"LABEL", pattern:/^[A-Z][A-Z0-9_]/y},
	{token:"REGEXP", pattern:/\/.*\/[gimuy]/y},
	{token:"STRING", pattern:/"([^"]|\")*"/y},
	{token:"WHITESPACE", pattern:/\s+/y},
	{token:"ENDLINE", pattern:/[ \f\t\v\u00a0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]*(\r\n|\r|\n)+/y},
	{token:"INVALID", pattern:/./y}
];
export type TokenList = Array<{token:Token, value:string}>;
export class Lexer{
	constructor(public def: LexDefinitions){
		// 正しいトークン定義が与えられているかチェック
		for(var i=0; i<this.def.length; i++){
			var token_pattern = this.def[i].pattern;
			if(typeof token_pattern == "string"){
				continue;
			}
			else if(token_pattern instanceof RegExp){
				continue;
			}
			throw new Error("invalid token definition: neither string nor RegExp object");
		}
	}
	exec(str: string):TokenList{
		var result:TokenList = [];
		let lastindex = 0;
		while(lastindex < str.length){
			for(var i=0; i<this.def.length; i++){
				var token:Token|null = this.def[i].token;
				var token_pattern = this.def[i].pattern;
				var match:string;
				if(typeof token_pattern == "string"){
					if(str.substring(lastindex,lastindex+token_pattern.length) != token_pattern) continue;
					match = token_pattern;
					lastindex += token_pattern.length;
				}
				else{
					// token_pattern: RegExp
					token_pattern.lastIndex = lastindex;
					let m = token_pattern.exec(str);
					if(m === null) continue; // マッチ失敗
					match = m[0];
					console.log("matched", match);
					lastindex = token_pattern.lastIndex; // lastindexを進める
				}
				// tokenがnullなら処理を飛ばします
				if(token != null) {
					console.log(token," : "+match);
					result.push({token:token, value:match});
				}
				break;
			}
		}
		// 最後にEOFトークンを付与
		result.push({token:SYMBOL_EOF, value:""});
		return result;
	}
}

{'fraction_non_alphanumeric': 0.1777262180974478, 'fraction_numerical': 0.011136890951276101, 'mean_word_length': 4.150537634408602, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2599415', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1572, 'n_words': 291}