<commit_before>import Scanner from "./scanner";
import scanComment from "./comment";
import scanKey from "./key";
import scanFormat from "./format";
import scanPunctuator from "./punctuator";
import scanStringLiteral from "./string-literal";
import scanNumericLiteral from "./numeric-literal";
import scanRaw from "./raw";
import finalize from "./finalize";
import Token from "./token";

export default function lexer (code, options) {
	const scanner = new Scanner(code, options);
	let tokens = [];
	let isSyntax = false;
	let prevChar, buffer, lineNumber, lineStart, start;

	for (const char of scanner) {
		if (char === "{" && prevChar !== "\\") {
			const token = new Token(Token.Punctuator, char);
			token.configure(scanner);
			tokens.push(token);
			isSyntax = true;
			continue;
		} else if (char === "}" && prevChar !== "\\") {
			const lastToken = tokens[tokens.length - 1];

			if (lastToken.type !== Token.Comment && lastToken.type !== Token.Raw) {
				const token = new Token(Token.Punctuator, char);
				token.configure(scanner);
				tokens.push(token);
			}

			isSyntax = false;
			continue;
		}

		if (isSyntax) {
			if (buffer) {
				const lastToken = tokens.pop();
				const token = new Token(Token.Buffer, buffer);

				token.lineNumber = lineNumber;
				token.lineStart = lineStart;
				token.start = start;

				tokens.push(token, lastToken);

				buffer = null;
			}

			let tokenFound = false;

			[
				scanComment,
				scanKey,
				scanFormat,
				scanPunctuator,
				scanStringLiteral,
				scanNumericLiteral,
				scanRaw
			].forEach((scanFunction) => {
				const token = scanFunction(scanner);

				if (token) {
					if (token.type === Token.Comment || token.type === Token.Raw) {
						tokens.pop();
					}

					if (Array.isArray(token)) {
						tokens = tokens.concat(token);
					} else {
						tokens.push(token);
					}

					tokenFound = true;
				}
			});

			if (!tokenFound) {
				throw new Error(`Unexpected character: ${char}`);
			}
		} else {
			if (!buffer) {
				buffer = char;
				lineNumber = scanner.lineNumber;
				lineStart = scanner.lineStart;
				start = scanner.index;

				if (scanner.isLineTerminator(buffer)) {
					start++;
				}
			} else {
				buffer += char;
			}

			if (scanner.peek() === undefined) {
				const token = new Token(Token.Buffer, buffer);<commit_msg>Fix error when key tokens end with a number<commit_after>import Scanner from "./scanner";
import scanComment from "./comment";
import scanKey from "./key";
import scanFormat from "./format";
import scanPunctuator from "./punctuator";
import scanStringLiteral from "./string-literal";
import scanNumericLiteral from "./numeric-literal";
import scanRaw from "./raw";
import finalize from "./finalize";
import Token from "./token";

export default function lexer (code, options) {
	const scanner = new Scanner(code, options);
	let tokens = [];
	let isSyntax = false;
	let prevChar, buffer, lineNumber, lineStart, start;

	for (const char of scanner) {
		if (char === "{" && prevChar !== "\\") {
			const token = new Token(Token.Punctuator, char);
			token.configure(scanner);
			tokens.push(token);
			isSyntax = true;
			continue;
		} else if (char === "}" && prevChar !== "\\") {
			const lastToken = tokens[tokens.length - 1];

			if (lastToken.type !== Token.Comment && lastToken.type !== Token.Raw) {
				const token = new Token(Token.Punctuator, char);
				token.configure(scanner);
				tokens.push(token);
			}

			isSyntax = false;
			continue;
		}

		if (isSyntax) {
			if (buffer) {
				const lastToken = tokens.pop();
				const token = new Token(Token.Buffer, buffer);

				token.lineNumber = lineNumber;
				token.lineStart = lineStart;
				token.start = start;

				tokens.push(token, lastToken);

				buffer = null;
			}

			let tokenFound = false;

			const scanFunctions = [
				scanComment,
				scanKey,
				scanFormat,
				scanPunctuator,
				scanStringLiteral,
				scanNumericLiteral,
				scanRaw
			];

			for (const scanFunction of scanFunctions) {
				const token = scanFunction(scanner);

				if (token) {
					if (token.type === Token.Comment || token.type === Token.Raw) {
						tokens.pop();
					}

					if (Array.isArray(token)) {
						tokens = tokens.concat(token);
					} else {
						tokens.push(token);
					}

					tokenFound = true;

					break;
				}
			}

			if (!tokenFound) {
				throw new Error(`Unexpected character: ${char}`);
			}
		} else {
			if (!buffer) {
				buffer = char;
				lineNumber = scanner.lineNumber;
				lineStart = scanner.lineStart;
				start = scanner.index;

				if (scanner.isLineTerminator(buffer)) {
					start++;
				}
			} else {
				buffer += char;
			}

			if (scanner.peek() === undefined) {
				const token = new Token(Token.Buffer, buffer);
{'fraction_non_alphanumeric': 0.14125844594594594, 'fraction_numerical': 0.0004222972972972973, 'mean_word_length': 2.9409317803660566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3664708', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1449, 'n_words': 388}