<commit_before>package gherkin.lexer;

import gherkin.I18n;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class I18nLexer implements Lexer {
    private static final Pattern COMMENT_OR_EMPTY_LINE_PATTERN = Pattern.compile("^\\s*#|^\\s*$");
    private static final Pattern LANGUAGE_PATTERN = Pattern.compile("^\\s*#\\s*language\\s*:\\s*([a-zA-Z\\-]+)");
    private final Listener listener;
    private I18n i18n;

    public I18nLexer(Listener listener) {
        this(listener, false);
    }

    public I18nLexer(Listener listener, boolean forceRubyDummy) {
        this.listener = listener;
    }

    /**
     * @return the i18n language from the previous scanned source.
     */
    public I18n getI18nLanguage() {
        return i18n;
    }

    public void scan(String source) {
        createDelegate(source).scan(source);
    }

    private Lexer createDelegate(String source) {
        i18n = i18nLanguageForSource(source);
        return i18n.lexer(listener);
    }

    private I18n i18nLanguageForSource(String source) {
        String key = "en";
        for (String line : source.split("\\n")) {
            if (!COMMENT_OR_EMPTY_LINE_PATTERN.matcher(line).find()) {
                break;
            }
            Matcher matcher = LANGUAGE_PATTERN.matcher(line);
            if (matcher.find()) {
                key = matcher.group(1);
                break;
            }
        }
        return new I18n(key);
    }
}<commit_msg>handle BOM in a non-english feature file before language specification - java<commit_after>package gherkin.lexer;

import gherkin.I18n;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class I18nLexer implements Lexer {
    private static final Pattern COMMENT_OR_EMPTY_LINE_PATTERN = Pattern.compile("^([\\W]+)?\\s*#|^\\s*$");
    private static final Pattern LANGUAGE_PATTERN = Pattern.compile("^([\\W]+)?\\s*#\\s*language\\s*:\\s*([a-zA-Z\\-]+)");
    private final Listener listener;
    private I18n i18n;

    public I18nLexer(Listener listener) {
        this(listener, false);
    }

    public I18nLexer(Listener listener, boolean forceRubyDummy) {
        this.listener = listener;
    }

    /**
     * @return the i18n language from the previous scanned source.
     */
    public I18n getI18nLanguage() {
        return i18n;
    }

    public void scan(String source) {
        createDelegate(source).scan(source);
    }

    private Lexer createDelegate(String source) {
        i18n = i18nLanguageForSource(source);
        return i18n.lexer(listener);
    }

    private I18n i18nLanguageForSource(String source) {
        String key = "en";
        for (String line : source.split("\\n")) {
            if (!COMMENT_OR_EMPTY_LINE_PATTERN.matcher(line).find()) {
                break;
            }
            Matcher matcher = LANGUAGE_PATTERN.matcher(line);
            if (matcher.find()) {
                key = matcher.group(2);
                break;
            }
        }
        return new I18n(key);
    }
}
{'fraction_non_alphanumeric': 0.11973684210526316, 'fraction_numerical': 0.021710526315789475, 'mean_word_length': 2.37139689578714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1554076', 'n_tokens_mistral': 1013, 'n_tokens_neox': 944, 'n_words': 202}