<commit_before>package com.hypirion.subtex;

import clojure.lang.Keyword;
import java.util.function.Function;

// DFA of a subset of LaTeX.
class Lexer {

    enum State {
        Continue,
        End,
        None
    }

    enum Token {
        Text("text"),
        ParaEnd("para-end"),
        Call("call"),
        OpenBrace("open-brace"),
        CloseBrace("close-brace"),
        Quoted("quoted"),
        Comment("comment");

        Token(String kw) {
            this.keyword = Keyword.intern(null, kw);
        }
        Keyword keyword;
        // TODO? Math
    }

    Token token;
    Function<Character, State> state;
    int backtrack;

    Lexer() {
        state = this::stateBegin;
        backtrack = 0;
    }

    State state(Character c) {
        return state.apply(c);
    }

    void reset() {
        state = this::stateBegin;
        backtrack = 0;
    }

    State eof() {
        return State.None;
    }

    State stateBegin(Character c) {
        switch (c) {
        case '\n':
            state = this::stateNewline0;
            return State.Continue;
        case '{':
            token = Token.OpenBrace;
            return State.End;
        case '}':
            token = Token.CloseBrace;
            return State.End;
        case '\\':
            state = this::stateSlash0;
            return State.Continue;
        case '%':
            state = this::stateInComment;
            return State.Continue;
        default:
            state = this::stateText;
            return State.Continue;
        }
    }

    State stateSlash0(Character c) {
        if (Character.isLetter(c)) {
            state = this::stateSlash;
            return State.Continue;
        }
        // Token may or may not be unexpected; this is up to the user.
        token = Token.Quoted;
        return State.End;
    }

    State stateSlash(Character c) {
        if (Character.isLetterOrDigit(c)) {
            // Remain in stateSlash until we've found something non-letter/digit
            return State.Continue;
        }
        backtrack = 1;
        token = Token.Call;
        return State.End;
    }

    State stateInComment(Character c) {
        if (c == '\n') {
            state = this::stateBegin;
            token = Token.Comment;
            backtrack = 1;
            return State.End;
        }
        return State.Continue;
    }

    State stateText(Character c) {
        switch (c) {
        case '{': case '}': case '%': case '\\':
            token = Token.Text;
            backtrack = 1;
            return State.End;
        case '\n':
            state = this::stateTextNewline;
            return State.Continue;
        }
        return State.Continue;
    }

    State stateTextNewline(Character c) {
        switch (c) {
        case '{': case '}': case '%': case '\\':
            token = Token.Text;
            backtrack = 1;
            return State.End;
        case '\n':
            token = Token.Text;
            backtrack = 2;
            return State.End;
        }
        state = this::stateText;
        return State.Continue;
    }

    State stateNewline0(Character c) {
        switch (c) {
        case '{': case '}': case '%': case '\\':
            token = Token.Text;
            backtrack = 1;
            return State.End;
        case '\n':
            state = this::stateNewlines;
            return State.Continue;

        }
        state = this::stateBegin;
        return State.Continue;
    }<commit_msg>Wrap state functions in an enum<commit_after>package com.hypirion.subtex;

import clojure.lang.Keyword;
import java.util.function.Function;
import java.util.function.BiFunction;

// DFA of a subset of LaTeX.
class Lexer {

    enum State {
        Continue,
        End,
        None
    }

    enum Token {
        Text("text"),
        ParaEnd("para-end"),
        Call("call"),
        OpenBrace("open-brace"),
        CloseBrace("close-brace"),
        Quoted("quoted"),
        Comment("comment");

        Token(String kw) {
            this.keyword = Keyword.intern(null, kw);
        }
        Keyword keyword;
        // TODO? Math
    }

    private enum StateFn {
        Begin((a, b) -> a.stateBegin(b)),
        Slash0((a, b) -> a.stateSlash0(b)),
        Slash((a, b) -> a.stateSlash(b)),
        InComment((a, b) -> a.stateInComment(b)),
        Text((a, b) -> a.stateText(b)),
        TextNewline((a, b) -> a.stateTextNewline(b)),
        Newline0((a, b) -> a.stateNewline0(b)),
        Newlines((a, b) -> a.stateNewlines(b));

        StateFn(BiFunction<Lexer, Character, State> bi) {
            this.fn = bi;
        }

        BiFunction<Lexer, Character, State> fn;
    }

    Token token;
    StateFn state;
    int backtrack;

    Lexer() {
        state = StateFn.Begin;
        backtrack = 0;
    }

    State state(Character c) {
        return state.fn.apply(this, c);
    }

    void reset() {
        state = StateFn.Begin;
        backtrack = 0;
    }

    State eof() {
        return State.None;
    }

    State stateBegin(Character c) {
        switch (c) {
        case '\n':
            state = StateFn.Newline0;
            return State.Continue;
        case '{':
            token = Token.OpenBrace;
            return State.End;
        case '}':
            token = Token.CloseBrace;
            return State.End;
        case '\\':
            state = StateFn.Slash0;
            return State.Continue;
        case '%':
            state = StateFn.InComment;
            return State.Continue;
        default:
            state = StateFn.Text;
            return State.Continue;
        }
    }

    State stateSlash0(Character c) {
        if (Character.isLetter(c)) {
            state = StateFn.Slash;
            return State.Continue;
        }
        // Token may or may not be unexpected; this is up to the user.
        token = Token.Quoted;
        return State.End;
    }

    State stateSlash(Character c) {
        if (Character.isLetterOrDigit(c)) {
            // Remain in stateSlash until we've found something non-letter/digit
            return State.Continue;
        }
        backtrack = 1;
        token = Token.Call;
        return State.End;
    }

    State stateInComment(Character c) {
        if (c == '\n') {
            state = StateFn.Begin;
            token = Token.Comment;
            backtrack = 1;
            return State.End;
        }
        return State.Continue;
    }

    State stateText(Character c) {
        switch (c) {
        case '{': case '}': case '%': case '\\':
            token = Token.Text;
            backtrack = 1;
            return State.End;
        case '\n':
            state = StateFn.TextNewline;
            return State.Continue;
        }
        return State.Continue;
    }

    State stateTextNewline(Character c) {
        switch (c) {
        case '{': case '}': case '%': case '\\':
            token = Token.Text;
            backtrack = 1;
            return State.End;
        case '\n':
            token = Token.Text;
            backtrack = 2;
            return State.End;
        }
        state = StateFn.Text;
        return State.Continue;
    }

    State stateNewline0(Character c) {
        switch (c) {
        case '{': case '}': case '%': case '\\':
            token = Token.Text;
            backtrack = 1;
            return State.End;
        case '\n':
            state = StateFn.Newlines;
            return State.Continue;

        }
        state = StateFn.Begin;
        return State.Continue;
    }
{'fraction_non_alphanumeric': 0.11837221633085897, 'fraction_numerical': 0.003711558854718982, 'mean_word_length': 1.441747572815534, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '117623', 'n_tokens_mistral': 2247, 'n_tokens_neox': 2156, 'n_words': 520}