<commit_before>package fitnesse.wikitext.parser;

import util.Maybe;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class Matcher {

    private interface ScanMatch {
        Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset);
    }

    private static final List<Character> defaultList =
            Collections.unmodifiableList(Arrays.asList('\0'));

    private ArrayList<ScanMatch> matches = new ArrayList<ScanMatch>();
    private List<Character> firsts = null;

    public List<Character> getFirsts() {
        return firsts != null ? firsts : defaultList;
    }

    public Matcher whitespace() {
        if (firsts == null) firsts = defaultList;
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int length = input.whitespaceLength(offset);
                return length > 0 ? new Maybe<Integer>(length) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher startLine() {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return isStartLine(input, symbols, offset) ? new Maybe<Integer>(0) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher startLineOrCell() {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return isStartLine(input, symbols, offset) || isStartCell(symbols)
                 ? new Maybe<Integer>(0) : Maybe.noInteger;
            }
        });
        return this;
    }

    private boolean isStartLine(ScanString input, SymbolStream symbols, int offset) {
        return input.startsLine(offset) || symbols.get(0).isStartLine();
    }

    private boolean isStartCell(SymbolStream symbols) {
        return symbols.get(0).isStartCell() ||
                (symbols.get(0).isType(SymbolType.Whitespace) &&
                        (symbols.get(1).isStartCell() || symbols.get(1).isLineType()));
    }

    public Matcher string(final String delimiter) {
        if (firsts == null) {
            firsts = new ArrayList<Character>();
            firsts.add(delimiter.charAt(0));
        }
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return input.matches(delimiter, offset) ? new Maybe<Integer>(delimiter.length()) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher listDigit() {
        firstIsDigit('1');
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return isDigitInput('1', input, offset) ? new Maybe<Integer>(1) : Maybe.noInteger;
            }
        });
        return this;
    }

    private boolean isDigitInput(char firstDigit, ScanString input, int offset) {
        for (char i = firstDigit; i <= '9'; i++) {
           if (input.matches(new String(new char[] {i}), offset)) return true;
        }
        return false;
    }

    private void firstIsDigit(char startDigit) {
        if (firsts == null) {
            firsts = new ArrayList<Character>();
            for (char i = startDigit; i <= '9'; i++) firsts.add(i);
        }
    }

    public Matcher digits() {
        firstIsDigit('0');
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int size = 0;
                while (isDigitInput('0', input, offset + size)) size++;
                return size > 0 ? new Maybe<Integer>(size) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher ignoreWhitespace() {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return new Maybe<Integer>(input.whitespaceLength(offset));
            }
        });
        return this;
    }

    public Matcher repeat(final char delimiter) {
        if (firsts == null) {
            firsts = new ArrayList<Character>();
            firsts.add(delimiter);
        }
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int size = 0;
                while (input.charAt(offset + size) == delimiter) size++;
                return size > 0 ? new Maybe<Integer>(size) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher endsWith(final char[] terminators) {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int size = 0;
                while (true) {
                    char candidate = input.charAt(offset + size);
                    if (candidate == 0) return Maybe.noInteger;
                    if (contains(terminators, candidate)) break;<commit_msg>Optimize 'matches' and 'first' allocation<commit_after>package fitnesse.wikitext.parser;

import util.Maybe;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class Matcher {

    private interface ScanMatch {
        Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset);
    }

    private static final List<Character> defaultList =
            Collections.unmodifiableList(Arrays.asList('\0'));

    private List<ScanMatch> matches = new ArrayList<ScanMatch>(4);
    private List<Character> firsts = null;

    public List<Character> getFirsts() {
        return firsts != null ? firsts : defaultList;
    }

    public Matcher whitespace() {
        if (firsts == null) firsts = defaultList;
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int length = input.whitespaceLength(offset);
                return length > 0 ? new Maybe<Integer>(length) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher startLine() {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return isStartLine(input, symbols, offset) ? new Maybe<Integer>(0) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher startLineOrCell() {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return isStartLine(input, symbols, offset) || isStartCell(symbols)
                 ? new Maybe<Integer>(0) : Maybe.noInteger;
            }
        });
        return this;
    }

    private boolean isStartLine(ScanString input, SymbolStream symbols, int offset) {
        return input.startsLine(offset) || symbols.get(0).isStartLine();
    }

    private boolean isStartCell(SymbolStream symbols) {
        return symbols.get(0).isStartCell() ||
                (symbols.get(0).isType(SymbolType.Whitespace) &&
                        (symbols.get(1).isStartCell() || symbols.get(1).isLineType()));
    }

    public Matcher string(final String delimiter) {
        if (firsts == null) {
            firsts = Collections.singletonList(delimiter.charAt(0));
        }
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return input.matches(delimiter, offset) ? new Maybe<Integer>(delimiter.length()) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher listDigit() {
        firstIsDigit('1');
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return isDigitInput('1', input, offset) ? new Maybe<Integer>(1) : Maybe.noInteger;
            }
        });
        return this;
    }

    private boolean isDigitInput(char firstDigit, ScanString input, int offset) {
        for (char i = firstDigit; i <= '9'; i++) {
           if (input.matches(new String(new char[] {i}), offset)) return true;
        }
        return false;
    }

    private void firstIsDigit(char startDigit) {
        if (firsts == null) {
            firsts = new ArrayList<Character>();
            for (char i = startDigit; i <= '9'; i++) firsts.add(i);
        }
    }

    public Matcher digits() {
        firstIsDigit('0');
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int size = 0;
                while (isDigitInput('0', input, offset + size)) size++;
                return size > 0 ? new Maybe<Integer>(size) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher ignoreWhitespace() {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                return new Maybe<Integer>(input.whitespaceLength(offset));
            }
        });
        return this;
    }

    public Matcher repeat(final char delimiter) {
        if (firsts == null) {
            firsts = Collections.singletonList(delimiter);
        }
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int size = 0;
                while (input.charAt(offset + size) == delimiter) size++;
                return size > 0 ? new Maybe<Integer>(size) : Maybe.noInteger;
            }
        });
        return this;
    }

    public Matcher endsWith(final char[] terminators) {
        matches.add(new ScanMatch() {
            public Maybe<Integer> match(ScanString input, SymbolStream symbols, int offset) {
                int size = 0;
                while (true) {
                    char candidate = input.charAt(offset + size);
                    if (candidate == 0) return Maybe.noInteger;
                    if (contains(terminators, candidate)) break;
{'fraction_non_alphanumeric': 0.10935695032730074, 'fraction_numerical': 0.004524451289949942, 'mean_word_length': 2.1415179921378895, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6333191', 'n_tokens_mistral': 2839, 'n_tokens_neox': 2730, 'n_words': 715}