<commit_before>package com.elderbyte.grammar.core.scanner;

import com.elderbyte.common.ArgumentNullException;

import java.util.*;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Stream;

/**
 * Scans an input string and turns it into a stream of tokens.
 *
 */
public class ExpressionScanner {

    /***************************************************************************
     *                                                                         *
     * Private Fields                                                          *
     *                                                                         *
     **************************************************************************/

    private final TerminalTokenManager terminalManager;
    private final Predicate<String> isIdentifierPredicate;
    private final Predicate<String> isLiteralValuePredicate;

    private static final Pattern DefaultIdentifierPattern = Pattern.compile("^[a-zA-Z]+\\w+$");

    private static final Pattern NumberPattern = Pattern.compile("^[0-9]+$");
    private static final Pattern FloatingPointNumberPattern = Pattern.compile("^[0-9]+\\.[0-9]+$");


    /***************************************************************************
     *                                                                         *
     * Constructors                                                            *
     *                                                                         *
     **************************************************************************/

    /**
     * Creates a new expression scanner using the given operator-set.
     * Uses the default word-pattern matcher, which matches alpha-characters as words.
     * @param terminalManager
     */
    public ExpressionScanner(TerminalTokenManager terminalManager){
        this(terminalManager, DefaultIdentifierPattern);
    }

    /**
     * Creates a new expression scanner using the given operator-set and word regex.
     * @param terminalManager
     */
    public ExpressionScanner(TerminalTokenManager terminalManager, Pattern isWordRegex){
        this(
                terminalManager,

                // Default identifier
                x -> isWordRegex.matcher(x).matches(),

                // Default literal values (strings not supported by default)
                x -> NumberPattern.matcher(x).matches() || FloatingPointNumberPattern.matcher(x).matches());
    }


    /**
     * Creates a new ExpressionTokenizer with the given Operators
     */
    public ExpressionScanner(
            TerminalTokenManager terminalManager,
            Predicate<String> isIdentifierPredicate,
            Predicate<String> isLiteralValuePredicate){

        if(terminalManager == null) throw new ArgumentNullException("terminalManager");
        if(isIdentifierPredicate == null) throw new ArgumentNullException("isWordPredicate");

        this.terminalManager = terminalManager;
        this.isIdentifierPredicate = isIdentifierPredicate;
        this.isLiteralValuePredicate = isLiteralValuePredicate;
    }


    /***************************************************************************
     *                                                                         *
     * Public API                                                              *
     *                                                                         *
     **************************************************************************/

    /**
     * Turns the given string into a token stream.
     * @param expression
     * @return
     * @throws ArgumentNullException If expression is null.
     */
    public Stream<Token> tokenize(String expression){

        if(expression == null) throw new ArgumentNullException("expression");


        List<Token> tokens = new ArrayList<>();


        RawTokenizer tokenizer = new RawTokenizer(expression, terminalManager.getTerminalKeywords());

        Token previous = null;

        for(String rawToken : tokenizer.tokenize()){
            Token t = emit(rawToken);
            if(t != null){

                if(t.getType() == TokenType.Operator){
                    if(previous == null || previous.getType() == TokenType.Operator){
                        t.markUnary();
                    }
                }

                tokens.add(t);
                previous = t;
            }

        }

        return tokens.stream();
    }

    /***************************************************************************
     *                                                                         *
     * Private methods                                                         *
     *                                                                         *
     **************************************************************************/


    private Token emit(String currentWord){
        Token t = terminalManager.findTerminal(currentWord);

        if(t == null){
            if(isLiteral(currentWord)){
                t = new Token(TokenType.Literal, currentWord);
            }else if(isIdentifier(currentWord)){
                t = new Token(TokenType.Identifier, currentWord);
            }
        }

        if(t != null && t.getType() != TokenType.Whitespace){
            return t;
        }else
            return null;
    }

    private boolean isIdentifier(String word){
        return isIdentifierPredicate.test(word);
    }

    private boolean isLiteral(String word){
        return isLiteralValuePredicate.test(word);
    }



}<commit_msg>improved error handling incase of unexpected tokens<commit_after>package com.elderbyte.grammar.core.scanner;

import com.elderbyte.common.ArgumentNullException;
import com.elderbyte.grammar.core.CodeDomException;

import java.util.*;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Stream;

/**
 * Scans an input string and turns it into a stream of tokens.
 *
 */
public class ExpressionScanner {

    /***************************************************************************
     *                                                                         *
     * Private Fields                                                          *
     *                                                                         *
     **************************************************************************/

    private final TerminalTokenManager terminalManager;
    private final Predicate<String> isIdentifierPredicate;
    private final Predicate<String> isLiteralValuePredicate;

    private static final Pattern DefaultIdentifierPattern = Pattern.compile("^[a-zA-Z]+\\w+$");

    private static final Pattern NumberPattern = Pattern.compile("^[0-9]+$");
    private static final Pattern FloatingPointNumberPattern = Pattern.compile("^[0-9]+\\.[0-9]+$");


    /***************************************************************************
     *                                                                         *
     * Constructors                                                            *
     *                                                                         *
     **************************************************************************/

    /**
     * Creates a new expression scanner using the given operator-set.
     * Uses the default word-pattern matcher, which matches alpha-characters as words.
     * @param terminalManager
     */
    public ExpressionScanner(TerminalTokenManager terminalManager){
        this(terminalManager, DefaultIdentifierPattern);
    }

    /**
     * Creates a new expression scanner using the given operator-set and word regex.
     * @param terminalManager
     */
    public ExpressionScanner(TerminalTokenManager terminalManager, Pattern isWordRegex){
        this(
                terminalManager,

                // Default identifier
                x -> isWordRegex.matcher(x).matches(),

                // Default literal values (strings not supported by default)
                x -> NumberPattern.matcher(x).matches() || FloatingPointNumberPattern.matcher(x).matches());
    }


    /**
     * Creates a new ExpressionTokenizer with the given Operators
     */
    public ExpressionScanner(
            TerminalTokenManager terminalManager,
            Predicate<String> isIdentifierPredicate,
            Predicate<String> isLiteralValuePredicate){

        if(terminalManager == null) throw new ArgumentNullException("terminalManager");
        if(isIdentifierPredicate == null) throw new ArgumentNullException("isWordPredicate");

        this.terminalManager = terminalManager;
        this.isIdentifierPredicate = isIdentifierPredicate;
        this.isLiteralValuePredicate = isLiteralValuePredicate;
    }


    /***************************************************************************
     *                                                                         *
     * Public API                                                              *
     *                                                                         *
     **************************************************************************/

    /**
     * Turns the given string into a token stream.
     * @param expression
     * @return
     * @throws ArgumentNullException If expression is null.
     */
    public Stream<Token> tokenize(String expression){

        if(expression == null) throw new ArgumentNullException("expression");


        List<Token> tokens = new ArrayList<>();


        RawTokenizer tokenizer = new RawTokenizer(expression, terminalManager.getTerminalKeywords());

        Token previous = null;

        for(String rawToken : tokenizer.tokenize()){
            Token t = emit(rawToken);
            if(t != null){

                if(t.getType() == TokenType.Operator){
                    if(previous == null || previous.getType() == TokenType.Operator){
                        t.markUnary();
                    }
                }

                tokens.add(t);
                previous = t;
            }

        }

        return tokens.stream();
    }

    /***************************************************************************
     *                                                                         *
     * Private methods                                                         *
     *                                                                         *
     **************************************************************************/


    private Token emit(String currentWord){
        Token t = terminalManager.findTerminal(currentWord);

        if(t == null){
            if(isLiteral(currentWord)){
                t = new Token(TokenType.Literal, currentWord);
            }else if(isIdentifier(currentWord)){
                t = new Token(TokenType.Identifier, currentWord);
            }else{
                throw new CodeDomException("Unexpected token: " + currentWord);
            }
        }

        if(t.getType() != TokenType.Whitespace){
            return t;
        }else
            return null;
    }

    private boolean isIdentifier(String word){
        return isIdentifierPredicate.test(word);
    }

    private boolean isLiteral(String word){
        return isLiteralValuePredicate.test(word);
    }



}
{'fraction_non_alphanumeric': 0.17828371278458843, 'fraction_numerical': 0.0010507880910683013, 'mean_word_length': 1.6879265709578724, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3406923', 'n_tokens_mistral': 2359, 'n_tokens_neox': 2181, 'n_words': 526}