<commit_before><?php
namespace Snout;

use \Ds\Map;
use \Ds\OutOfBoundsException;
use \Snout\Exceptions\ConfigurationException;
use \Snout\Exceptions\ParserException;
use \Snout\Token;
use \Snout\Lexer;

/**
 * Parser.
 */
class Parser
{
    /**
     * @var Lexer $lexer
     */
    private $lexer;

    /**
     * @var Map $config
     */
    private $config;

    /**
     * @param Map   $config
     * @param Lexer $lexer
     */
    public function __construct(Map $config, Lexer $lexer)
    {
        $this->configure($config);
        $this->lexer = $lexer;
    }

    /**
     * @param Map $config
     * @return void
     **/
    public function configure(Map $config) : void
    {
        try {
            $config = $config->get('parser');
        } catch (OutOfBoundsException $e) {
            throw new ConfigurationException('parser');
        }

        $config->put(
            'invalid',
            $config->get('invalid')->map(
                function ($key, $value) {
                    return constant("\Snout\Token::{$value}");
                }
            )
        );

        $this->config = $config;
    }

    /**
     * @return string Current token.
     */
    public function getToken() : string
    {
        return $this->lexer->getToken();
    }

    /**
     * @return bool Payload flag.
     */
    public function hasPayload() : bool
    {
        return $this->lexer->hasPayload();
    }

    /**
     * @return string Current payload.
     */
    public function getPayload() : string
    {
        return $this->lexer->getPayload();
    }

    /**
     * Accept and scan.
     *
     * @return void
     */
    public function accept() : void
    {
        $token = $this->lexer->getToken();
        $this->checkInvalid($token);
        $this->lexer->next();
    }

    /**
     * Accept token and scan.
     *
     * @param string $valid Valid next token.
     * @throws ParserException On unexpected token.
     * @return void
     */
    public function acceptToken(string $valid) : void
    {
        $token = $this->lexer->getToken();
        $this->checkInvalid($token);

        if ($token !== $valid) {
            $column = $this->lexer->getColumn();

            throw new ParserException(
                "Unexpected token '{$token}'. Expecting token '{$valid}'. "
                . "At char {$column}."
            );
        }

        $this->lexer->next();
    }

    /**
     * Check if token is invalid.
     *
     * @param string $token Token.
     * @throws ParserException On invalid token.
     * @return void
     */
    private function checkInvalid(string $token) : void
    {
        if (!$this->config->get('invalid')->hasValue($token)) {
            return;
        }

        $column = $this->lexer->getColumn();

        throw new ParserException(
            "Invalid token '{$token}'. At char {$column}."
        );
    }
}<commit_msg>Combine accept methods, add end check method<commit_after><?php
namespace Snout;

use \Ds\Map;
use \Ds\OutOfBoundsException;
use \Snout\Exceptions\ConfigurationException;
use \Snout\Exceptions\ParserException;
use \Snout\Token;
use \Snout\Lexer;

/**
 * Parser.
 */
class Parser
{
    /**
     * @var Lexer $lexer
     */
    private $lexer;

    /**
     * @var Map $config
     */
    private $config;

    /**
     * @param Map   $config
     * @param Lexer $lexer
     */
    public function __construct(Map $config, Lexer $lexer)
    {
        $this->configure($config);
        $this->lexer = $lexer;
    }

    /**
     * @param Map $config
     * @return void
     **/
    public function configure(Map $config) : void
    {
        try {
            $config = $config->get('parser');
        } catch (OutOfBoundsException $e) {
            throw new ConfigurationException('parser');
        }

        $config->put(
            'invalid',
            $config->get('invalid')->map(
                function ($key, $value) {
                    return constant("\Snout\Token::{$value}");
                }
            )
        );

        $this->config = $config;
    }

    /**
     * @return string Current token.
     */
    public function getToken() : string
    {
        return $this->lexer->getToken();
    }

    /**
     * @return bool Payload flag.
     */
    public function hasPayload() : bool
    {
        return $this->lexer->hasPayload();
    }

    /**
     * @return string Current payload.
     */
    public function getPayload() : string
    {
        return $this->lexer->getPayload();
    }

    /**
     * @return bool
     */
    public function isEnd() : string
    {
        return $this->lexer->getToken() === Token::END;
    }

    /**
     * Accept token and scan.
     *
     * @param string $token   Valid next token.
     * @param mixed  $payload Valid next payload.
     * @throws ParserException On unexpected token.
     * @return void
     */
    public function accept(string $token = null, $payload = null) : void
    {
        $next_token = $this->lexer->getToken();

        if ($this->config->get('invalid')->hasValue($next_token)) {
            throw new ParserException(
                "Invalid token '{$next_token}'. "
                . "At char {$this->lexer->getColumn()}."
            );
        }

        if ($token !== null && $next_token !== $token) {
            throw new ParserException(
                "Unexpected token '{$next_token}'. Expecting token '{$token}'. "
                . "At char {$this->lexer->getColumn()}."
            );
        }

        if ($payload !== null) {
            if (!$this->lexer->hasPayload()) {
                throw new ParserException(
                    "Expecting '{$payload}'. "
                    . "At char {$this->lexer->getColumn()}."
                );
            }

            if (!$this->lexer->getPayload() !== $payload) {
                throw new ParserException(
                    "Unexpected '{$this->lexer->getPayload()}'"
                    . "Expecting '{$payload}'. "
                    . "At char {$this->lexer->getColumn()}."
                );
            }
        }

        $this->lexer->next();
    }
}
{'fraction_non_alphanumeric': 0.15038205169891075, 'fraction_numerical': 0.0, 'mean_word_length': 1.599070553443177, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6223290', 'n_tokens_mistral': 1815, 'n_tokens_neox': 1764, 'n_words': 398}