<commit_before><?php

namespace RomansTest\Lexer;

use PHPUnit\Framework\TestCase;
use Romans\Grammar\Grammar;
use Romans\Lexer\Exception as LexerException;
use Romans\Lexer\Lexer;

/**
 * Lexer Test
 */
class LexerTest extends TestCase
{
    /**
     * {@inheritdoc}
     */
    protected function setUp()
    {
        $this->lexer = new Lexer();
    }

    /**
     * Test Tokenize
     */
    public function testTokenize()
    {
        $this->assertSame([Grammar::T_I], $this->lexer->tokenize('I'));
        $this->assertSame([Grammar::T_V], $this->lexer->tokenize('V'));
        $this->assertSame([Grammar::T_X], $this->lexer->tokenize('X'));
    }

    /**
     * Test Invalid Token
     */
    public function testInvalidToken()
    {
        $this->expectException(LexerException::class);

        $this->lexer->tokenize('.');
    }

    /**
     * Test Tokenize with Multiple Tokens
     */
    public function testTokenizeWithMultipleTokens()
    {
        $this->assertSame([Grammar::T_I, Grammar::T_I], $this->lexer->tokenize('II'));
        $this->assertSame([Grammar::T_V, Grammar::T_I, Grammar::T_I], $this->lexer->tokenize('VII'));
        $this->assertSame([Grammar::T_X, Grammar::T_V, Grammar::T_I], $this->lexer->tokenize('XVI'));
    }

    /**
     * Test Invalid Token Message
     */
    public function testInvalidTokenMessage()
    {
        $this->expectException(LexerException::class);
        $this->expectExceptionMessage('Unknown token "." at position 2');

        $this->lexer->tokenize('XV.I');
    }

    /**
     * Test Tokenize with Lookahead
     */
    public function testTokenizeWithLookahead()
    {
        $this->assertSame([Grammar::T_IV], $this->lexer->tokenize('IV'));
        $this->assertSame([Grammar::T_IX], $this->lexer->tokenize('IX'));

        $this->assertSame([Grammar::T_CD, Grammar::T_XC, Grammar::T_IX], $this->lexer->tokenize('CDXCIX'));
    }

    /**
     * Test Tokenize With Simple And Lookahead
     */
    public function testTokenizeWithSimpleAndLookahead()
    {
        $this->assertSame([Grammar::T_CD, Grammar::T_L, Grammar::T_X, Grammar::T_IX], $this->lexer->tokenize('CDLXIX'));
    }
}<commit_msg>Create empty input test<commit_after><?php

namespace RomansTest\Lexer;

use PHPUnit\Framework\TestCase;
use Romans\Grammar\Grammar;
use Romans\Lexer\Exception as LexerException;
use Romans\Lexer\Lexer;

/**
 * Lexer Test
 */
class LexerTest extends TestCase
{
    /**
     * {@inheritdoc}
     */
    protected function setUp()
    {
        $this->lexer = new Lexer();
    }

    /**
     * Test Tokenize
     */
    public function testTokenize()
    {
        $this->assertSame([Grammar::T_I], $this->lexer->tokenize('I'));
        $this->assertSame([Grammar::T_V], $this->lexer->tokenize('V'));
        $this->assertSame([Grammar::T_X], $this->lexer->tokenize('X'));
    }

    /**
     * Test Invalid Token
     */
    public function testInvalidToken()
    {
        $this->expectException(LexerException::class);

        $this->lexer->tokenize('.');
    }

    /**
     * Test Tokenize with Multiple Tokens
     */
    public function testTokenizeWithMultipleTokens()
    {
        $this->assertSame([Grammar::T_I, Grammar::T_I], $this->lexer->tokenize('II'));
        $this->assertSame([Grammar::T_V, Grammar::T_I, Grammar::T_I], $this->lexer->tokenize('VII'));
        $this->assertSame([Grammar::T_X, Grammar::T_V, Grammar::T_I], $this->lexer->tokenize('XVI'));
    }

    /**
     * Test Invalid Token Message
     */
    public function testInvalidTokenMessage()
    {
        $this->expectException(LexerException::class);
        $this->expectExceptionMessage('Unknown token "." at position 2');

        $this->lexer->tokenize('XV.I');
    }

    /**
     * Test Tokenize with Lookahead
     */
    public function testTokenizeWithLookahead()
    {
        $this->assertSame([Grammar::T_IV], $this->lexer->tokenize('IV'));
        $this->assertSame([Grammar::T_IX], $this->lexer->tokenize('IX'));

        $this->assertSame([Grammar::T_CD, Grammar::T_XC, Grammar::T_IX], $this->lexer->tokenize('CDXCIX'));
    }

    /**
     * Test Tokenize With Simple And Lookahead
     */
    public function testTokenizeWithSimpleAndLookahead()
    {
        $this->assertSame([Grammar::T_CD, Grammar::T_L, Grammar::T_X, Grammar::T_IX], $this->lexer->tokenize('CDLXIX'));
    }

    /**
     * Test Tokenize With Empty Input
     */
    public function testTokenizeWithEmptyInput()
    {
        $this->assertSame([], $this->lexer->tokenize(''));
    }
}
{'fraction_non_alphanumeric': 0.17648356496801235, 'fraction_numerical': 0.00044120891242003087, 'mean_word_length': 2.994713656387665, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1275220', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1505, 'n_words': 227}