<commit_before><?php

use KzykHys\ClassGenerator\Lexer;

/**
 * @author <NAME> <<EMAIL>>
 */
class LexerTest extends \PHPUnit_Framework_TestCase
{
    
    public function testTokenizeReturnsStream()
    {
        $lexer = new Lexer();
        $stream = $lexer->tokenize('Class < Extends << Implements');
        $this->assertInstanceOf('KzykHys\\ClassGenerator\\Token\\TokenStream', $stream);
    }
    
    public function testTokenizeAllTokenType()
    {
        $lexer = new Lexer();
        $stream = $lexer->tokenize(
<<<EOF
\Node\Node < Node\Node << Node \Node\Node\Node > COMMENT
> TOKEN INSIDE COMMENT < << : () [] 
+ PUBLIC:TYPE[set get is bind] > COMMENT
# PROTECTED[set get] > COMMENT
> COMMENT
- PRIVATE:TYPE

+ PUBLIC(ARG:TYPE ARG:TYPE):TYPE > COMMENT
# PROTECTED(ARG:TYPE) > COMMENT
- PRIVATE():TYPE
EOF
        );
        
        $className = 'KzykHys\\ClassGenerator\\Token\\Token';
        
        // line 1
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // \Node\Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_EXTD)); // <
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // Node\Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_IMPL)); // <<
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // \Node\Node\Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); //
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_CMNT)); // > COMMENT
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_EOL));
        
        // line 2
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_CMNT)); // > COMMENT
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_EOL));
        
        // line 3
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_PUBL)); // +
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // PUBLIC
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_TYPE)); // :
    }
    
}<commit_msg>fixed a test case<commit_after><?php

use KzykHys\ClassGenerator\Lexer;

/**
 * @author <NAME> <<EMAIL>>
 */
class LexerTest extends \PHPUnit_Framework_TestCase
{
    
    public function testTokenizeReturnsStream()
    {
        $lexer = new Lexer();
        $stream = $lexer->tokenize('Class < Extends << Implements');
        $this->assertInstanceOf('KzykHys\\ClassGenerator\\Token\\TokenStream', $stream);
    }
    
    public function testTokenizeAllTokenType()
    {
        $lexer = new Lexer();
        $stream = $lexer->tokenize(
<<<EOF
\Node\Node < Node\Node << Node \Node\Node\Node > COMMENT
> TOKEN INSIDE COMMENT < << : () [] 
+ PUBLIC:TYPE[set get is bind] > COMMENT
# PROTECTED[set get] > COMMENT
> COMMENT
- PRIVATE:TYPE

+ PUBLIC(ARG:TYPE ARG:TYPE):TYPE > COMMENT
# PROTECTED(ARG:TYPE) > COMMENT
- PRIVATE():TYPE
EOF
        );
        
        $className = 'KzykHys\\ClassGenerator\\Token\\Token';
        
        // line 1
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // \Node\Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_EXTD)); // <
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // Node\Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_IMPL)); // <<
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // \Node\Node\Node
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); //
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_CMNT)); // > COMMENT
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_EOL));
        
        // line 2
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_CMNT)); // > COMMENT
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_EOL));
        
        // line 3
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_PUBL)); // +
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_WTSP)); // 
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // PUBLIC
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_TYPE)); // :
        $this->assertInstanceOf($className, $stream->expect(Lexer::TOKEN_NODE)); // TYPE
    }
    
}
{'fraction_non_alphanumeric': 0.17927719423179633, 'fraction_numerical': 0.0010681858643403952, 'mean_word_length': 3.4800637958532694, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4657530', 'n_tokens_mistral': 1812, 'n_tokens_neox': 1762, 'n_words': 228}