<commit_before><?php

namespace ESJ\Earley\Configuration\Tokenizer\String;

use ESJ\Earley\Parser;
use ESJ\Earley\Parser\Tree;
use ESJ\Earley\Recognizer;
use ESJ\Earley\Recognizer\Rule\Entry\Reference;
use ESJ\Earley\Recognizer\Rule\Entry\TokenReference;
use ESJ\Earley\Recognizer\Rule\Rule;
use ESJ\Earley\Recognizer\Rule\RuleCollection;
use ESJ\Earley\Tokenizer;
use ESJ\Earley\Tokenizer\Definition;

class TokenizerStringReader
{
    const START_RULE_NAME = 'program';

    /**
     * @var RuleCollection
     */
    private $rules;

    /**
     * @var Tokenizer
     */
    private $tokenizer;

    public function __construct()
    {
        $this->rules = $this->createEarleyRules();
        $this->tokenizer = $this->createTokenizer();
    }

    /**
     * @param string $input
     * @return Tree
     * @throws \Exception
     */
    public function readInput($input)
    {
        $recognizer = new Recognizer($this->rules, static::START_RULE_NAME);
        $parser = new Parser();
        $tokens = $this->tokenizer->tokenize($input);

        $state = $recognizer->recognize($tokens);
        $parseTree = $parser->parse($state, $tokens);

        return $parseTree;
    }

    /**
     * @return RuleCollection<commit_msg>Updated TokenizerStringReader to use the ParseTreeVisitor, returning a tokenizer instead of a useless parse tree.<commit_after><?php

namespace ESJ\Earley\Configuration\Tokenizer\String;

use ESJ\Earley\Configuration\Tokenizer\Visitor\ParseTreeVisitor;
use ESJ\Earley\Parser;
use ESJ\Earley\Recognizer;
use ESJ\Earley\Recognizer\Rule\Entry\Reference;
use ESJ\Earley\Recognizer\Rule\Entry\TokenReference;
use ESJ\Earley\Recognizer\Rule\Rule;
use ESJ\Earley\Recognizer\Rule\RuleCollection;
use ESJ\Earley\Tokenizer;
use ESJ\Earley\Tokenizer\Definition;

class TokenizerStringReader
{
    const START_RULE_NAME = 'program';

    /**
     * @var RuleCollection
     */
    private $rules;

    /**
     * @var Tokenizer
     */
    private $tokenizer;

    /**
     * @var ParseTreeVisitor
     */
    private $visitor;

    /**
     * @param ParseTreeVisitor $visitor
     */
    public function __construct(ParseTreeVisitor $visitor)
    {
        $this->visitor = $visitor;

        $this->rules = $this->createEarleyRules();
        $this->tokenizer = $this->createTokenizer();
    }

    /**
     * @param string $input
     * @return Tokenizer
     * @throws \Exception
     */
    public function readInput($input)
    {
        $recognizer = new Recognizer($this->rules, static::START_RULE_NAME);
        $parser = new Parser();
        $tokens = $this->tokenizer->tokenize($input);

        $state = $recognizer->recognize($tokens);
        $parseTree = $parser->parse($state, $tokens);

        return $this->visitor->dispatch($parseTree);
    }

    /**
     * @return RuleCollection
{'fraction_non_alphanumeric': 0.13123452423063317, 'fraction_numerical': 0.0, 'mean_word_length': 2.9497206703910615, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3107264', 'n_tokens_mistral': 926, 'n_tokens_neox': 899, 'n_words': 161}