<commit_before><?php
/**
 * Annotation - Parsing PHPDoc style annotations from comments
 *
 * @author  <EMAIL>
 * @link    https://github.com/panlatent/annotation
 * @license https://opensource.org/licenses/MIT
 */

namespace Panlatent\Annotation\Parser;

class CharacterStream implements SyntaxPositionInterface
{
    /**
     * @var string
     */
    protected $content;

    /**
     * @var int
     */
    protected $length;

    /**
     * @var int
     */
    protected $index = 0;

    /**
     * @var int
     */
    protected $lineNumber = 1;

    /**
     * @var int
     */
    protected $columnNumber = 1;

    /**
     * CharacterStream constructor.
     *
     * @param string $content
     */
    public function __construct($content)
    {
        $this->content = $content;
        $this->length = strlen($content);
    }

    /**
     * @return \Generator
     */
    public function generator()
    {
        $this->lineNumber = 1;
        $this->columnNumber = 1;

        for ($this->index = 0; $this->index < $this->length; ++$this->index) {
            $c = $this->content[$this->index];
            if (yield $c) {
                //yield [$lineNumber, $columnNumber];
            }

            if ("\n" == $c) {
                ++$this->lineNumber;
                $this->columnNumber = 1;
            } else {
                ++$this->columnNumber;
            }
        }

        yield "\0";
    }

    /**
     * @param string $char
     * @param bool  $regular
     * @return bool
     */
    public function expected($char, $regular = false)
    {
        if ($this->index + 1 > $this->length) {
            return false;
        } elseif ($regular) {
            return (bool)preg_match($regular, $char);
        } elseif (strlen($char) == 1) {
            return isset($this->content[$this->index + 1]) && $char === $this->content[$this->index + 1];
        }
        for ($i = 0; $i < strlen($char); ++$i) {
            if (isset($this->content[$this->index + 1]) && $char[$i] === $this->content[$this->index + 1]){
                return true;
            }
        }

        return false;
    }

    public function trace()
    {

    }

    public function back()
    {

    }

    public function skip($number = 1)
    {
        $this->index += $number;
    }

    /**
     * @return string
     */
    public function getContent()
    {
        return $this->content;
    }

    /**
     * @return int
     */
    public function getLength()<commit_msg>Add getChar method to CharacterStream<commit_after><?php
/**
 * Annotation - Parsing PHPDoc style annotations from comments
 *
 * @author  <EMAIL>
 * @link    https://github.com/panlatent/annotation
 * @license https://opensource.org/licenses/MIT
 */

namespace Panlatent\Annotation\Parser;

class CharacterStream implements SyntaxPositionInterface
{
    /**
     * @var string
     */
    protected $content;

    /**
     * @var int
     */
    protected $length;

    /**
     * @var int
     */
    protected $index = 0;

    /**
     * @var int
     */
    protected $lineNumber = 1;

    /**
     * @var int
     */
    protected $columnNumber = 1;

    /**
     * CharacterStream constructor.
     *
     * @param string $content
     */
    public function __construct($content)
    {
        $this->content = $content;
        $this->length = strlen($content);
    }

    /**
     * @return \Generator
     */
    public function generator()
    {
        $this->lineNumber = 1;
        $this->columnNumber = 1;

        for ($this->index = 0; $this->index < $this->length; ++$this->index) {
            $c = $this->content[$this->index];
            if (yield $c) {
                //yield [$lineNumber, $columnNumber];
            }

            if ("\n" == $c) {
                ++$this->lineNumber;
                $this->columnNumber = 1;
            } else {
                ++$this->columnNumber;
            }
        }

        yield "\0";
    }

    /**
     * @param string $char
     * @param bool  $regular
     * @return bool
     */
    public function expected($char, $regular = false)
    {
        if ($this->index + 1 > $this->length) {
            return false;
        } elseif ($regular) {
            return (bool)preg_match($regular, $char);
        } elseif (strlen($char) == 1) {
            return isset($this->content[$this->index + 1]) && $char === $this->content[$this->index + 1];
        }
        for ($i = 0; $i < strlen($char); ++$i) {
            if (isset($this->content[$this->index + 1]) && $char[$i] === $this->content[$this->index + 1]){
                return true;
            }
        }

        return false;
    }

    public function trace()
    {

    }

    public function back()
    {

    }

    public function skip($number = 1)
    {
        $this->index += $number;
    }

    public function getChar()
    {
        return $this->content[$this->index];
    }

    /**
     * @return string
     */
    public function getContent()
    {
        return $this->content;
    }

    /**
     * @return int
     */
    public function getLength()
{'fraction_non_alphanumeric': 0.15738925911407292, 'fraction_numerical': 0.006272050176401411, 'mean_word_length': 1.5916708989334687, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 59, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2861764', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1497, 'n_words': 330}