<commit_before><?php

/**
 * This file is part of the Miny framework.
 * (c) <NAME> <<EMAIL>>
 *
 * For licensing information see the LICENSE file.
 */

namespace Modules\Annotation;

class UseStatementParser
{
    private $namespace;
    private $position;

    public function __construct($source)
    {
        $this->tokens = token_get_all($source);
    }

    public function getNamespace()
    {
        if (!isset($this->namespace)) {
            $this->position = count($this->tokens) - 1;
            while (!$this->match(T_NAMESPACE)) {
                --$this->position;
                if (!$this->valid()) {
                    $this->namespace = '\\';

                    return $this->namespace;
                }
            }
            $this->skip(T_WHITESPACE);
            $this->skip(T_NS_SEPARATOR);
            $this->namespace = $this->parseClassName();
        }

        return $this->namespace;
    }

    public function getImports()
    {
        $imports = array();
        $this->getNamespace();

        $first = true;
        while ($this->valid()) {
            //find first use
            while (!$this->match(T_USE) && ($first || !$this->match(','))) {
                $this->step();
                if (!$this->valid()) {
                    break 2;
                }
            }
            $first = false;

            $this->skip(T_WHITESPACE);
            $this->skip(T_COMMENT);
            $this->skip(T_DOC_COMMENT);
            $this->skip(T_WHITESPACE);
            $this->skip(T_NS_SEPARATOR);

            $fqn       = $this->parseClassName();
            $shortName = $this->tokens[$this->position - 1][1];

            $this->skip(T_WHITESPACE);
            if ($this->match(T_AS) && $this->match(T_WHITESPACE)) {
                $shortName = $this->getCurrentToken();
            }
            $imports[$shortName] = $fqn;
        }

        return $imports;
    }

    private function skip($token)
    {
        if ($this->tokens[$this->position][0] === $token) {
            $this->step();
        }
    }

    /**
     * @param $token
     * @return bool
     */
    private function match($token)
    {
        if (!$this->valid()) {
            return false;
        }
        $current = $this->tokens[$this->position];
        if ($current === $token) {
            $this->step();

            return true;
        }
        if (!is_array($current)) {
            return false;
        }
        if ($current[0] === $token) {
            $this->step();

            return true;
        }

        return false;
    }

    private function step()
    {
        ++$this->position;
    }

    /**
     * @return mixed
     */
    private function getCurrentToken()
    {
        return $this->tokens[$this->position++][1];
    }

    /**
     * @return bool
     */
    private function valid()
    {
        return isset($this->tokens[$this->position]);
    }

    private function parseClassName()
    {
        $name = $this->getCurrentToken();
        while ($this->match(T_NS_SEPARATOR)) {
            $name .= '\\' . $this->getCurrentToken();
        }

        return $name;
    }
}
<commit_msg>Allow skipping any number of multiple tokens<commit_after><?php

/**
 * This file is part of the Miny framework.
 * (c) <NAME> <<EMAIL>>
 *
 * For licensing information see the LICENSE file.
 */

namespace Modules\Annotation;

class UseStatementParser
{
    private $namespace;
    private $position;

    public function __construct($source)
    {
        $this->tokens = token_get_all($source);
    }

    public function getNamespace()
    {
        if (!isset($this->namespace)) {
            $this->position = count($this->tokens) - 1;
            while (!$this->match(T_NAMESPACE)) {
                --$this->position;
                if (!$this->valid()) {
                    $this->namespace = '\\';

                    return $this->namespace;
                }
            }
            $this->skip(T_WHITESPACE, T_NS_SEPARATOR);
            $this->namespace = $this->parseClassName();
        }

        return $this->namespace;
    }

    public function getImports()
    {
        $imports = array();
        $this->getNamespace();

        $first = true;
        while ($this->valid()) {
            //find first use
            while (!$this->match(T_USE) && ($first || !$this->match(','))) {
                $this->step();
                if (!$this->valid()) {
                    break 2;
                }
            }
            $first = false;

            $this->skip(T_WHITESPACE, T_COMMENT, T_DOC_COMMENT, T_NS_SEPARATOR);

            $fqn       = $this->parseClassName();
            $shortName = $this->tokens[$this->position - 1][1];

            $this->skip(T_WHITESPACE);
            if ($this->match(T_AS) && $this->match(T_WHITESPACE)) {
                $shortName = $this->getCurrentToken();
            }
            $imports[$shortName] = $fqn;
        }

        return $imports;
    }

    private function skip()
    {
        $tokens = func_get_args();
        while (in_array($this->tokens[$this->position][0], $tokens)) {
            $this->step();
        }
    }

    /**
     * @param $token
     * @return bool
     */
    private function match($token)
    {
        if (!$this->valid()) {
            return false;
        }
        $current = $this->tokens[$this->position];
        if ($current === $token) {
            $this->step();

            return true;
        }
        if (!is_array($current)) {
            return false;
        }
        if ($current[0] === $token) {
            $this->step();

            return true;
        }

        return false;
    }

    private function step()
    {
        ++$this->position;
    }

    /**
     * @return mixed
     */
    private function getCurrentToken()
    {
        return $this->tokens[$this->position++][1];
    }

    /**
     * @return bool
     */
    private function valid()
    {
        return isset($this->tokens[$this->position]);
    }

    private function parseClassName()
    {
        $name = $this->getCurrentToken();
        while ($this->match(T_NS_SEPARATOR)) {
            $name .= '\\' . $this->getCurrentToken();
        }

        return $name;
    }
}

{'fraction_non_alphanumeric': 0.1575079872204473, 'fraction_numerical': 0.0022364217252396168, 'mean_word_length': 1.5094188376753508, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 100, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1974639', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1792, 'n_words': 337}