<commit_before><?php

namespace Zend\Code\Scanner;

use Zend\Code\Scanner,
    Zend\Code\Exception;

class TokenArrayScanner implements Scanner
{
    protected $isScanned = false;

    protected $tokens    = array();

    protected $infos     = array();
    
    public function __construct($tokens = null, $options = null)
    {
        if ($tokens) {
            $this->setTokens($tokens);
        }
    }

    public function reset()
    {
        $this->isScanned = false;
        $this->infos     = array();
    }
    
    public function setTokens(array $tokens)
    {
        $this->tokens = $tokens;
        $this->reset();
    }
    
    protected function scan()
    {
        if ($this->isScanned) {
            return;
        }
        
        if (!$this->tokens) {
            throw new Exception\RuntimeException('No tokens were provided');
        }

        $currentNamespace = null;
        
        for ($tokenIndex = 0; $tokenIndex < count($this->tokens); $tokenIndex++) {
            $token = $this->tokens[$tokenIndex];
            
            // tokens with some value are arrays (will have a token identifier, & line num)
            $fastForward = 0;
            switch ($token[0]) {
                case T_DOC_COMMENT:
                    echo 'Found Doc Comment' . PHP_EOL;
                    break;

                case T_NAMESPACE:
                    $currentNamespace = $this->scanNamespace($tokenIndex, $fastForward);
                    break;

                case T_USE:
                    // process uses
                    $this->scanUse($tokenIndex, $fastForward);
                    break;

                case T_INCLUDE:
                case T_INCLUDE_ONCE:
                case T_REQUIRE:
                case T_REQUIRE_ONCE:
                    // process include
                    $this->scanInclude($tokenIndex, $fastForward);
                    break;

                case T_FINAL:
                case T_ABSTRACT:
                case T_CLASS:
                case T_INTERFACE:
                    $this->scanClass($tokenIndex, $fastForward, $currentNamespace);
                    break;

                case T_FUNCTION:
                    $this->scanFunction($tokenIndex, $fastForward, $currentNamespace);<commit_msg>Altered Zend\Di with additional tests, ability to hash parameter calls, and ability to configure the component<commit_after><?php

namespace Zend\Code\Scanner;

use Zend\Code\Scanner,
    Zend\Code\Exception;

class TokenArrayScanner implements Scanner
{
    protected $isScanned = false;

    protected $tokens    = array();

    protected $infos     = array();
    
    public function __construct($tokens = null, $options = null)
    {
        if ($tokens) {
            $this->setTokens($tokens);
        }
    }

    public function reset()
    {
        $this->isScanned = false;
        $this->infos     = array();
    }
    
    public function setTokens(array $tokens)
    {
        $this->tokens = $tokens;
        $this->reset();
    }
    
    protected function scan()
    {
        if ($this->isScanned) {
            return;
        }
        
        if (!$this->tokens) {
            throw new Exception\RuntimeException('No tokens were provided');
        }

        $currentNamespace = null;
        
        for ($tokenIndex = 0; $tokenIndex < count($this->tokens); $tokenIndex++) {
            $token = $this->tokens[$tokenIndex];
            
            // tokens with some value are arrays (will have a token identifier, & line num)
            $fastForward = 0;
            switch ($token[0]) {
                case T_DOC_COMMENT:
                    // @todo Implement docblock scanner
                    break;

                case T_NAMESPACE:
                    $currentNamespace = $this->scanNamespace($tokenIndex, $fastForward);
                    break;

                case T_USE:
                    // process uses
                    $this->scanUse($tokenIndex, $fastForward);
                    break;

                case T_INCLUDE:
                case T_INCLUDE_ONCE:
                case T_REQUIRE:
                case T_REQUIRE_ONCE:
                    // process include
                    $this->scanInclude($tokenIndex, $fastForward);
                    break;

                case T_FINAL:
                case T_ABSTRACT:
                case T_CLASS:
                case T_INTERFACE:
                    $this->scanClass($tokenIndex, $fastForward, $currentNamespace);
                    break;

                case T_FUNCTION:
                    $this->scanFunction($tokenIndex, $fastForward, $currentNamespace);
{'fraction_non_alphanumeric': 0.09795918367346938, 'fraction_numerical': 0.0012889366272824919, 'mean_word_length': 1.4111859140341791, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6939566', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1165, 'n_words': 281}