framework/db/SqlTokenizer.php
public function tokenize()
    {
        $this->length = mb_strlen($this->sql, 'UTF-8');
        $this->offset = 0;
        $this->_substrings = [];
        $this->_buffer = '';
        $this->_token = new SqlToken([
            'type' => SqlToken::TYPE_CODE,
            'content' => $this->sql,
        ]);
        $this->_tokenStack = new \SplStack();
        $this->_tokenStack->push($this->_token);
        $this->_token[] = new SqlToken(['type' => SqlToken::TYPE_STATEMENT]);
        $this->_tokenStack->push($this->_token[0]);
        $this->_currentToken = $this->_tokenStack->top();
        while (!$this->isEof()) {
            if ($this->isWhitespace($length) || $this->isComment($length)) {
                $this->addTokenFromBuffer();
                $this->advance($length);
                continue;
            }

            if ($this->tokenizeOperator($length) || $this->tokenizeDelimitedString($length)) {
                $this->advance($length);
                continue;
            }

            $this->_buffer .= $this->substring(1);
            $this->advance(1);
        }
        $this->addTokenFromBuffer();
        if ($this->_token->getHasChildren() && !$this->_token[-1]->getHasChildren()) {
            unset($this->_token[-1]);
        }

        return $this->_token;
    }
public function tokenize()
    {
        $this->length = mb_strlen($this->sql, 'UTF-8');
        $this->offset = 0;
        $this->_substrings = [];
        $this->_buffer = '';
        $this->_token = new SqlToken([
            'type' => SqlToken::TYPE_CODE,
            'content' => $this->sql,
        ]);
        $this->_tokenStack = new \SplStack();
        $this->_tokenStack->push($this->_token);
        $this->_token[] = new SqlToken(['type' => SqlToken::TYPE_STATEMENT]);
        $this->_tokenStack->push($this->_token[0]);
        $this->_currentToken = $this->_tokenStack->top();
        while (!$this->isEof()) {
            if ($this->isWhitespace($length) || $this->isComment($length)) {
                $this->addTokenFromBuffer();
                $this->advance($length);
                continue;
            }

            if ($this->tokenizeOperator($length) || $this->tokenizeDelimitedString($length)) {
                $this->advance($length);
                continue;
            }

            $this->_buffer .= $this->substring(1);
            $this->advance(1);
        }
        $this->addTokenFromBuffer();
        if ($this->_token->getHasChildren() && !$this->_token[-1]->getHasChildren()) {
            unset($this->_token[-1]);
        }

        return $this->_token;
    }
Tokenizes and returns a code type token.
@return SqlToken code type token.