framework/db/SqlTokenizer.php
private function addTokenFromBuffer()
    {
        if ($this->_buffer === '') {
            return;
        }

        $isKeyword = $this->isKeyword($this->_buffer, $content);
        $this->_currentToken[] = new SqlToken([
            'type' => $isKeyword ? SqlToken::TYPE_KEYWORD : SqlToken::TYPE_TOKEN,
            'content' => is_string($content) ? $content : $this->_buffer,
            'startOffset' => $this->offset - mb_strlen($this->_buffer, 'UTF-8'),
            'endOffset' => $this->offset,
        ]);
        $this->_buffer = '';
    }
private function addTokenFromBuffer()
    {
        if ($this->_buffer === '') {
            return;
        }

        $isKeyword = $this->isKeyword($this->_buffer, $content);
        $this->_currentToken[] = new SqlToken([
            'type' => $isKeyword ? SqlToken::TYPE_KEYWORD : SqlToken::TYPE_TOKEN,
            'content' => is_string($content) ? $content : $this->_buffer,
            'startOffset' => $this->offset - mb_strlen($this->_buffer, 'UTF-8'),
            'endOffset' => $this->offset,
        ]);
        $this->_buffer = '';
    }
Determines a type of text in the buffer, tokenizes it and adds it to the token children.