src/JmesPath/Lexer.php
public function tokenize($input)
    {
        $tokens = [];

        if ($input === '') {
            goto eof;
        }

        $chars = str_split($input);

        while (false !== ($current = current($chars))) {

            // Every character must be in the transition character table.
            if (!isset(self::$transitionTable[$current])) {
                $tokens[] = [
                    'type'  => self::T_UNKNOWN,
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);
                continue;
            }

            $state = self::$transitionTable[$current];

            if ($state === self::STATE_SINGLE_CHAR) {

                // Consume simple tokens like ".", ",", "@", etc.
                $tokens[] = [
                    'type'  => $this->simpleTokens[$current],
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);

            } elseif ($state === self::STATE_IDENTIFIER) {

                // Consume identifiers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->validIdentifier[$current]));
                $tokens[] = [
                    'type'  => self::T_IDENTIFIER,
                    'value' => $buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_WHITESPACE) {

                // Skip whitespace
                next($chars);

            } elseif ($state === self::STATE_LBRACKET) {

                // Consume "[", "[?", and "[]"
                $position = key($chars);
                $actual = next($chars);
                if ($actual === ']') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FLATTEN,
                        'pos'   => $position,
                        'value' => '[]'
                    ];
                } elseif ($actual === '?') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FILTER,
                        'pos'   => $position,
                        'value' => '[?'
                    ];
                } else {
                    $tokens[] = [
                        'type'  => self::T_LBRACKET,
                        'pos'   => $position,
                        'value' => '['
                    ];
                }

            } elseif ($state === self::STATE_STRING_LITERAL) {

                // Consume raw string literals
                $t = $this->inside($chars, "'", self::T_LITERAL);
                $t['value'] = str_replace("\\'", "'", $t['value']);
                $tokens[] = $t;

            } elseif ($state === self::STATE_PIPE) {

                // Consume pipe and OR
                $tokens[] = $this->matchOr($chars, '|', '|', self::T_OR, self::T_PIPE);

            } elseif ($state == self::STATE_JSON_LITERAL) {

                // Consume JSON literals
                $token = $this->inside($chars, '`', self::T_LITERAL);
                if ($token['type'] === self::T_LITERAL) {
                    $token['value'] = str_replace('\\`', '`', $token['value']);
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state == self::STATE_NUMBER) {

                // Consume numbers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->numbers[$current]));
                $tokens[] = [
                    'type'  => self::T_NUMBER,
                    'value' => (int)$buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_QUOTED_STRING) {

                // Consume quoted identifiers
                $token = $this->inside($chars, '"', self::T_QUOTED_IDENTIFIER);
                if ($token['type'] === self::T_QUOTED_IDENTIFIER) {
                    $token['value'] = '"' . $token['value'] . '"';
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state === self::STATE_EQ) {

                // Consume equals
                $tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_UNKNOWN);

            } elseif ($state == self::STATE_AND) {

                $tokens[] = $this->matchOr($chars, '&', '&', self::T_AND, self::T_EXPREF);

            } elseif ($state === self::STATE_NOT) {

                // Consume not equal
                $tokens[] = $this->matchOr($chars, '!', '=', self::T_COMPARATOR, self::T_NOT);

            } else {

                // either '<' or '>'
                // Consume less than and greater than
                $tokens[] = $this->matchOr($chars, $current, '=', self::T_COMPARATOR, self::T_COMPARATOR);

            }
        }

        eof:
        $tokens[] = [
            'type'  => self::T_EOF,
            'pos'   => strlen($input),
            'value' => null
        ];

        return $tokens;
    }
public function tokenize($input)
    {
        $tokens = [];

        if ($input === '') {
            goto eof;
        }

        $chars = str_split($input);

        while (false !== ($current = current($chars))) {

            // Every character must be in the transition character table.
            if (!isset(self::$transitionTable[$current])) {
                $tokens[] = [
                    'type'  => self::T_UNKNOWN,
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);
                continue;
            }

            $state = self::$transitionTable[$current];

            if ($state === self::STATE_SINGLE_CHAR) {

                // Consume simple tokens like ".", ",", "@", etc.
                $tokens[] = [
                    'type'  => $this->simpleTokens[$current],
                    'pos'   => key($chars),
                    'value' => $current
                ];
                next($chars);

            } elseif ($state === self::STATE_IDENTIFIER) {

                // Consume identifiers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->validIdentifier[$current]));
                $tokens[] = [
                    'type'  => self::T_IDENTIFIER,
                    'value' => $buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_WHITESPACE) {

                // Skip whitespace
                next($chars);

            } elseif ($state === self::STATE_LBRACKET) {

                // Consume "[", "[?", and "[]"
                $position = key($chars);
                $actual = next($chars);
                if ($actual === ']') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FLATTEN,
                        'pos'   => $position,
                        'value' => '[]'
                    ];
                } elseif ($actual === '?') {
                    next($chars);
                    $tokens[] = [
                        'type'  => self::T_FILTER,
                        'pos'   => $position,
                        'value' => '[?'
                    ];
                } else {
                    $tokens[] = [
                        'type'  => self::T_LBRACKET,
                        'pos'   => $position,
                        'value' => '['
                    ];
                }

            } elseif ($state === self::STATE_STRING_LITERAL) {

                // Consume raw string literals
                $t = $this->inside($chars, "'", self::T_LITERAL);
                $t['value'] = str_replace("\\'", "'", $t['value']);
                $tokens[] = $t;

            } elseif ($state === self::STATE_PIPE) {

                // Consume pipe and OR
                $tokens[] = $this->matchOr($chars, '|', '|', self::T_OR, self::T_PIPE);

            } elseif ($state == self::STATE_JSON_LITERAL) {

                // Consume JSON literals
                $token = $this->inside($chars, '`', self::T_LITERAL);
                if ($token['type'] === self::T_LITERAL) {
                    $token['value'] = str_replace('\\`', '`', $token['value']);
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state == self::STATE_NUMBER) {

                // Consume numbers
                $start = key($chars);
                $buffer = '';
                do {
                    $buffer .= $current;
                    $current = next($chars);
                } while ($current !== false && isset($this->numbers[$current]));
                $tokens[] = [
                    'type'  => self::T_NUMBER,
                    'value' => (int)$buffer,
                    'pos'   => $start
                ];

            } elseif ($state === self::STATE_QUOTED_STRING) {

                // Consume quoted identifiers
                $token = $this->inside($chars, '"', self::T_QUOTED_IDENTIFIER);
                if ($token['type'] === self::T_QUOTED_IDENTIFIER) {
                    $token['value'] = '"' . $token['value'] . '"';
                    $token = $this->parseJson($token);
                }
                $tokens[] = $token;

            } elseif ($state === self::STATE_EQ) {

                // Consume equals
                $tokens[] = $this->matchOr($chars, '=', '=', self::T_COMPARATOR, self::T_UNKNOWN);

            } elseif ($state == self::STATE_AND) {

                $tokens[] = $this->matchOr($chars, '&', '&', self::T_AND, self::T_EXPREF);

            } elseif ($state === self::STATE_NOT) {

                // Consume not equal
                $tokens[] = $this->matchOr($chars, '!', '=', self::T_COMPARATOR, self::T_NOT);

            } else {

                // either '<' or '>'
                // Consume less than and greater than
                $tokens[] = $this->matchOr($chars, $current, '=', self::T_COMPARATOR, self::T_COMPARATOR);

            }
        }

        eof:
        $tokens[] = [
            'type'  => self::T_EOF,
            'pos'   => strlen($input),
            'value' => null
        ];

        return $tokens;
    }
Tokenize the JMESPath expression into an array of tokens hashes that
contain a 'type', 'value', and 'key'.

@param string $input JMESPath input

@return array
@throws SyntaxErrorException