src/Lexer.php
public function tokenise ($pattern, $expand = false) {
        preg_match_all('
        /
                (?<class_U_explicit>\\\U)
                \{
                    (?<class_U_repetition>[0-9]+)
                \}
            |
                (?<class_U_implicit>\\\U)
            |
                \[
                    (?<range_token_explicit>[^]]+)
                \]
                \{
                    (?<range_repetition>[0-9]+)
                \}
            |
                \[
                    (?<range_token_implicit>[^]]+)
                \]
            |
                (?<literal_string>[^\\\[]+)

        /x
        ', $pattern, $matches, \PREG_SET_ORDER);

        $tokens = [];

        foreach ($matches as $match) {
            
            if (!empty($match['class_U_explicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => (int) $match['class_U_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['class_U_implicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_explicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_explicit'],
                    'repetition' => (int) $match['range_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_explicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_implicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_implicit'],
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_implicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['literal_string'])) {
                $tokens[] = [
                    'type' => 'literal',
                    'string' => $match['literal_string']
                ];
            }
        }

        return $tokens;
    }
public function tokenise ($pattern, $expand = false) {
        preg_match_all('
        /
                (?<class_U_explicit>\\\U)
                \{
                    (?<class_U_repetition>[0-9]+)
                \}
            |
                (?<class_U_implicit>\\\U)
            |
                \[
                    (?<range_token_explicit>[^]]+)
                \]
                \{
                    (?<range_repetition>[0-9]+)
                \}
            |
                \[
                    (?<range_token_implicit>[^]]+)
                \]
            |
                (?<literal_string>[^\\\[]+)

        /x
        ', $pattern, $matches, \PREG_SET_ORDER);

        $tokens = [];

        foreach ($matches as $match) {
            
            if (!empty($match['class_U_explicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => (int) $match['class_U_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['class_U_implicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_explicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_explicit'],
                    'repetition' => (int) $match['range_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_explicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_implicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_implicit'],
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_implicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['literal_string'])) {
                $tokens[] = [
                    'type' => 'literal',
                    'string' => $match['literal_string']
                ];
            }
        }

        return $tokens;
    }
Tokeniser explodes input into components describing the properties expressed in the pattern.

@param string $pattern
@param boolean $expand Augment token definition with the haystack of possible values.
@return array