token constants autogenerated by toolsbuildgeneratetoken py all tokname isterminal isnonterminal iseof endmarker 0 name 1 number 2 string 3 newline 4 indent 5 dedent 6 lpar 7 rpar 8 lsqb 9 rsqb 10 colon 11 comma 12 semi 13 plus 14 minus 15 star 16 slash 17 vbar 18 amper 19 less 20 greater 21 equal 22 dot 23 percent 24 lbrace 25 rbrace 26 eqequal 27 notequal 28 lessequal 29 greaterequal 30 tilde 31 circumflex 32 leftshift 33 rightshift 34 doublestar 35 plusequal 36 minequal 37 starequal 38 slashequal 39 percentequal 40 amperequal 41 vbarequal 42 circumflexequal 43 leftshiftequal 44 rightshiftequal 45 doublestarequal 46 doubleslash 47 doubleslashequal 48 at 49 atequal 50 rarrow 51 ellipsis 52 colonequal 53 exclamation 54 op 55 typeignore 56 typecomment 57 softkeyword 58 fstringstart 59 fstringmiddle 60 fstringend 61 comment 62 nl 63 these aren t used by the c tokenizer but are needed for tokenize py errortoken 64 encoding 65 ntokens 66 special definitions for cooperation with parser ntoffset 256 tokname value name for name value in globals items if isinstancevalue int and not name startswith all extendtokname values exacttokentypes exclamation notequal percent percentequal amper amperequal lpar rpar star doublestar doublestarequal starequal plus plusequal comma minus minequal rarrow dot ellipsis slash doubleslash doubleslashequal slashequal colon colonequal semi less leftshift leftshiftequal lessequal equal eqequal greater greaterequal rightshift rightshiftequal at atequal lsqb rsqb circumflex circumflexequal lbrace vbar vbarequal rbrace tilde def isterminalx return x ntoffset def isnonterminalx return x ntoffset def iseofx return x endmarker auto generated by tools build generate_token py these aren t used by the c tokenizer but are needed for tokenize py special definitions for cooperation with parser
__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']

ENDMARKER = 0
NAME = 1
NUMBER = 2
STRING = 3
NEWLINE = 4
INDENT = 5
DEDENT = 6
LPAR = 7
RPAR = 8
LSQB = 9
RSQB = 10
COLON = 11
COMMA = 12
SEMI = 13
PLUS = 14
MINUS = 15
STAR = 16
SLASH = 17
VBAR = 18
AMPER = 19
LESS = 20
GREATER = 21
EQUAL = 22
DOT = 23
PERCENT = 24
LBRACE = 25
RBRACE = 26
EQEQUAL = 27
NOTEQUAL = 28
LESSEQUAL = 29
GREATEREQUAL = 30
TILDE = 31
CIRCUMFLEX = 32
LEFTSHIFT = 33
RIGHTSHIFT = 34
DOUBLESTAR = 35
PLUSEQUAL = 36
MINEQUAL = 37
STAREQUAL = 38
SLASHEQUAL = 39
PERCENTEQUAL = 40
AMPEREQUAL = 41
VBAREQUAL = 42
CIRCUMFLEXEQUAL = 43
LEFTSHIFTEQUAL = 44
RIGHTSHIFTEQUAL = 45
DOUBLESTAREQUAL = 46
DOUBLESLASH = 47
DOUBLESLASHEQUAL = 48
AT = 49
ATEQUAL = 50
RARROW = 51
ELLIPSIS = 52
COLONEQUAL = 53
EXCLAMATION = 54
OP = 55
TYPE_IGNORE = 56
TYPE_COMMENT = 57
SOFT_KEYWORD = 58
FSTRING_START = 59
FSTRING_MIDDLE = 60
FSTRING_END = 61
COMMENT = 62
NL = 63

ERRORTOKEN = 64
ENCODING = 65
N_TOKENS = 66

NT_OFFSET = 256

tok_name = {value: name
            for name, value in globals().items()
            if isinstance(value, int) and not name.startswith('_')}
__all__.extend(tok_name.values())

EXACT_TOKEN_TYPES = {
    '!': EXCLAMATION,
    '!=': NOTEQUAL,
    '%': PERCENT,
    '%=': PERCENTEQUAL,
    '&': AMPER,
    '&=': AMPEREQUAL,
    '(': LPAR,
    ')': RPAR,
    '*': STAR,
    '**': DOUBLESTAR,
    '**=': DOUBLESTAREQUAL,
    '*=': STAREQUAL,
    '+': PLUS,
    '+=': PLUSEQUAL,
    ',': COMMA,
    '-': MINUS,
    '-=': MINEQUAL,
    '->': RARROW,
    '.': DOT,
    '...': ELLIPSIS,
    '/': SLASH,
    '//': DOUBLESLASH,
    '//=': DOUBLESLASHEQUAL,
    '/=': SLASHEQUAL,
    ':': COLON,
    ':=': COLONEQUAL,
    ';': SEMI,
    '<': LESS,
    '<<': LEFTSHIFT,
    '<<=': LEFTSHIFTEQUAL,
    '<=': LESSEQUAL,
    '=': EQUAL,
    '==': EQEQUAL,
    '>': GREATER,
    '>=': GREATEREQUAL,
    '>>': RIGHTSHIFT,
    '>>=': RIGHTSHIFTEQUAL,
    '@': AT,
    '@=': ATEQUAL,
    '[': LSQB,
    ']': RSQB,
    '^': CIRCUMFLEX,
    '^=': CIRCUMFLEXEQUAL,
    '{': LBRACE,
    '|': VBAR,
    '|=': VBAREQUAL,
    '}': RBRACE,
    '~': TILDE,
}

def ISTERMINAL(x):
    return x < NT_OFFSET

def ISNONTERMINAL(x):
    return x >= NT_OFFSET

def ISEOF(x):
    return x == ENDMARKER