def tokenize(s):
    keywords = {'IF', 'THEN', 'ENDIF', 'FOR', 'NEXT', 'GOSUB', 'RETURN'}
token_specification = [('NUMBER', '\\d+(\\.\\d*)?'), ('ASSIGN', ':='), (
    'END', ';'), ('ID', '[A-Za-z]+'), ('OP', '[+*\\/\\-]'), ('NEWLINE',
    '\\n'), ('SKIP', '[ \\t]')]
tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification)
get_token = re.compile(tok_regex).match
line = 1
pos = line_start = 0
mo = get_token(s)
while mo is not None:
    pass
Efficiently match multiple regexes in Python