import re
scanner = re.Scanner([('[0-9]+', lambda scanner, token: ('INTEGER', token)),
    ('[a-z_]+', lambda scanner, token: ('IDENTIFIER', token)), ('[,.]+', lambda
    scanner, token: ('PUNCTUATION', token)), ('\\s+', None)])
results, remainder = scanner.scan('45 pigeons, 23 cows, 11 spiders.')
print(results)
Pythonic way to implement a tokenizer