<commit_before>import sys, re

def indent(s):
	yield 'nl', '\n'
	yield 'indent', len(s)

MATCHING = {
	None: [
		(r'\n', 'nl', 'indent', indent),
		(r' ', '!sp', None, None),
		(r'->|[,\[\]:()]', 'op', None, None),
		(r'[a-zA-Z_][a-zA-Z0-9_]*', 'name', None, None),
		(r'[0-9\-.]+', 'num', None, None),
		(r"'(.*?)'", 'str', None, None),
	],
	'indent': [
		(r'[ \t]', 'indent', None, None),
		(r'', 'end', None, None),
	],
}

REGEX = {}<commit_msg>Support the + operator in the tokenizer.<commit_after>import sys, re

def indent(s):
	yield 'nl', '\n'
	yield 'indent', len(s)

MATCHING = {
	None: [
		(r'\n', 'nl', 'indent', indent),
		(r' ', '!sp', None, None),
		(r'->|[,\[\]:()+]', 'op', None, None),
		(r'[a-zA-Z_][a-zA-Z0-9_]*', 'name', None, None),
		(r'[0-9\-.]+', 'num', None, None),
		(r"'(.*?)'", 'str', None, None),
	],
	'indent': [
		(r'[ \t]', 'indent', None, None),
		(r'', 'end', None, None),
	],
}

REGEX = {}
{'fraction_non_alphanumeric': 0.34523809523809523, 'fraction_numerical': 0.008658008658008658, 'mean_word_length': 3.534313725490196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1010839', 'n_tokens_mistral': 466, 'n_tokens_neox': 442, 'n_words': 101}