<commit_before>(function () {
	'use strict'

	const SAMPLE_TEXT = [
		'(fun double (n)',
		' (* n 2))',
	].join('\n')


	// setting up the editors
	const inputEditor = ace.edit('input-editor')
	inputEditor.setTheme('ace/theme/monokai')
	inputEditor.setFontSize(18)
	inputEditor.setValue(SAMPLE_TEXT, 1)
	inputEditor.on('input', onInput)

	const outputEditor = ace.edit('output-editor')
	outputEditor.setTheme('ace/theme/monokai')
	outputEditor.getSession().setMode('ace/mode/javascript')
	outputEditor.getSession().setUseWrapMode(true)
	outputEditor.setReadOnly(true)
	outputEditor.setFontSize(18)


	// getting a tokenizer
	const tokenizer = espace.Tokenizer({ coords: true })

	function onInput () {
		const inputText = inputEditor.getValue()
		try {
			const tokens = tokenizer(inputText)<commit_msg>Update the tree example to allow prefixes<commit_after>(function () {
	'use strict'

	const SAMPLE_TEXT = [
		'(fun double (n)',
		' (* n 2))',
	].join('\n')


	// setting up the editors
	const inputEditor = ace.edit('input-editor')
	inputEditor.setTheme('ace/theme/monokai')
	inputEditor.setFontSize(18)
	inputEditor.setValue(SAMPLE_TEXT, 1)
	inputEditor.on('input', onInput)

	const outputEditor = ace.edit('output-editor')
	outputEditor.setTheme('ace/theme/monokai')
	outputEditor.getSession().setMode('ace/mode/javascript')
	outputEditor.getSession().setUseWrapMode(true)
	outputEditor.setReadOnly(true)
	outputEditor.setFontSize(18)


	// getting a tokenizer
	const tokenizer = espace.Tokenizer({
		coords: true,
		prefixes: {
			'@': 'set',
			'#': 'map',
		},
	})

	function onInput () {
		const inputText = inputEditor.getValue()
		try {
			const tokens = tokenizer(inputText)
{'fraction_non_alphanumeric': 0.14878482513337285, 'fraction_numerical': 0.007113218731475993, 'mean_word_length': 4.881533101045296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3788084', 'n_tokens_mistral': 609, 'n_tokens_neox': 568, 'n_words': 106}