<commit_before>var testCase = require('nodeunit').testCase,
	parser = require('../language/parser'),
	tokenizer = require('../language/tokenizer')

test('TextLiteral', '"hello world"', static("hello world"))
test('NumberLiteral', '1', static(1))
test('Declaration', 'let greeting = "hello"', {
	type:'DECLARATION',
	name:'greeting',
	value: static("hello")
})

/* UTIL */

function test(name, code, expectedAST) {
	module.exports['test '+name] = function(assert) {
		var ast = parse(code)
		assert.deepEqual(ast, expectedAST)
		assert.done()
	}
}

function static(value) {
	var ast = { type:'STATIC', value:value }
	ast.valueType = typeof value
	return ast
}

function parse(code) {
	tokens = tokenizer.tokenize(code)
	return pruneInfo(parser.parse(tokens))
}

function pruneInfo(ast) {
	for (var key in ast) {
		if (key == 'info') { delete ast[key] }
		else if (typeof ast[key] == 'object') { pruneInfo(ast[key]) }
	}
	return ast
}
<commit_msg>Make the tests more reader friendly<commit_after>var testCase = require('nodeunit').testCase,
	parser = require('../language/parser'),
	tokenizer = require('../language/tokenizer')

/* TESTS */
test('text literal', '"hello world"', static("hello world"))
test('number literal', '1', static(1))
test('declaration', 'let greeting = "hello"', { type:'DECLARATION', name:'greeting', value: static("hello") })

/* UTIL */

function test(name, code, expectedAST) {
	module.exports['test '+name] = function(assert) {
		var ast = parse(code)
		assert.deepEqual(ast, expectedAST)
		assert.done()
	}
}

function static(value) {
	var ast = { type:'STATIC', value:value }
	ast.valueType = typeof value
	return ast
}

function parse(code) {
	tokens = tokenizer.tokenize(code)
	return pruneInfo(parser.parse(tokens))
}

function pruneInfo(ast) {
	for (var key in ast) {
		if (key == 'info') { delete ast[key] }
		else if (typeof ast[key] == 'object') { pruneInfo(ast[key]) }
	}
	return ast
}

{'fraction_non_alphanumeric': 0.17784711388455537, 'fraction_numerical': 0.0020800832033281333, 'mean_word_length': 5.0125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3046260', 'n_tokens_mistral': 667, 'n_tokens_neox': 650, 'n_words': 156}