<commit_before>import codegen, jinja2, spidermonkey, sys
import simplejson as json

TESTS = [
	('{{ test }}', {'test': 'crap'}),
	('{% if a %}x{% endif %}', {'a': True}),
	('{% if a %}c{% endif %}b', {'a': False}),
	('{{ 1 if a else 2 }}', {'a': True}),
	('{{ 1 if a else 2 }}', {'a': False}),
	('{% if a %}d{% else %}e{% endif %}', {'a': False}),
	('{% if a %}f{% elif b %}g{% endif %}', {'b': True}),
	("{{ '%4.2f'|format(x) }}", {'x': 17.0}),
	('{{ d[:7] }}', {'d': '2011-05-27'}),
	('{{ a.x }}', {'a': {'x': 'z'}}),
	('{{ "%.6f"|format(a / b) }}', {'a': 5.0, 'b': 3}),
	('{{ "%.1f"|format(a.x / b.y * 100) }}', {'a': {'x': 20}, 'b': {'y': 5}}),
	('{% macro x(y) %}{{ y / 2 }}{% endmacro %}{{ x(z) }}', {'z': 512}),
	('{{ x is none }}', {'x': None}),
]

def loader(i):
	return jinja2.DictLoader({'index': TESTS[i][0]})

def jstest(env, src, data):
	run = spidermonkey.Runtime()
	ctx = run.new_context()
	js = codegen.generate(env, ['index'])
	jsobj = json.dumps(data)
	code = js + '\ntemplates.index.render(%s);' % jsobj
	return ctx.execute(code)

def pytest(env, src, data):
	tmpl = env.from_string(src)
	return tmpl.render(data)

def run(i, quiet=True):
	
	src, data = TESTS[i]
	env = jinja2.Environment(loader=loader(i))
	ast = codegen.compile(env, src)
	
	if not quiet:
		print ast
		print codegen.generate(env, ['index'])
	
	js = jstest(env, src, data)
	py = pytest(env, src, data)
	
	if not quiet:
		print 'js:', repr(js)
		print 'py:', repr(py)
	
	if js.isdigit():
		return float(js) == float(py)
	return js == py

def test():
	for i, t in enumerate(TESTS):
		res = run(i)
		sys.stdout.write('.' if res else 'F')
	sys.stdout.write('\n')

if __name__ == '__main__':
	args = sys.argv[1:]
	if args:
		run(int(args[0]), False)
	else:
		test()
<commit_msg>Fix js/py comparison for booleans.<commit_after>import codegen, jinja2, spidermonkey, sys
import simplejson as json

TESTS = [
	('{{ test }}', {'test': 'crap'}),
	('{% if a %}x{% endif %}', {'a': True}),
	('{% if a %}c{% endif %}b', {'a': False}),
	('{{ 1 if a else 2 }}', {'a': True}),
	('{{ 1 if a else 2 }}', {'a': False}),
	('{% if a %}d{% else %}e{% endif %}', {'a': False}),
	('{% if a %}f{% elif b %}g{% endif %}', {'b': True}),
	("{{ '%4.2f'|format(x) }}", {'x': 17.0}),
	('{{ d[:7] }}', {'d': '2011-05-27'}),
	('{{ a.x }}', {'a': {'x': 'z'}}),
	('{{ "%.6f"|format(a / b) }}', {'a': 5.0, 'b': 3}),
	('{{ "%.1f"|format(a.x / b.y * 100) }}', {'a': {'x': 20}, 'b': {'y': 5}}),
	('{% macro x(y) %}{{ y / 2 }}{% endmacro %}{{ x(z) }}', {'z': 512}),
	('{{ x is none }}', {'x': None}),
]

def loader(i):
	return jinja2.DictLoader({'index': TESTS[i][0]})

def jstest(env, src, data):
	run = spidermonkey.Runtime()
	ctx = run.new_context()
	js = codegen.generate(env, ['index'])
	jsobj = json.dumps(data)
	code = js + '\ntemplates.index.render(%s);' % jsobj
	return ctx.execute(code)

def pytest(env, src, data):
	tmpl = env.from_string(src)
	return tmpl.render(data)

def run(i, quiet=True):
	
	src, data = TESTS[i]
	env = jinja2.Environment(loader=loader(i))
	ast = codegen.compile(env, src)
	
	if not quiet:
		print ast
		print codegen.generate(env, ['index'])
	
	js = jstest(env, src, data)
	py = pytest(env, src, data)
	
	if not quiet:
		print 'js:', repr(js)
		print 'py:', repr(py)
	
	if js.isdigit():
		return float(js) == float(py)
	if {'true': 'True', 'false': 'False'}.get(js, js) == py:
		return True
	return js == py

def test():
	for i, t in enumerate(TESTS):
		res = run(i)
		sys.stdout.write('.' if res else 'F')
	sys.stdout.write('\n')

if __name__ == '__main__':
	args = sys.argv[1:]
	if args:
		run(int(args[0]), False)
	else:
		test()

{'fraction_non_alphanumeric': 0.28437067773167357, 'fraction_numerical': 0.021576763485477178, 'mean_word_length': 3.8930987821380243, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6353340', 'n_tokens_mistral': 1689, 'n_tokens_neox': 1551, 'n_words': 407}