<commit_before>var tape = require('tape')
var parser = require('./')
var fs = require('fs')


tape('process the test file', function(t){
	var content = fs.readFileSync(__dirname + '/testpage.md', 'utf8')	

	parser(content, function(err, result){
		if(err){
			t.fail(err, 'process result')
			t.end()
			return
		}

		t.equal(result.attributes.apples, 'hello', 'apples')
		t.deepEqual(result.attributes.list, [5, 10], 'list')
		t.deepEqual(result.attributes.map, {apple:1,orange:2}, 'map')

		t.end()
	})


	
})<commit_msg>full test<commit_after>var tape = require('tape')
var parser = require('./')
var fs = require('fs')


tape('process the test file', function(t){
	var content = fs.readFileSync(__dirname + '/testpage.md', 'utf8')	

	parser(content, function(err, result){
		if(err){
			t.fail(err, 'process result')
			t.end()
			return
		}

		t.equal(result.attributes.apples, 'hello', 'apples')
		t.deepEqual(result.attributes.list, [5, 10], 'list')
		t.deepEqual(result.attributes.map, {apple:1,orange:2}, 'map')

		t.equal(result.body, '\nhello world\n\n```\nsome code\n```\n\n * list 1\n * list 2', 'body')
		t.equal(result.html, '<p>hello world</p>\n<pre><code>some code\n</code></pre><ul>\n<li>list 1</li>\n<li>list 2</li>\n</ul>\n', 'html')

		t.end()
	})


	
})
{'fraction_non_alphanumeric': 0.2164705882352941, 'fraction_numerical': 0.012549019607843137, 'mean_word_length': 4.251028806584362, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '428160', 'n_tokens_mistral': 537, 'n_tokens_neox': 485, 'n_words': 95}