<commit_before>require.paths.push('lib');
var HTML5 = require('html5'),
	events = require('events'),
	sys = require('sys'),
	fs = require('fs'),
	assert = require('assert'),
	serialize = require('./tests/support/serializeTestOutput').serializeTestOutput;

if(process.argv[4]) {
	var debugs = process.argv[4].split(',');
	for(var d in debugs) HTML5.enableDebug(debugs[d]);
}

var base = 'testdata/tree-construction/'
var l = fs.readdirSync(base);
for(var t in l) {
	if(process.argv[2] && process.argv[2] != l[t]) continue;
	var testname = l[t];
	if(testname.match(/\.js$/)) continue;
	sys.puts("Test file: " + testname);
	var f = require('./tests/support/readTestData')
	var td = f.readTestData(base+testname);
	var tests = 1;
	for(var i in td) {
		try {
			if(process.argv[3] && process.argv[3] != i) continue;
			sys.puts("Test #" + i + ": ");
			sys.puts("Input data: " + sys.inspect(td[i].data.trimRight()));
			HTML5.debug('testdata.data', "Data: " + td[i].data.trimRight());
			var p = new HTML5.Parser(td[i].data.trimRight());
			var errorsFixed = p.errors.map(function(e) {
				if(!HTML5.E[e[0]]) return e;
				return HTML5.E[e[0]].replace(/%\(.*?\)/, function(r) {
					if(e[1]) {
						return e[1][r.slice(2).slice(0, r.length - 3)];
					} else {
						return r;
					}
				});
			});

		if(td[i].errors) {
			//HTML5.debug('testdata.errors', "Expected ", td[i].errors);
			//HTML5.debug('testdata.errors', "Actual ", errorsFixed);
			var serialized = serialize(p.tree.document);
			sys.puts("Output : " + sys.inspect(serialized));
			sys.puts("Check  : " + sys.inspect(td[i].document));
			assert.deepEqual(serialized, td[i].document);
			if(p.errors.length !== td[i].errors.length) {
				sys.puts("Expected errors: " + sys.inspect(td[i].errors));
				sys.puts("Actual errors  : " + sys.inspect(p.errors));
			}
		}
		} catch(e) {
                        sys.puts(e.stack + '\n');
		}
	}
}<commit_msg>- make testbed show raw trees<commit_after>require.paths.push('lib');
var HTML5 = require('html5'),
	events = require('events'),
	sys = require('sys'),
	fs = require('fs'),
	assert = require('assert'),
	serialize = require('./tests/support/serializeTestOutput').serializeTestOutput;

if(process.argv[4]) {
	var debugs = process.argv[4].split(',');
	for(var d in debugs) HTML5.enableDebug(debugs[d]);
}

var base = 'testdata/tree-construction/'
var l = fs.readdirSync(base);
for(var t in l) {
	if(process.argv[2] && process.argv[2] != l[t]) continue;
	var testname = l[t];
	if(testname.match(/\.js$/)) continue;
	sys.puts("Test file: " + testname);
	var f = require('./tests/support/readTestData')
	var td = f.readTestData(base+testname);
	var tests = 1;
	for(var i in td) {
		try {
			if(process.argv[3] && process.argv[3] != i) continue;
			sys.puts("Test #" + i + ": ");
			sys.puts("Input data: " + sys.inspect(td[i].data.trimRight()));
			HTML5.debug('testdata.data', "Data: " + td[i].data.trimRight());
			var p = new HTML5.Parser(td[i].data.trimRight());
			var errorsFixed = p.errors.map(function(e) {
				if(!HTML5.E[e[0]]) return e;
				return HTML5.E[e[0]].replace(/%\(.*?\)/, function(r) {
					if(e[1]) {
						return e[1][r.slice(2).slice(0, r.length - 3)];
					} else {
						return r;
					}
				});
			});

		if(td[i].errors) {
			//HTML5.debug('testdata.errors', "Expected ", td[i].errors);
			//HTML5.debug('testdata.errors', "Actual ", errorsFixed);
			var serialized = serialize(p.tree.document);
			sys.puts("Output : " + serialized);
			sys.puts("Check  : " + td[i].document);
			assert.deepEqual(serialized, td[i].document);
			if(p.errors.length !== td[i].errors.length) {
				sys.puts("Expected errors: " + sys.inspect(td[i].errors));
				sys.puts("Actual errors  : " + sys.inspect(p.errors));
			}
		}
		} catch(e) {
                        sys.puts(e.stack + '\n');
		}
	}
}
{'fraction_non_alphanumeric': 0.21735710540115363, 'fraction_numerical': 0.012060828526481384, 'mean_word_length': 3.4935217903415783, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4277411', 'n_tokens_mistral': 1600, 'n_tokens_neox': 1439, 'n_words': 243}