<commit_before>var assert = require('assert');
var fs = require('fs');
var trace = require('line-trace');

var files = [
	__dirname + '/kv.data',
	__dirname + '/kv2.data'
];

var raw_data = [
	fs.readFileSync(files[0], {encoding: 'utf8'}),
	fs.readFileSync(files[1], {encoding: 'utf8'})
];

function textParser(s) {return s.split(' ').map(parseFloat);}

var v = [
	raw_data[0].split('\n').filter(function (l) {return l != '';}).map(textParser),
	raw_data[1].split('\n').filter(function (l) {return l != '';}).map(textParser)
];

// Helper functions for tests
function compareResults(r1, r2, opt) {
	opt = opt || {};
	if (opt.lengthOnly) {
		assert.equal(r1.length, r2.length);
		return;
	}
	if (Array.isArray(r1)) sort(r1);
	if (Array.isArray(r2)) sort(r2);
	//assert.deepEqual(r1, r2);
	assert.equal(JSON.stringify(r1), JSON.stringify(r2));
}

function filter(e) {return e[1] % 2 === 0;}

function flatMapper(e) {
	return [e, e];
}

function mapper(e) {
	// copy input
	var cpy = JSON.parse(JSON.stringify(e));
	cpy[1] *= 2; return cpy;
}

function reducer(a, b) {
	if (Array.isArray(b[0]))
		a[0] += b[0].reduce(sum);
	else if (Array.isArray(b)) {
		a[0] += b[0];
	} else {<commit_msg>ugrid-test: fix compare function where actual and expected results were inverted<commit_after>var assert = require('assert');
var fs = require('fs');
var trace = require('line-trace');

var files = [
	__dirname + '/kv.data',
	__dirname + '/kv2.data'
];

var raw_data = [
	fs.readFileSync(files[0], {encoding: 'utf8'}),
	fs.readFileSync(files[1], {encoding: 'utf8'})
];

function textParser(s) {return s.split(' ').map(parseFloat);}

var v = [
	raw_data[0].split('\n').filter(function (l) {return l != '';}).map(textParser),
	raw_data[1].split('\n').filter(function (l) {return l != '';}).map(textParser)
];

// Helper functions for tests
function compareResults(r1, r2, opt) {
	opt = opt || {};
	if (opt.lengthOnly) {
		assert.equal(r2.length, r1.length);
		return;
	}
	if (Array.isArray(r1)) sort(r1);
	if (Array.isArray(r2)) sort(r2);
	//assert.deepEqual(r1, r2);
	assert.equal(JSON.stringify(r2), JSON.stringify(r1));
}

function filter(e) {return e[1] % 2 === 0;}

function flatMapper(e) {
	return [e, e];
}

function mapper(e) {
	// copy input
	var cpy = JSON.parse(JSON.stringify(e));
	cpy[1] *= 2; return cpy;
}

function reducer(a, b) {
	if (Array.isArray(b[0]))
		a[0] += b[0].reduce(sum);
	else if (Array.isArray(b)) {
		a[0] += b[0];
	} else {
{'fraction_non_alphanumeric': 0.21712412945514134, 'fraction_numerical': 0.02376075378943056, 'mean_word_length': 4.575342465753424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7619406', 'n_tokens_mistral': 988, 'n_tokens_neox': 984, 'n_words': 210}