<commit_before>var Benchmark = require('benchmark');
var most = require('../../most');
var rx = require('rx');
var kefir = require('kefir');
var bacon = require('baconjs');
var lodash = require('lodash');
var highland = require('highland');

var runners = require('./runners');
var kefirFromArray = runners.kefirFromArray;

// flatMapping n streams, each containing m items.
// Results in a single stream that merges in n x m items
// In Array parlance: Take an Array containing n Arrays, each of length m,
// and flatten it to an Array of length n x m.
var n = 1000, m = 1000;
var a = build(m, n);

function build(m, n) {
	var a = new Array(n);
	for(var i = 0; i< a.length; ++i) {
		a[i] = buildArray(i*1000, m);
	}
	return a;
}

function buildArray(base, n) {
	var a = new Array(n);
	for(var i = 0; i< a.length; ++i) {
		a[i] = base + i;
	}
	return a;
}

var suite = Benchmark.Suite('concatMap ' + n + ' x ' + m + ' streams');
var options = {
	defer: true,
	onError: function(e) {
		e.currentTarget.failure = e.error;
	}
};

suite
	.add('most', function(deferred) {
		runners.runMost(deferred, most.from(a).concatMap(most.from).reduce(sum, 0));
	}, options)
	.add('rx', function(deferred) {
		runners.runRx(deferred, rx.Observable.fromArray(a).concatMap(rx.Observable.fromArray).reduce(sum, 0));
	}, options)
	.add('kefir', function(deferred) {
		runners.runKefir(deferred, kefirFromArray(a).flatMapConcat(kefirFromArray).reduce(sum, 0));
	}, options)
	.add('bacon', function(deferred) {
		runners.runBacon(deferred, bacon.fromArray(a).flatMapConcat(bacon.fromArray).reduce(0, sum));
	}, options)
	// Highland doesn't have concatMap
	//.add('highland', function(deferred) {
	//	runners.runHighland(deferred, highland(a).flatMap(highland).reduce(0, sum));
	//}, options)
	.add('lodash', function() {
		return lodashConcatMap(identity, a).reduce(sum, 0);
	})
	.add('Array', function() {
		return arrayConcatMap(identity, a).reduce(sum, 0);
	});

runners.runSuite(suite);

function arrayConcatMap(f, a) {
	return a.reduce(function(a, x) {
		return a.concat(f(x));
	}, []);
}

function lodashConcatMap(f, a) {
	return lodash(a).map(f).flatten(true);
}

function sum(x, y) {
	return x + y;
}

function identity(x) {
	return x;
}
<commit_msg>Make lodash flatten shallow. Thanks @jdalton<commit_after>var Benchmark = require('benchmark');
var most = require('../../most');
var rx = require('rx');
var kefir = require('kefir');
var bacon = require('baconjs');
var lodash = require('lodash');
var highland = require('highland');

var runners = require('./runners');
var kefirFromArray = runners.kefirFromArray;

// flatMapping n streams, each containing m items.
// Results in a single stream that merges in n x m items
// In Array parlance: Take an Array containing n Arrays, each of length m,
// and flatten it to an Array of length n x m.
var n = 1000, m = 1000;
var a = build(m, n);

function build(m, n) {
	var a = new Array(n);
	for(var i = 0; i< a.length; ++i) {
		a[i] = buildArray(i*1000, m);
	}
	return a;
}

function buildArray(base, n) {
	var a = new Array(n);
	for(var i = 0; i< a.length; ++i) {
		a[i] = base + i;
	}
	return a;
}

var suite = Benchmark.Suite('concatMap ' + n + ' x ' + m + ' streams');
var options = {
	defer: true,
	onError: function(e) {
		e.currentTarget.failure = e.error;
	}
};

suite
	.add('most', function(deferred) {
		runners.runMost(deferred, most.from(a).concatMap(most.from).reduce(sum, 0));
	}, options)
	.add('rx', function(deferred) {
		runners.runRx(deferred, rx.Observable.fromArray(a).concatMap(rx.Observable.fromArray).reduce(sum, 0));
	}, options)
	.add('kefir', function(deferred) {
		runners.runKefir(deferred, kefirFromArray(a).flatMapConcat(kefirFromArray).reduce(sum, 0));
	}, options)
	.add('bacon', function(deferred) {
		runners.runBacon(deferred, bacon.fromArray(a).flatMapConcat(bacon.fromArray).reduce(0, sum));
	}, options)
	// Highland doesn't have concatMap
	//.add('highland', function(deferred) {
	//	runners.runHighland(deferred, highland(a).flatMap(highland).reduce(0, sum));
	//}, options)
	.add('lodash', function() {
		return lodashConcatMap(identity, a).reduce(sum, 0);
	})
	.add('Array', function() {
		return arrayConcatMap(identity, a).reduce(sum, 0);
	});

runners.runSuite(suite);

function arrayConcatMap(f, a) {
	return a.reduce(function(a, x) {
		return a.concat(f(x));
	}, []);
}

function lodashConcatMap(f, a) {
	return lodash(a).map(f).flatten();
}

function sum(x, y) {
	return x + y;
}

function identity(x) {
	return x;
}

{'fraction_non_alphanumeric': 0.17943593160115479, 'fraction_numerical': 0.009327115256495669, 'mean_word_length': 4.4726609963548, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5864160', 'n_tokens_mistral': 1774, 'n_tokens_neox': 1699, 'n_words': 407}