<commit_before>var path = require('path');
var url = require('url');
var resolve = require('browser-resolve');
var through = require('through2');
var falafel = require('falafel');
var _ = require('lodash');
var removeTrailingSlash = require('remove-trailing-slash');
var defaultConfig = require('./lib/config');
var meta = require('./lib/meta');
var Bundle = require('./lib/bundle');
var config;
var instanceCache = [];

function shouldParseFile ( file ) {
	return _.contains(config.extensions, path.extname(resolve.sync(file, { filename: file })));
}

function isRequireAsync ( node ) {

	var c = node.callee;

	return c &&
	node.type === 'CallExpression' &&
	c.type === 'MemberExpression' &&
	(c.object && c.object.type === 'Identifier' && c.object.name === 'require') &&
	(c.property && c.property.type === 'Identifier' && c.property.name === 'async');

}

function getDepsChain ( deps, file ) {
	return _.chain(deps)
		.map(function ( dep ) {
			try {
				return {
					file: resolve.sync(dep, { filename: file }),
					expose: dep
				};
			} catch ( e ) {
				throw new Error(e);
			}
		})
		.value();
}

function getNewBundles ( depsChain ) {
	return _.chain(depsChain)
		.reject(function ( opts ) {
			return _.contains(_.pluck(instanceCache, 'input'), opts.file);
		})
		.map(function ( opts ) {
			var b = new Bundle(_.extend({}, opts, {
				config: config
			}));
			instanceCache.push(b);
			return b;
		})
		.value();
}

function getAllBundles ( depsChain ) {
	return _.chain(depsChain)
		.map(function ( opts ) {
			return _.findWhere(instanceCache, { input: opts.file });
		})
		.value();
}

function transform ( file, opts ) {

	var rootUrl;
	config = _.extend({}, defaultConfig, opts);

	rootUrl = removeTrailingSlash(config.url) + '/';

	return through(function ( buf, enc, next ) {

		var content = buf.toString('utf8');
		var transformedContent;

		if ( !shouldParseFile(file) ) {
			this.push(content);
			next();
			return;
		}

		try {
			transformedContent = falafel(content, function ( node ) {

				var arg, deps, depsChain, newBundles, allBundles;

				if ( isRequireAsync(node) ) {

					node.callee.update('require(' + JSON.stringify(meta.name + '/loader') + ')(require, ' + JSON.stringify(rootUrl) + ')');
					arg = node.arguments[0];

					if ( arg.type === 'ArrayExpression' ) {
						deps = _.pluck(arg.elements, 'value');
					} else {
						deps = [arg.value];
					}

					depsChain = getDepsChain(deps, file);
					newBundles = getNewBundles(depsChain);
					allBundles = getAllBundles(depsChain);

					_.invoke(newBundles, 'create');
					_.invoke(newBundles, 'write');

					arg.update(JSON.stringify(_.invoke(allBundles, 'getConfig')));

				}

			});

			this.push(transformedContent.toString());
			next();
		} catch ( err ) {
			next(err, content);
		}

	});

}

module.exports = transform;<commit_msg>Create bundles on stream flush<commit_after>var path = require('path');
var url = require('url');
var resolve = require('browser-resolve');
var through = require('through2');
var falafel = require('falafel');
var _ = require('lodash');
var removeTrailingSlash = require('remove-trailing-slash');
var defaultConfig = require('./lib/config');
var meta = require('./lib/meta');
var Bundle = require('./lib/bundle');
var config;
var instanceCache = [];

function shouldParseFile ( file ) {
	return _.contains(config.extensions, path.extname(resolve.sync(file, { filename: file })));
}

function isRequireAsync ( node ) {

	var c = node.callee;

	return c &&
	node.type === 'CallExpression' &&
	c.type === 'MemberExpression' &&
	(c.object && c.object.type === 'Identifier' && c.object.name === 'require') &&
	(c.property && c.property.type === 'Identifier' && c.property.name === 'async');

}

function getDepsChain ( deps, file ) {
	return _.chain(deps)
		.map(function ( dep ) {
			try {
				return {
					file: resolve.sync(dep, { filename: file }),
					expose: dep
				};
			} catch ( e ) {
				throw new Error(e);
			}
		})
		.value();
}

function getNewBundles ( depsChain ) {
	return _.chain(depsChain)
		.reject(function ( opts ) {
			return _.contains(_.pluck(instanceCache, 'input'), opts.file);
		})
		.map(function ( opts ) {
			var b = new Bundle(_.extend({}, opts, {
				config: config
			}));
			instanceCache.push(b);
			return b;
		})
		.value();
}

function getAllBundles ( depsChain ) {
	return _.chain(depsChain)
		.map(function ( opts ) {
			return _.findWhere(instanceCache, { input: opts.file });
		})
		.value();
}

function transform ( file, opts ) {

	var rootUrl;
	var newBundles = [];
	config = _.extend({}, defaultConfig, opts);

	rootUrl = removeTrailingSlash(config.url) + '/';

	return through(function ( buf, enc, next ) {

		var content = buf.toString('utf8');
		var transformedContent;

		if ( !shouldParseFile(file) ) {
			this.push(content);
			next();
			return;
		}

		try {
			transformedContent = falafel(content, function ( node ) {

				var arg, deps, depsChain, allBundles;

				if ( isRequireAsync(node) ) {

					node.callee.update('require(' + JSON.stringify(meta.name + '/loader') + ')(require, ' + JSON.stringify(rootUrl) + ')');
					arg = node.arguments[0];

					if ( arg.type === 'ArrayExpression' ) {
						deps = _.pluck(arg.elements, 'value');
					} else {
						deps = [arg.value];
					}

					depsChain = getDepsChain(deps, file);
					newBundles = newBundles.concat(getNewBundles(depsChain));
					allBundles = getAllBundles(depsChain);

					arg.update(JSON.stringify(_.invoke(allBundles, 'getConfig')));

				}

			});

			this.push(transformedContent.toString());
			next();
		} catch ( err ) {
			next(err, content);
		}

	}, function ( cb ) {
		_.invoke(newBundles, 'create');
		_.invoke(newBundles, 'write');
		cb();
	});

}

module.exports = transform;
{'fraction_non_alphanumeric': 0.17421481867083116, 'fraction_numerical': 0.001041124414367517, 'mean_word_length': 3.4786324786324787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2778612', 'n_tokens_mistral': 2241, 'n_tokens_neox': 1926, 'n_words': 403}