<commit_before>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
	Author <NAME> @sokra
*/
var RawSource = require("webpack/lib/RawSource");
var Template = require("webpack/lib/Template");
var async = require("async");

var nextId = 0;

function ExtractTextPlugin(id, filename, options) {
	if(typeof filename !== "string") {
		options = filename;
		filename = id;
		id = ++nextId;
	}
	if(!options) options = {};
	this.filename = filename;
	this.options = options;
	this.id = id;
}
module.exports = ExtractTextPlugin;

ExtractTextPlugin.loader = function(options) {
	return require.resolve("./loader") + "?" + JSON.stringify(options);
};

ExtractTextPlugin.prototype.loader = function(options) {
	options = JSON.parse(JSON.stringify(options || {}));
	options.id = this.id;
	return ExtractTextPlugin.loader(options);
};

ExtractTextPlugin.prototype.apply = function(compiler) {
	var options = this.options;
	compiler.plugin("compilation", function(compilation) {
		compilation.plugin("normal-module-loader", function(loaderContext, module) {
			loaderContext[__dirname] = function(text, opt) {
				module.meta[__dirname] = {
					text: text,
					options: opt
				};
				return options.allChunks || module.meta[__dirname + "/extract"];
			};
		}.bind(this));
		var filename = this.filename;
		var id = this.id;
		compilation.plugin("optimize-tree", function(chunks, modules, callback) {
			var texts = {};
			async.forEach(chunks, function(chunk, callback) {
				var shouldExtract = !!(options.allChunks || chunk.initial);
				var text = [];
				async.forEach(chunk.modules, function(module, callback) {
					var meta = module.meta[__dirname];
					if(meta) {
						var wasExtracted = typeof meta.text === "string";
						if(shouldExtract !== wasExtracted) {
							module.meta[__dirname + "/extract"] = shouldExtract
							compilation.buildModule(module, function(err) {
								if(err) return callback(err);
								meta = module.meta[__dirname];
								if(typeof meta.text !== "string") return callback(new Error(module.identifier() + " doesn't export text"));
								text.push(meta.text);
								callback();
							});
						} else {
							text.push(meta.text);
							callback();
						}
					} else callback();
				}, function(err) {
					if(err) return callback(err);
					if(text.length > 0) {
						var file = filename.replace(Template.REGEXP_NAME, chunk.name);
						texts[file] = (texts[file] || []).concat(text);
					}
					callback();
				}.bind(this));
			}.bind(this), function(err) {
				if(err) return callback(err);<commit_msg>Ignore modules which we didn't see during loading<commit_after>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
	Author <NAME> @sokra
*/
var RawSource = require("webpack/lib/RawSource");
var Template = require("webpack/lib/Template");
var async = require("async");

var nextId = 0;

function ExtractTextPlugin(id, filename, options) {
	if(typeof filename !== "string") {
		options = filename;
		filename = id;
		id = ++nextId;
	}
	if(!options) options = {};
	this.filename = filename;
	this.options = options;
	this.id = id;
}
module.exports = ExtractTextPlugin;

ExtractTextPlugin.loader = function(options) {
	return require.resolve("./loader") + "?" + JSON.stringify(options);
};

ExtractTextPlugin.prototype.loader = function(options) {
	options = JSON.parse(JSON.stringify(options || {}));
	options.id = this.id;
	return ExtractTextPlugin.loader(options);
};

ExtractTextPlugin.prototype.apply = function(compiler) {
	var options = this.options;
	compiler.plugin("compilation", function(compilation) {
		compilation.plugin("normal-module-loader", function(loaderContext, module) {
			loaderContext[__dirname] = function(text, opt) {
				module.meta[__dirname] = {
					text: text,
					options: opt
				};
				return options.allChunks || module.meta[__dirname + "/extract"];
			};
		}.bind(this));
		var filename = this.filename;
		var id = this.id;
		compilation.plugin("optimize-tree", function(chunks, modules, callback) {
			var texts = {};
			async.forEach(chunks, function(chunk, callback) {
				var shouldExtract = !!(options.allChunks || chunk.initial);
				var text = [];
				async.forEach(chunk.modules, function(module, callback) {
					var meta = module.meta && module.meta[__dirname];
					if(meta) {
						var wasExtracted = typeof meta.text === "string";
						if(shouldExtract !== wasExtracted) {
							module.meta[__dirname + "/extract"] = shouldExtract
							compilation.buildModule(module, function(err) {
								if(err) return callback(err);
								meta = module.meta[__dirname];
								if(typeof meta.text !== "string") return callback(new Error(module.identifier() + " doesn't export text"));
								text.push(meta.text);
								callback();
							});
						} else {
							text.push(meta.text);
							callback();
						}
					} else callback();
				}, function(err) {
					if(err) return callback(err);
					if(text.length > 0) {
						var file = filename.replace(Template.REGEXP_NAME, chunk.name);
						texts[file] = (texts[file] || []).concat(text);
					}
					callback();
				}.bind(this));
			}.bind(this), function(err) {
				if(err) return callback(err);
{'fraction_non_alphanumeric': 0.1563404748118124, 'fraction_numerical': 0.0007720517274657402, 'mean_word_length': 3.639212175470009, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '429317', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1580, 'n_words': 325}