<commit_before>'use strict';

var path = require('path');
var util = require('util');
var _ = require('lodash');
var sourceMapModule = require('source-map');
var SourceMapGenerator = sourceMapModule.SourceMapGenerator;
var SourceMapConsumer = sourceMapModule.SourceMapConsumer;
var jsonParse = require('../json-parse');
var Processor = require('../processor');
var Batch = require('../batch');
var Graph = require('../graph');

var processor = new Processor();

// Note: These regexps only work with invidually minified modules concatenated with newline separators
function parseModule(line, lineMappings) {
	var matches = line.match(
		/(em|playtotv)\.define\("([\w\.]+)",\[([\w\.",]*)\],function\(([\w,]*)\)\{([\w\W]*)\}\);$/
	);

	if (!matches) {
		return null;
	}

	var deps = matches[3];
	deps = deps.length > 0 ? deps.substr(1, deps.length - 2).split(/","/) : [];
	var params = matches[4];
	params = params.length > 0 ? params.split(/,/) : [];

	return {
		id: matches[2],
		deps: deps,
		params: params,
		body: matches[5],
		line: line,
		lineMappings: lineMappings
	};
}

function getUndefinedDeps(modules) {
	return modules.reduce(function(undefDeps, module) {
		return undefDeps.concat(module.deps.filter(function(dep) {
			return !_.find(modules, { id: dep });
		}).map(function(dep) {
			return { id: module.id, dep: dep };
		}));
	}, []);
}

function getDependencyErrors(modules) {
	return modules.reduce(function(depErrors, module) {
		var deps = module.deps;
		var params = module.params;
		if (deps.length > 0 && params.length === 0) {
			deps.forEach(function(dep) {
				// Assuming: var c = playtotv.importModules(this.dependencies);
				params.push('[a-z].' + dep.substr(dep.lastIndexOf('.') + 1));
			});
		} else if (params.length !== deps.length) {
			depErrors.push({
				id: module.id,
				error: util.format('Nr of params: %d <> Nr of deps: %d', params.length, deps.length)
			});
		}
		params.forEach(function(param, i) {
			// TODO: Improve this check, it matches too much
			if (!module.body.match(new RegExp('\\W' + param + '\\W'))) {
				depErrors.push({
					id: module.id,
					error: util.format('Unused: %s (%s)', deps[i], param)
				});
			}
		});
		return depErrors;
	}, []);
}

function pruneUnusedModules(exports, modules) {
	if (exports.length === 0) {
		return modules;
	}

	exports.forEach(function markUsed(id) {
		var module = _.find(modules, { id: id });
		if (!module) {
			throw new Error('Export module undefined: ' + id);
		}
		if (!module.used) {
			module.used = true;
			module.deps.forEach(markUsed);
		}
	});

	return modules.filter(function(module) {<commit_msg>Made EM AMD processor regexp's more flexible so they also work when mangling results in argument with character: "$"<commit_after>'use strict';

var path = require('path');
var util = require('util');
var _ = require('lodash');
var sourceMapModule = require('source-map');
var SourceMapGenerator = sourceMapModule.SourceMapGenerator;
var SourceMapConsumer = sourceMapModule.SourceMapConsumer;
var jsonParse = require('../json-parse');
var Processor = require('../processor');
var Batch = require('../batch');
var Graph = require('../graph');

var processor = new Processor();

// Note: These regexps only work with invidually minified modules concatenated with newline separators
function parseModule(line, lineMappings) {
	var matches = line.match(
		/(em|playtotv)\.define\("([^"]+)",\[([^\]]*)\],function\(([^\)]*)\)\{([\w\W]*)\}\);$/
	);

	if (!matches) {
		return null;
	}

	var deps = matches[3];
	deps = deps.length > 0 ? deps.substr(1, deps.length - 2).split(/","/) : [];
	var params = matches[4];
	params = params.length > 0 ? params.split(/,/) : [];

	return {
		id: matches[2],
		deps: deps,
		params: params,
		body: matches[5],
		line: line,
		lineMappings: lineMappings
	};
}

function getUndefinedDeps(modules) {
	return modules.reduce(function(undefDeps, module) {
		return undefDeps.concat(module.deps.filter(function(dep) {
			return !_.find(modules, { id: dep });
		}).map(function(dep) {
			return { id: module.id, dep: dep };
		}));
	}, []);
}

function getDependencyErrors(modules) {
	return modules.reduce(function(depErrors, module) {
		var deps = module.deps;
		var params = module.params;
		if (deps.length > 0 && params.length === 0) {
			deps.forEach(function(dep) {
				// Assuming: var c = playtotv.importModules(this.dependencies);
				params.push('[a-z].' + dep.substr(dep.lastIndexOf('.') + 1));
			});
		} else if (params.length !== deps.length) {
			depErrors.push({
				id: module.id,
				error: util.format('Nr of params: %d <> Nr of deps: %d', params.length, deps.length)
			});
		}
		params.forEach(function(param, i) {
			// TODO: Improve this check, it matches too much
			if (!module.body.match(new RegExp('\\W' + _.escapeRegExp(param) + '\\W'))) {
				depErrors.push({
					id: module.id,
					error: util.format('Unused: %s (%s)', deps[i], param)
				});
			}
		});
		return depErrors;
	}, []);
}

function pruneUnusedModules(exports, modules) {
	if (exports.length === 0) {
		return modules;
	}

	exports.forEach(function markUsed(id) {
		var module = _.find(modules, { id: id });
		if (!module) {
			throw new Error('Export module undefined: ' + id);
		}
		if (!module.used) {
			module.used = true;
			module.deps.forEach(markUsed);
		}
	});

	return modules.filter(function(module) {
{'fraction_non_alphanumeric': 0.1869857890800299, 'fraction_numerical': 0.004487658937920718, 'mean_word_length': 3.8232642019837693, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4496587', 'n_tokens_mistral': 2019, 'n_tokens_neox': 1827, 'n_words': 424}