<commit_before>#! /usr/bin/env node
// -*- js2 -*-

global.sys = require(/^v0\.[012]/.test(process.version) ? "sys" : "util");
var fs = require("fs");
var uglify = require("uglify-js"), // symlink ~/.node_libraries/uglify-js.js to ../uglify-js.js
    jsp = uglify.parser,
    pro = uglify.uglify;

pro.set_logger(function(msg){
	sys.debug(msg);
});

var options = {
	ast: false,
	mangle: true,
	mangle_toplevel: false,
	squeeze: true,
	make_seqs: true,
	dead_code: true,
	beautify: false,
	verbose: false,
	show_copyright: true,
	out_same_file: false,
	max_line_length: 32 * 1024,
	unsafe: false,
	beautify_options: {
		indent_level: 4,
		indent_start: 0,
		quote_keys: false,
		space_colon: false
	},
	output: true	    // stdout
};

var args = jsp.slice(process.argv, 2);
var filenames = [];
var main;
var paths = [];
var combine;
var commonjs = false;
var forceCommonJS = false;
var time = new Date().getTime();

while (args.length > 0) {
	var v = args.shift();
	switch (v) {
	    case "-b":
	    case "--beautify":
		options.beautify = true;
		break;
	    case "-i":
	    case "--indent":
		options.beautify_options.indent_level = args.shift();
		break;
	    case "-q":
	    case "--quote-keys":
		options.beautify_options.quote_keys = true;
		break;
	    case "-mt":
	    case "--mangle-toplevel":
		options.mangle_toplevel = true;
		break;
	    case "--no-mangle":
	    case "-nm":
		options.mangle = false;
		break;
	    case "--no-squeeze":
	    case "-ns":
		options.squeeze = false;
		break;
	    case "--no-seqs":
		options.make_seqs = false;
		break;
	    case "--no-dead-code":
		options.dead_code = false;
		break;
	    case "--no-copyright":
	    case "-nc":
		options.show_copyright = false;
		break;
	    case "-o":
	    case "--ouput":
	    case "-cp":
	    case "--compile":
		options.output = args.shift();
		break;
	    case "--overwrite":
		options.out_same_file = true;
		break;
	    case "-v":
	    case "--verbose":
		options.verbose = true;
		break;
	    case "--ast":
		options.ast = true;
		break;
	    case "--unsafe":
		options.unsafe = true;
		break;
	    case "--max-line-len":
		options.max_line_length = parseInt(args.shift(), 10);
		break;
		case "-m":
		case "--main":
		main = args.shift();
		commonjs = true;
		break;
		case "-p":
		case "--path":
		paths.push(args.shift().replace(/\/?$/, '/'));
		break;
		case "-cb":
		case "--combine":
		combine = args.shift();
		break;
		case "--commonjs":
		case "-cm":
		commonjs = true;
		break;
		case "--force-commonjs":
		case "-fcm":
		forceCommonJS = true;
		break;
	    default:
		filenames.push(v);
		break;
	}
}

/////// CUSTOM COMBINER

RegExp.escape = function(text) {
	return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
};

var modules = {};
var combined = '';
var loading = 0;
var files = [];
//var rootRegex = root ? new RegExp('^' + RegExp.escape(root)) : undefined;
var requireRegex = /\brequire\(("|')(.+?)\1\)/g;
var reqOrExpRegex = /\brequire\(("|')(.+?)\1\)|\bexports.+=/;


function finish(text) {
	if (combine) {
		console.log('loaded...', '(' + (new Date().getTime() - time) + 'ms)');
		time = new Date().getTime();
		
		fs.writeFile(combine, text, function (err) {
			if (err) throw err;
			if (options.output !== true) {
				console.log('combined...', '(' + (new Date().getTime() - time) + 'ms)');
				time = new Date().getTime();
				output(squeeze_it(text));
			} else {
				console.log('combined!', '(' + (new Date().getTime() - time) + 'ms)')
			}
		});
	} else {
		output(squeeze_it(text));
	}
}


function loadModule(moduleId, currentModuleId, pathIndex) {
	moduleId = resolveModuleId(moduleId, currentModuleId);
	if (modules.hasOwnProperty(moduleId)) return;
	
	pathIndex = pathIndex || 0;
	var file = paths[pathIndex] + moduleId + '.js';
	++loading;
	
	fs.readFile(file, "utf8", function(err, code){
		if (err) {
			if (++pathIndex == paths.length) throw new Error('No module "' + moduleId + '" exists in paths [' + paths.join(', ') + ']');
			--loading;
			loadModule(moduleId, currentModuleId, pathIndex);
			return;
		}
		
		modules[moduleId] = true;
		findRequired(code, moduleId);
		combined += 'require.addModuleClosure("' + moduleId + '", function(require, exports, module) {\n' + code + '\n});\n\n';
		
		// if finished loading all dependencies finish combine and minimize it
		if (!--loading) {
			console.log('done:', main, !!modules[main]);
			if (main && !modules[main]) {
				loadMain();
			} else {
				if (main) combined += 'require.run("' + main + '");';
				finish(combined);
			}
		}
	});
}

function loadFile(file) {
	// don't work on directories
	if (!/\.js\^?$/.test(file)) {
		return;
	}
	
	var order = loading++;<commit_msg>Fixed compile to not double-add files<commit_after>#! /usr/bin/env node
// -*- js2 -*-

global.sys = require(/^v0\.[012]/.test(process.version) ? "sys" : "util");
var fs = require("fs");
var uglify = require("uglify-js"), // symlink ~/.node_libraries/uglify-js.js to ../uglify-js.js
    jsp = uglify.parser,
    pro = uglify.uglify;

pro.set_logger(function(msg){
	sys.debug(msg);
});

var options = {
	ast: false,
	mangle: true,
	mangle_toplevel: false,
	squeeze: true,
	make_seqs: true,
	dead_code: true,
	beautify: false,
	verbose: false,
	show_copyright: true,
	out_same_file: false,
	max_line_length: 32 * 1024,
	unsafe: false,
	beautify_options: {
		indent_level: 4,
		indent_start: 0,
		quote_keys: false,
		space_colon: false
	},
	output: true	    // stdout
};

var args = jsp.slice(process.argv, 2);
var filenames = [];
var main;
var paths = [];
var combine;
var commonjs = false;
var forceCommonJS = false;
var time = new Date().getTime();

while (args.length > 0) {
	var v = args.shift();
	switch (v) {
	    case "-b":
	    case "--beautify":
		options.beautify = true;
		break;
	    case "-i":
	    case "--indent":
		options.beautify_options.indent_level = args.shift();
		break;
	    case "-q":
	    case "--quote-keys":
		options.beautify_options.quote_keys = true;
		break;
	    case "-mt":
	    case "--mangle-toplevel":
		options.mangle_toplevel = true;
		break;
	    case "--no-mangle":
	    case "-nm":
		options.mangle = false;
		break;
	    case "--no-squeeze":
	    case "-ns":
		options.squeeze = false;
		break;
	    case "--no-seqs":
		options.make_seqs = false;
		break;
	    case "--no-dead-code":
		options.dead_code = false;
		break;
	    case "--no-copyright":
	    case "-nc":
		options.show_copyright = false;
		break;
	    case "-o":
	    case "--ouput":
	    case "-cp":
	    case "--compile":
		options.output = args.shift();
		break;
	    case "--overwrite":
		options.out_same_file = true;
		break;
	    case "-v":
	    case "--verbose":
		options.verbose = true;
		break;
	    case "--ast":
		options.ast = true;
		break;
	    case "--unsafe":
		options.unsafe = true;
		break;
	    case "--max-line-len":
		options.max_line_length = parseInt(args.shift(), 10);
		break;
		case "-m":
		case "--main":
		main = args.shift();
		commonjs = true;
		break;
		case "-p":
		case "--path":
		paths.push(args.shift().replace(/\/?$/, '/'));
		break;
		case "-cb":
		case "--combine":
		combine = args.shift();
		break;
		case "--commonjs":
		case "-cm":
		commonjs = true;
		break;
		case "--force-commonjs":
		case "-fcm":
		forceCommonJS = true;
		break;
	    default:
		filenames.push(v);
		break;
	}
}

/////// CUSTOM COMBINER

RegExp.escape = function(text) {
	return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
};

var modules = {};
var combined = '';
var loading = 0;
var files = [];
//var rootRegex = root ? new RegExp('^' + RegExp.escape(root)) : undefined;
var requireRegex = /\brequire\(("|')(.+?)\1\)/g;
var reqOrExpRegex = /\brequire\(("|')(.+?)\1\)|\bexports.+=/;


function finish(text) {
	if (combine) {
		console.log('loaded...', '(' + (new Date().getTime() - time) + 'ms)');
		time = new Date().getTime();
		
		fs.writeFile(combine, text, function (err) {
			if (err) throw err;
			if (options.output !== true) {
				console.log('combined...', '(' + (new Date().getTime() - time) + 'ms)');
				time = new Date().getTime();
				output(squeeze_it(text));
			} else {
				console.log('combined!', '(' + (new Date().getTime() - time) + 'ms)')
			}
		});
	} else {
		output(squeeze_it(text));
	}
}


function loadModule(moduleId, currentModuleId, pathIndex) {
	moduleId = resolveModuleId(moduleId, currentModuleId);
	var status = modules[moduleId];
	
	if (status === true || status == (pathIndex || 0)) return;
	
	console.log('loading module:', moduleId);
	
	modules[moduleId] = pathIndex = pathIndex || 0;
	var file = paths[pathIndex] + moduleId + '.js';
	++loading;
	
	fs.readFile(file, "utf8", function(err, code) {
		if (err) {
			if (++pathIndex == paths.length) throw new Error('No module "' + moduleId + '" exists in paths [' + paths.join(', ') + ']');
			--loading;
			loadModule(moduleId, currentModuleId, pathIndex);
			return;
		}
		
		modules[moduleId] = true;
		findRequired(code, moduleId);
		combined += 'require.addModuleClosure("' + moduleId + '", function(require, exports, module) {\n' + code + '\n});\n\n';
		
		// if finished loading all dependencies finish combine and minimize it
		if (!--loading) {
			if (main && !modules[main]) {
				loadMain();
			} else {
				if (main) combined += 'require.run("' + main + '");';
				finish(combined);
			}
		}
	});
}

function loadFile(file) {
	// don't work on directories
	if (!/\.js\^?$/.test(file)) {
		return;
	}
	
	var order = loading++;
{'fraction_non_alphanumeric': 0.19866213633467827, 'fraction_numerical': 0.0047780845190061585, 'mean_word_length': 2.980980557903635, 'pattern_counts': {'":': 72, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1613460', 'n_tokens_mistral': 3825, 'n_tokens_neox': 3434, 'n_words': 794}