<commit_before>module.exports = {
	compileJS: compileJS,
	compileJSFile: compileJSFile,
	compressJS: compressJS,
	indentJS: indentJS
}

var fs = require('fs'),
	sys = require('sys'),
	path = require('path'),
	util = require('util'),
	child_process = require('child_process')

/* Compile a javascript file
 ***************************/
function compileJSFile(filePath) {
	return compileJS(_readFile(filePath), path.dirname(filePath))
}

function compileJS(code, basePath) {
	return indentJS('var require = {}\n' + compileJSModule(code, {}, basePath))
}

/* Compress/minify with google closure
 *************************************/
// TODO: Look into
// provide a closure to make all variables local: code = '(function(){'+code+'})()'
// --compilation_level [WHITESPACE_ONLY | SIMPLE_OPTIMIZATIONS | ADVANCED_OPTIMIZATIONS]
// --compute_phase_ordering: Runs the compile job many times, then prints out the best phase ordering from this run
// --define (--D, -D) VAL Override the value of a variable annotated @define. The format is <name>[=<val>], where <name> is the name of a @define variable and <val> is a boolean, number, or a single-quot ed string that contains no single quotes. If [=<val>] is omitted, the variable is marked true
// --print_ast, --print_pass_graph, --print_tree
function compressJS(code, callback) {
	var closureArgs = ['-jar', __dirname + '/google-closure.jar']
	
	var closure = child_process.spawn('java', closureArgs)
		stdout = [],
		stderr = []
	closure.stdout.on('data', function(data) { stdout.push(data); });
	closure.stderr.on('data', function(data) { stderr.push(data); });
	closure.on('exit', function(code) {
		if (code == 0) {
			callback(stdout.join(''))
		} else {
			util.debug(stderr.join(''))
			callback('')
		}
	})
	closure.stdin.write(code)
	closure.stdin.end()
}

/* Compile a javascript module
 *****************************/
var globalRequireRegex = /require\s*\(['"][\w\/\.-]*['"]\)/g,
	pathnameGroupingRegex = /require\s*\(['"]([\w\/\.-]*)['"]\)/
function compileJSModule(code, modules, pathBase) {
	var requireStatements = code.match(globalRequireRegex)
	
	if (!requireStatements) { return code }
	
	for (var i=0, requireStatement; requireStatement = requireStatements[i]; i++) {
		
		var rawModulePath = requireStatement.match(pathnameGroupingRegex)[1],
			isRelative = (rawModulePath[0] == '.'),
			searchPath = isRelative ? path.join(pathBase, rawModulePath) : (require.resolve(rawModulePath) || '').replace(/\.js$/, ''), // use node's resolution system is it's an installed package, e.g. require('socket.io/support/clients/socket.io')
			modulePath = findTruePath(searchPath, modules)

		if (modules[modulePath]) {
			code = code.replace(requireStatement, 'require["'+modulePath+'"].exports')
		} else {
			modules[modulePath] = true
			var newPathBase = path.dirname(modulePath)
			code = code.replace(requireStatement, [
					// we could replace require('path') with (function(){})(). However, if
					// you have require('path1') require('path2'), then we end up with
					// (function a(){})() (function b(){}()). This causes a runtime error,
					// since it gets interpreted as an invocation of the return value of
					// function a. Instead, insert a throwaway variable assignment, such that
					// we get require._=(function a(){})() require._=(function b(){})(). This
					// gets interpreted as two separate statements, which solves the problem.
					// (an alternative faulty approach is to put a semi-colon at the end of
					// the function invocation, but that would fail in the case of
					// var a = require('a'), b = require('b'))
					'require._=(function() {',
					'	var module = require["'+modulePath+'"] = {exports:{}}, exports = module.exports',
					'	// start '+modulePath+' code',
						compileJSModule(_readFile(modulePath + '.js'), modules, newPathBase),
					'	// end '+modulePath+' module code',
					'	return module.exports',
					'})()'
				].join('\n'))
		}
	}
	return code
}

function findTruePath(modulePath, modules) {
	function tryPath(p) {
		return (!!modules[p] || path.existsSync(p+'.js'))
	}
	if (tryPath(modulePath)) { return modulePath }
	if (tryPath(modulePath + '/index')) { return modulePath + '/index' }
	if (tryPath(modulePath + 'index')) { return modulePath + 'index' }
	throw 'require compiler: could not resolve "' + modulePath + '"'
}


/* Code indentation
 ******************/
function indentJS(code) {
	var lines = code.replace(/\t/g, '').split('\n'),
		result = [],
		indentation = 0
	
	for (var i=0, line; i < lines.length; i++) {
		line = lines[i]
		
		if (line.match(/^\s*\}/)) { indentation-- }
		result.push(_repeat('\t', indentation) + line)
		if (!line.match(/^\s*\/\//) && line.match(/\{\s*$/)) { indentation++ }
	}<commit_msg>Seperate out the replacing of require statements into its own function, in prep for some other massaging in compileJSModule<commit_after>module.exports = {
	compileJS: compileJS,
	compileJSFile: compileJSFile,
	compressJS: compressJS,
	indentJS: indentJS
}

var fs = require('fs'),
	sys = require('sys'),
	path = require('path'),
	util = require('util'),
	child_process = require('child_process')

/* Compile a javascript file
 ***************************/
function compileJSFile(filePath) {
	return compileJS(_readFile(filePath), path.dirname(filePath))
}

function compileJS(code, basePath) {
	return indentJS('var require = {}\n' + compileJSModule(code, basePath))
}

/* Compress/minify with google closure
 *************************************/
// TODO: Look into
// provide a closure to make all variables local: code = '(function(){'+code+'})()'
// --compilation_level [WHITESPACE_ONLY | SIMPLE_OPTIMIZATIONS | ADVANCED_OPTIMIZATIONS]
// --compute_phase_ordering: Runs the compile job many times, then prints out the best phase ordering from this run
// --define (--D, -D) VAL Override the value of a variable annotated @define. The format is <name>[=<val>], where <name> is the name of a @define variable and <val> is a boolean, number, or a single-quot ed string that contains no single quotes. If [=<val>] is omitted, the variable is marked true
// --print_ast, --print_pass_graph, --print_tree
function compressJS(code, callback) {
	var closureArgs = ['-jar', __dirname + '/google-closure.jar']
	
	var closure = child_process.spawn('java', closureArgs)
		stdout = [],
		stderr = []
	closure.stdout.on('data', function(data) { stdout.push(data); });
	closure.stderr.on('data', function(data) { stderr.push(data); });
	closure.on('exit', function(code) {
		if (code == 0) {
			callback(stdout.join(''))
		} else {
			util.debug(stderr.join(''))
			callback('')
		}
	})
	closure.stdin.write(code)
	closure.stdin.end()
}

/* Compile require statements
 ****************************/
function compileJSModule(code, pathBase) {
	var modules = {}
	code = _replaceRequireStatements(code, modules, pathBase)
	return code
}

var _globalRequireRegex = /require\s*\(['"][\w\/\.-]*['"]\)/g,
	_pathnameGroupingRegex = /require\s*\(['"]([\w\/\.-]*)['"]\)/

var _replaceRequireStatements = function(code, modules, pathBase) {
	var requireStatements = code.match(_globalRequireRegex)
	
	if (!requireStatements) { return code }
	
	for (var i=0, requireStatement; requireStatement = requireStatements[i]; i++) {
		
		var rawModulePath = requireStatement.match(_pathnameGroupingRegex)[1],
			isRelative = (rawModulePath[0] == '.'),
			// use node's resolution system is it's an installed package, e.g. require('socket.io/support/clients/socket.io')
			searchPath = isRelative ? path.join(pathBase, rawModulePath) : (require.resolve(rawModulePath) || '').replace(/\.js$/, ''),
			modulePath = _findTruePath(searchPath, modules)

		if (modules[modulePath]) {
			code = code.replace(requireStatement, 'require["'+modulePath+'"].exports')
		} else {
			modules[modulePath] = true
			var newPathBase = path.dirname(modulePath)
			code = code.replace(requireStatement, [
					// we could replace require('path') with (function(){})(). However, if
					// you have require('path1') require('path2'), then we end up with
					// (function a(){})() (function b(){}()). This causes a runtime error,
					// since it gets interpreted as an invocation of the return value of
					// function a. Instead, insert a throwaway variable assignment, such that
					// we get require._=(function a(){})() require._=(function b(){})(). This
					// gets interpreted as two separate statements, which solves the problem.
					// (an alternative faulty approach is to put a semi-colon at the end of
					// the function invocation, but that would fail in the case of
					// var a = require('a'), b = require('b'))
					'require._=(function() {',
					'	var module = require["'+modulePath+'"] = {exports:{}}, exports = module.exports',
					'	// start '+modulePath+' code',
						_replaceRequireStatements(_readFile(modulePath + '.js'), modules, newPathBase),
					'	// end '+modulePath+' module code',
					'	return module.exports',
					'})()'
				].join('\n'))
		}
	}
	return code
}

var _findTruePath = function(modulePath, modules) {
	function tryPath(p) {
		return (!!modules[p] || path.existsSync(p+'.js'))
	}
	if (tryPath(modulePath)) { return modulePath }
	if (tryPath(modulePath + '/index')) { return modulePath + '/index' }
	if (tryPath(modulePath + 'index')) { return modulePath + 'index' }
	throw 'require compiler: could not resolve "' + modulePath + '"'
}


/* Code indentation
 ******************/
function indentJS(code) {
	var lines = code.replace(/\t/g, '').split('\n'),
		result = [],
		indentation = 0
	
	for (var i=0, line; i < lines.length; i++) {
		line = lines[i]
		
		if (line.match(/^\s*\}/)) { indentation-- }
		result.push(_repeat('\t', indentation) + line)
		if (!line.match(/^\s*\/\//) && line.match(/\{\s*$/)) { indentation++ }
	}
{'fraction_non_alphanumeric': 0.18813905930470348, 'fraction_numerical': 0.0016359918200408998, 'mean_word_length': 4.128998426848453, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3658706', 'n_tokens_mistral': 3288, 'n_tokens_neox': 2997, 'n_words': 836}