<commit_before>jsio('from base import *');
var logger = logging.get('jsio_compiler');

var JSIO = 'jsio';

var supportedEnvs = {
	node: true,
	browser: true
};

var interface = null;

exports.start = function(/*optional*/ args, opts) {
	if (!jsio.__env.name in supportedEnvs) {
		logger.error("autostart failed: unknown environment.\n\n\tTry using compiler.run(args, opts) instead.");
		return;
	}
	
	var DYNAMIC_IMPORT_COMPILER = '.' + jsio.__env.name + '_interface';
	
	interface = jsio(DYNAMIC_IMPORT_COMPILER);
	
	// expects the interface to eventually call startWithOpts to do the actual compile
	interface.init(exports, args, opts);
}

function getPackage(fileName) {
	try {
		var pkg = eval('(' + jsio.__env.fetch(fileName) + ')');
		logger.info('Package definition loaded from', fileName);
		return pkg;
	} catch(e) {
		logger.log(jsio.__env.getCwd())
		logger.warn('If "' + fileName + '" is a package file, it could not be read.', e);
	}
	return false;
}

/**
 * args : array of arguments
 *   - args[0] : string - initial import string (optional if opts.package is provided)
 * opts : object
 *   - package : string - filename of a package definition
 *   - debug : integer - debug level (1 - 4)
 */
exports.run = function(args, opts) {
	
	var debugLevel = 'debug' in opts ? opts.debug : 4;

	logger.setLevel(debugLevel);
	interface.logger.setLevel(debugLevel);
	logging.get('preprocessors.compiler').setLevel(debugLevel);
	
	if (debugLevel >= 3) {
		var strOpts = JSON.stringify(opts, null, '\t');
		logger.info('Starting compiler with args: ', args, 'and options:', strOpts.substring(1, strOpts.length - 1));
	}
	
	// use external copy of jsio rather than cached copy
	if (opts.jsioPath) {
		// force the path
		jsio.path.set([opts.jsioPath]);
		
		// reset cached path
		for (var key in jsio.path.cache) {
			delete jsio.path.cache[key];
		}
		
		// delete the cache copy
		var sourceCache = jsio.__jsio.__srcCache;
		for (var i in sourceCache) {
			delete sourceCache[i];
		}
	}
	
	if (opts.path) {
		for(var i = 0, len = opts.path.length; i < len; ++i) {
			if (opts.path[i]) {
				jsio.path.add(opts.path[i]);
			}
		}
	}
	
	logger.info('js.io path:', JSON.stringify(jsio.path.get()));
	
	var initial;
	
	// -- parse options --
	// try to maintain consistency with pyjsiocompile
	
	// accept a pkg file as the first argument
	if (/\.pkg$/.test(args[2])) {
		var pkg = getPackage(args[2]);

		// was it a valid pkg file?
		// (our test would also return true for "import foo.bar.pkg")
		if (pkg != false) {
			args.splice(2, 1); // consume the argument
			opts.package = pkg; // treat the package the same as if it was specified on the command line
		}
	}
	
	// opts.package is probably the filename of the package
	if (typeof opts.package == 'string' && /\.pkg$/.test(opts.package)) {
		opts.package = getPackage(opts.package);
	} 

	// parse the package contents
	if (opts.package) {
		var pkgDef = opts.package;
		
		// in pyjsiocompile, root does two things:
		if ('root' in pkgDef) {
			// 1. provide the initial import
			initial = pkgDef.root;

			// pyjsiocompile package files don't have a relative import indicator (a prefix dot: '.')<commit_msg>if a custom jsio path is provided to the compiler, it should use that path for jsio.js rather than the compiler copy<commit_after>jsio('from base import *');
var logger = logging.get('jsio_compiler');

var JSIO = 'jsio';

var supportedEnvs = {
	node: true,
	browser: true
};

var interface = null;

exports.start = function(/*optional*/ args, opts) {
	if (!jsio.__env.name in supportedEnvs) {
		logger.error("autostart failed: unknown environment.\n\n\tTry using compiler.run(args, opts) instead.");
		return;
	}
	
	var DYNAMIC_IMPORT_COMPILER = '.' + jsio.__env.name + '_interface';
	
	interface = jsio(DYNAMIC_IMPORT_COMPILER);
	
	// expects the interface to eventually call startWithOpts to do the actual compile
	interface.init(exports, args, opts);
}

function getPackage(fileName) {
	try {
		var pkg = eval('(' + jsio.__env.fetch(fileName) + ')');
		logger.info('Package definition loaded from', fileName);
		return pkg;
	} catch(e) {
		logger.log(jsio.__env.getCwd())
		logger.warn('If "' + fileName + '" is a package file, it could not be read.', e);
	}
	return false;
}

/**
 * args : array of arguments
 *   - args[0] : string - initial import string (optional if opts.package is provided)
 * opts : object
 *   - package : string - filename of a package definition
 *   - debug : integer - debug level (1 - 4)
 */
exports.run = function(args, opts) {
	
	var debugLevel = 'debug' in opts ? opts.debug : 4;

	logger.setLevel(debugLevel);
	interface.logger.setLevel(debugLevel);
	logging.get('preprocessors.compiler').setLevel(debugLevel);
	
	if (debugLevel >= 3) {
		var strOpts = JSON.stringify(opts, null, '\t');
		logger.info('Starting compiler with args: ', args, 'and options:', strOpts.substring(1, strOpts.length - 1));
	}
	
	// use external copy of jsio rather than cached copy
	if (opts.jsioPath) {
		// force the path
		jsio.path.set([opts.jsioPath]);
		
		// hack to 'set' the js.io source code
		jsio.__jsio.__init__.toString = function() { return jsio.__env.fetch(jsio.__jsio.__util.buildPath(opts.jsioPath, 'jsio.js')); }
		
		// reset cached path
		for (var key in jsio.path.cache) {
			delete jsio.path.cache[key];
		}
		
		// delete the cache copy
		var sourceCache = jsio.__jsio.__srcCache;
		for (var i in sourceCache) {
			delete sourceCache[i];
		}
	}
	
	if (opts.path) {
		for(var i = 0, len = opts.path.length; i < len; ++i) {
			if (opts.path[i]) {
				jsio.path.add(opts.path[i]);
			}
		}
	}
	
	logger.info('js.io path:', JSON.stringify(jsio.path.get()));
	
	var initial;
	
	// -- parse options --
	// try to maintain consistency with pyjsiocompile
	
	// accept a pkg file as the first argument
	if (/\.pkg$/.test(args[2])) {
		var pkg = getPackage(args[2]);

		// was it a valid pkg file?
		// (our test would also return true for "import foo.bar.pkg")
		if (pkg != false) {
			args.splice(2, 1); // consume the argument
			opts.package = pkg; // treat the package the same as if it was specified on the command line
		}
	}
	
	// opts.package is probably the filename of the package
	if (typeof opts.package == 'string' && /\.pkg$/.test(opts.package)) {
		opts.package = getPackage(opts.package);
	} 

	// parse the package contents
	if (opts.package) {
		var pkgDef = opts.package;
		
		logger.debug(pkgDef);
		
		// in pyjsiocompile, root does two things:
		if ('root' in pkgDef) {
			// 1. provide the initial import
			initial = pkgDef.root;

			// pyjsiocompile package files don't have a relative import indicator (a prefix dot: '.')
{'fraction_non_alphanumeric': 0.1420403250075233, 'fraction_numerical': 0.003912127595546194, 'mean_word_length': 3.293927648578811, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4501422', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2193, 'n_words': 666}