<commit_before>// | Trace server |________________________/
// |
// | (C) Mozilla Corp
// | licensed under MPL 2.0 http://www.mozilla.org/MPL/
// \____________________________________________/   

require("../core/define")
define(function(require){

	if(process.version.indexOf('v0.6') != -1 || process.version.indexOf('v0.4') != -1){
		console.log("Node version too old, try 0.8 or 0.10")
		process.exit(-1)
		return
	}

	var path = require('path')
	var fs = require('fs')
	var url = require('url')
	var zlib = require('zlib')
	var http = require('http')
	var https = require('https')
	var crypto = require('crypto')
	var instrument = require('./instrument')
	var childproc = require('child_process')

	// the nodejs loader
	if(process.argv[2] && process.argv[2].indexOf('-l')==0) return nodeLoader()
	
	function nodeLoader(){ 
		var filter = makeFilter(process.argv[2].slice(2))

		var m = require('module').Module.prototype;
		var oldCompile = m._compile;
		var did = 1
		m._compile = function(content, filename){

			if(filter.active && filter(filename)) 
				return oldCompile.call(this, content, filename)
			// lets instrument 
			var t = instrument(filename, content, did, filter.opt)
			did = t.id
			// send the dictionary out
			var m = {dict:1, src:t.input, f:filename, d:t.d}
			if(process.send)	process.send(m)
			else process.stderr.write('\x1f'+JSON.stringify(m)+'\x17')
			return oldCompile.call(this, t.output, filename)
		}
		process.argv.splice(1,2) // remove leading arguments
		// clear require cache
		for(var k in define.require.cache) delete define.require.cache[k]
		var file = path.resolve(process.argv[1])
		define.require(file);
	}

	var fn = require('../core/fn')
	var ssl = require('../core/io_ssl')
	var ioServer = require('../core/io_server')

	function out() {
		for (var v = Array.prototype.slice.call(arguments), i = 0, c = out.colors; i < v.length; i++) {
			v[i] = String(v[i]).replace(/~(\w*)~/g, function(m, a) {
				return "\033[" + (c[a] || 0) + "m";
			}) + "\033[0m";
			process.stderr.write(v[i]);
		}
	}

	out.colors = {
		bl:"30",bo:"1",r:"0;31",g:"0;32",y:"0;33",b:"0;34",m:"0;35",c:"0;36",
		w:"0;37",br:"1;31",bg:"1;32",by:"1;33",bb:"1;34",bm:"1;35",bc:"1;36",bw:"1;37"
	}

	function makeFilter(fspec){

		if(typeof fspec == 'string') fspec = JSON.parse(fspec)

		var _do = init(fspec._do)
		var _no = init(fspec._no)

		function init(a){
			var d = []
			for(var i = 0;i<a.length;i++){
				if(a[i].charAt(0)==':') d[i] = a[i].slice(1) 
				else d[i] = new RegExp(a[i].slice(1),"i")
			}
			return d
		}

		function match(d, f){
			if(!d.length) return 0
			for(var i = 0;i<d.length;i++){
				if(typeof d[i] == 'string'){
					if(f.indexOf(d[i]) != -1) return 2
				} else if(f.match(d[i])) return 2
			}
			return 1
		}

		function f(file){
			if(match(_do, file) == 1) return true
			if(match(_no, file) == 2) return true
			return false
		}

		f.opt = fspec._opt
		f.active = _do.length || _no.length
	
		f.stringify = function(){
			return JSON.stringify(fspec)
		}

		return f
	}

	out('~~[trace.GL] ~bc~See~w~ your code. \n')

	function loadSettings(file){
		if(!fs.existsSync(file)) return
		try{
			var data = fs.readFileSync(file).toString().replace(/\/\*[\s|S]?\*\//g,'').replace(/\/\/.*?\n/g,'')
			define.settings = JSON.parse(data)
			define.settingsData = data
			define.settingsFile = file
		}
		catch(e){	
			console.log("Error reading settings file ("+file+") ",e)
		}
	}

	if(!loadSettings(path.resolve(process.cwd(),"tracegl.json")) && 
		!loadSettings("~/tracegl.json") && 
		!loadSettings(path.resolve(path.dirname(__filename),"tracegl.json")) &&
		!define.settings)
		loadSettings(path.resolve(path.dirname(__filename),"tracegl.json"))

	// argument parse variables
	function processArgs(arg){
		var sender // send messages to ui or zip
		var uiport = 2000
		var bind = "0.0.0.0"
		var tgtport = 2080
		var fspec = {_no:[], _do:[], _opt:{}}

		function usage(err){
			out('~br~'+err+'\n')
			out('~w~Usage:\n')
			out('~w~node tracegl ~c~[flag] ~g~target ~y~[args]\n')
			out('  ~g~../path/to/wwwroot ~w~Trace browser js via static fileserver\n')
			out('  ~g~http://proxytarget:port ~w~Trace browser js via proxy\n')
			out('  ~g~nodefile.js ~y~[args] ~w~Trace Node.js process\n')
			out('  ~g~trace.gz ~w~Play back trace.gz file\n')
			out('  ~c~-gz[:trace.gz] ~w~Record trace to gzip file. No trace UI started\n')
			out('  ~c~-do[/:]match ~w~Only trace filenames containing match. Filters -do set first, then -no\n')
			out('  ~c~-no[/:]match ~w~Ignore filenames containing match. Replace : with / for a regexp, use double escaped \\\\ \n')
			out('  ~c~-nolib ~w~Short for -no/jquery.* -no:require.js -no/node\\\\_modules \n')
			out('  ~c~-nocatch ~w~Don\'t create exception catching\n')
			out('  ~c~-bind:0.0.0.0 ~w~Set the hostname to bind our external ports to, default 0.0.0.0\n')
			out('  ~c~-ui:port ~w~Set trace UI port. default: 2000\n')
			out('  ~c~-tgt:port ~w~Set browser JS port. default: 2080\n')
			out('~w~node tracegl.js ~r~[commmand]\n')
			out('  ~r~-settings ~w~write a .tracegl settings template in the current dir\n')
			return
		}
		// process arguments
		for(var i = 2;i<arg.length;i++){
			var a = arg[i]
			if(a.charAt(0) == '-'){
				var d = a.indexOf(':')
				var b
				if(d!=-1) b = a.slice(d+1)

				if(a.indexOf('-gz') == 0){
					if(d!=-1) sender = gzSender(a.slice(d+1))
					else sender = gzSender('trace.gz')
				} else if(a.indexOf('-install') == 0){
				} else if(a.indexOf('-ui') == 0){
					if(d==-1) return usage("No port specified")
					uiport = parseInt(b)
				} else if(a.indexOf('-tgt') == 0){
					if(d==-1) return usage("No port specified")
					tgtport = parseInt(b)
				} else if(a.indexOf('-no') == 0){
					if(a == '-nocatch'){
						fspec._opt.nocatch = 1
					} else
					if(a == '-nolib'){
						fspec._no.push("/jquery.*")
						fspec._no.push(":require.js")
						fspec._no.push("/node\\_modules")
					} else {
						fspec._no.push(a.slice(3))
					}
				} else if(a.indexOf('-do') == 0){
					fspec._do.push(a.slice(3))
				} else if(a.indexOf('-settings') == 0){
					if(fs.existsSync('tracegl.json')){
						return out('~r~ERROR: ~~ .tracegl file already exists, remove before creating a new template\n')
					}
					fs.writeFileSync("tracegl.json", define.settingsData)
					return out('~g~OK: ~~ tracegl.jsonl file written in current directory, open it in an editor to modify settings\n')
				} else if(a.indexOf('-bind')== 0){
					bind = a.slice(6)
				} else return usage("Invalid argument "+a)
			} else {
				if(!sender) sender = uiSender(uiport, bind)
				var f = makeFilter(fspec)

				var isfile;
				try{ isfile = fs.statSync(a).isFile() } catch(e){}

				// execute the right mode
				if(a.match(/\.gz$/i)) return gzPlaybackMode(f, a, sender)
				if(a.match(/\.js$/i) || isfile) return nodeJSMode(f, a, arg.slice(i+1), sender)
				if(a.match(/^https?/)) return proxyMode(f, tgtport, bind, a, sender)
				return browserJSMode(f, tgtport, bind, path.resolve(process.cwd(), a), sender)

				break
			}<commit_msg>Added -meteor flag to ignore Meteor packages<commit_after>// | Trace server |________________________/
// |
// | (C) Mozilla Corp
// | licensed under MPL 2.0 http://www.mozilla.org/MPL/
// \____________________________________________/   

require("../core/define")
define(function(require){

	if(process.version.indexOf('v0.6') != -1 || process.version.indexOf('v0.4') != -1){
		console.log("Node version too old, try 0.8 or 0.10")
		process.exit(-1)
		return
	}

	var path = require('path')
	var fs = require('fs')
	var url = require('url')
	var zlib = require('zlib')
	var http = require('http')
	var https = require('https')
	var crypto = require('crypto')
	var instrument = require('./instrument')
	var childproc = require('child_process')

	// the nodejs loader
	if(process.argv[2] && process.argv[2].indexOf('-l')==0) return nodeLoader()
	
	function nodeLoader(){ 
		var filter = makeFilter(process.argv[2].slice(2))

		var m = require('module').Module.prototype;
		var oldCompile = m._compile;
		var did = 1
		m._compile = function(content, filename){

			if(filter.active && filter(filename)) 
				return oldCompile.call(this, content, filename)
			// lets instrument 
			var t = instrument(filename, content, did, filter.opt)
			did = t.id
			// send the dictionary out
			var m = {dict:1, src:t.input, f:filename, d:t.d}
			if(process.send)	process.send(m)
			else process.stderr.write('\x1f'+JSON.stringify(m)+'\x17')
			return oldCompile.call(this, t.output, filename)
		}
		process.argv.splice(1,2) // remove leading arguments
		// clear require cache
		for(var k in define.require.cache) delete define.require.cache[k]
		var file = path.resolve(process.argv[1])
		define.require(file);
	}

	var fn = require('../core/fn')
	var ssl = require('../core/io_ssl')
	var ioServer = require('../core/io_server')

	function out() {
		for (var v = Array.prototype.slice.call(arguments), i = 0, c = out.colors; i < v.length; i++) {
			v[i] = String(v[i]).replace(/~(\w*)~/g, function(m, a) {
				return "\033[" + (c[a] || 0) + "m";
			}) + "\033[0m";
			process.stderr.write(v[i]);
		}
	}

	out.colors = {
		bl:"30",bo:"1",r:"0;31",g:"0;32",y:"0;33",b:"0;34",m:"0;35",c:"0;36",
		w:"0;37",br:"1;31",bg:"1;32",by:"1;33",bb:"1;34",bm:"1;35",bc:"1;36",bw:"1;37"
	}

	function makeFilter(fspec){

		if(typeof fspec == 'string') fspec = JSON.parse(fspec)

		var _do = init(fspec._do)
		var _no = init(fspec._no)

		function init(a){
			var d = []
			for(var i = 0;i<a.length;i++){
				if(a[i].charAt(0)==':') d[i] = a[i].slice(1) 
				else d[i] = new RegExp(a[i].slice(1),"i")
			}
			return d
		}

		function match(d, f){
			if(!d.length) return 0
			for(var i = 0;i<d.length;i++){
				if(typeof d[i] == 'string'){
					if(f.indexOf(d[i]) != -1) return 2
				} else if(f.match(d[i])) return 2
			}
			return 1
		}

		function f(file){
			if(match(_do, file) == 1) return true
			if(match(_no, file) == 2) return true
			return false
		}

		f.opt = fspec._opt
		f.active = _do.length || _no.length
	
		f.stringify = function(){
			return JSON.stringify(fspec)
		}

		return f
	}

	out('~~[trace.GL] ~bc~See~w~ your code. \n')

	function loadSettings(file){
		if(!fs.existsSync(file)) return
		try{
			var data = fs.readFileSync(file).toString().replace(/\/\*[\s|S]?\*\//g,'').replace(/\/\/.*?\n/g,'')
			define.settings = JSON.parse(data)
			define.settingsData = data
			define.settingsFile = file
		}
		catch(e){	
			console.log("Error reading settings file ("+file+") ",e)
		}
	}

	if(!loadSettings(path.resolve(process.cwd(),"tracegl.json")) && 
		!loadSettings("~/tracegl.json") && 
		!loadSettings(path.resolve(path.dirname(__filename),"tracegl.json")) &&
		!define.settings)
		loadSettings(path.resolve(path.dirname(__filename),"tracegl.json"))

	// argument parse variables
	function processArgs(arg){
		var sender // send messages to ui or zip
		var uiport = 2000
		var bind = "0.0.0.0"
		var tgtport = 2080
		var fspec = {_no:[], _do:[], _opt:{}}

		function usage(err){
			out('~br~'+err+'\n')
			out('~w~Usage:\n')
			out('~w~node tracegl ~c~[flag] ~g~target ~y~[args]\n')
			out('  ~g~../path/to/wwwroot ~w~Trace browser js via static fileserver\n')
			out('  ~g~http://proxytarget:port ~w~Trace browser js via proxy\n')
			out('  ~g~nodefile.js ~y~[args] ~w~Trace Node.js process\n')
			out('  ~g~trace.gz ~w~Play back trace.gz file\n')
			out('  ~c~-gz[:trace.gz] ~w~Record trace to gzip file. No trace UI started\n')
			out('  ~c~-do[/:]match ~w~Only trace filenames containing match. Filters -do set first, then -no\n')
			out('  ~c~-no[/:]match ~w~Ignore filenames containing match. Replace : with / for a regexp, use double escaped \\\\ \n')
			out('  ~c~-nolib ~w~Short for -no/jquery.* -no:require.js -no/node\\\\_modules \n')
			out('  ~c~-meteor ~w~Short for -no/jquery.* -no:/packages \n')
			out('  ~c~-nocatch ~w~Don\'t create exception catching\n')
			out('  ~c~-bind:0.0.0.0 ~w~Set the hostname to bind our external ports to, default 0.0.0.0\n')
			out('  ~c~-ui:port ~w~Set trace UI port. default: 2000\n')
			out('  ~c~-tgt:port ~w~Set browser JS port. default: 2080\n')
			out('~w~node tracegl.js ~r~[commmand]\n')
			out('  ~r~-settings ~w~write a .tracegl settings template in the current dir\n')
			return
		}
		// process arguments
		for(var i = 2;i<arg.length;i++){
			var a = arg[i]
			if(a.charAt(0) == '-'){
				var d = a.indexOf(':')
				var b
				if(d!=-1) b = a.slice(d+1)

				if(a.indexOf('-gz') == 0){
					if(d!=-1) sender = gzSender(a.slice(d+1))
					else sender = gzSender('trace.gz')
				} else if(a.indexOf('-install') == 0){
				} else if(a.indexOf('-ui') == 0){
					if(d==-1) return usage("No port specified")
					uiport = parseInt(b)
				} else if(a.indexOf('-tgt') == 0){
					if(d==-1) return usage("No port specified")
					tgtport = parseInt(b)
				} else if(a.indexOf('-no') == 0){
					if(a == '-nocatch'){
						fspec._opt.nocatch = 1
					} else
					if(a == '-nolib'){
						fspec._no.push("/jquery.*")
						fspec._no.push(":require.js")
						fspec._no.push("/node\\_modules")
					} else {
						fspec._no.push(a.slice(3))
					}
				} else if (a == '-meteor') {
					fspec._no.push("/jquery.*")
					fspec._no.push("/packages")
				}else if(a.indexOf('-do') == 0){
					fspec._do.push(a.slice(3))
				} else if(a.indexOf('-settings') == 0){
					if(fs.existsSync('tracegl.json')){
						return out('~r~ERROR: ~~ .tracegl file already exists, remove before creating a new template\n')
					}
					fs.writeFileSync("tracegl.json", define.settingsData)
					return out('~g~OK: ~~ tracegl.jsonl file written in current directory, open it in an editor to modify settings\n')
				} else if(a.indexOf('-bind')== 0){
					bind = a.slice(6)
				} else return usage("Invalid argument "+a)
			} else {
				if(!sender) sender = uiSender(uiport, bind)
				var f = makeFilter(fspec)

				var isfile;
				try{ isfile = fs.statSync(a).isFile() } catch(e){}

				// execute the right mode
				if(a.match(/\.gz$/i)) return gzPlaybackMode(f, a, sender)
				if(a.match(/\.js$/i) || isfile) return nodeJSMode(f, a, arg.slice(i+1), sender)
				if(a.match(/^https?/)) return proxyMode(f, tgtport, bind, a, sender)
				return browserJSMode(f, tgtport, bind, path.resolve(process.cwd(), a), sender)

				break
			}
{'fraction_non_alphanumeric': 0.20315664501711014, 'fraction_numerical': 0.01983378727564774, 'mean_word_length': 3.2797369994022714, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '77686', 'n_tokens_mistral': 6328, 'n_tokens_neox': 5425, 'n_words': 1195}