<commit_before>var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);

var SourceMapConsumer = require('source-map').SourceMapConsumer;
var request = require('request');
var nUrl = require('url');
var validate = require('sourcemap-validator');

function GetMapFileUrl(mapFiles, url) {
	var mapFile = mapFiles[0].substr('//# sourceMappingURL='.length);

	var host = nUrl.parse(url);

	var mapUrl = host.protocol + "//" + host.host;

	var urlPaths = host.path.split('/');
	for (var i = 0; i < urlPaths.length - 1; i++) {
		mapUrl += urlPaths[i] + '/';
	}

	mapUrl = nUrl.resolve(mapUrl, mapFile);

	return mapUrl;
}

function GetSourceFileUrl(sourceFile, url) {
	var host = nUrl.parse(url);

	var mapUrl = host.protocol + "//" + host.host;

	var urlPaths = host.path.split('/');
	for (var i = 0; i < urlPaths.length - 1; i++) {
		mapUrl += urlPaths[i] + '/';
	}

	mapUrl = nUrl.resolve(mapUrl, sourceFile);

	return mapUrl;
}

function GetScriptName(url) {
	var bits = url.split('/');
	return bits[bits.length - 1];
}

app.get('/', function(req, res) {
	res.sendFile(__dirname + '/index.html');
});

io.on('connection', function(socket) {
	socket.on('chat message', function(msg) {
		io.emit('chat message', msg);
	});

	socket.on('process', function(url) {
		if(!url){
			socket.emit('log','ERROR: No url');
			return;
		}

		if (url.indexOf("http://") === -1 && url.indexOf('https://') === -1) {
			url = 'http://' + url;
		}

		var errors = [];

		var minifiedJS = null;
		var sourceMap = null;
		var normalJS = null;

		socket.emit('log', "Trying to get " + url);
		request.get(url, function(error, response, body) {
			minifiedJS = body;

			var regex = /\/\/# sourceMappingURL=(.*)/gmi;

			if (error || response.statusCode !== 200) {
				socket.emit('log', "Getting " + url + " failed - " + response.statusCode + error);
				console.log("Getting", url, "failed -", response.statusCode, error);
				return;
			}

			socket.emit('log', "Got " + url + " (" + response.statusCode + ")");
			console.log("Got", url, "(" + response.statusCode + ")");
			var mapFiles = body.match(regex);

			if(!mapFiles){
				socket.emit('log','ERROR: no log files found');
				return;
			}

			if (mapFiles.length > 1) {
				socket.emit('log', "ERROR: Too many map files");
				return;
			}

			var mapUrl = GetMapFileUrl(mapFiles, url);
			socket.emit('log', 'Trying to get map file ' + mapUrl);

			request.get(mapUrl, function(err, response, body) {
				if (err) {
					socket.emit('log', 'ERROR: Couldn\'t get map file, error: ' + response.statusCodee);
					return;
				}

				socket.emit('log', 'Got the map file');
				sourceMap = body;<commit_msg>Updated message<commit_after>var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);

var SourceMapConsumer = require('source-map').SourceMapConsumer;
var request = require('request');
var nUrl = require('url');
var validate = require('sourcemap-validator');

function GetMapFileUrl(mapFiles, url) {
	var mapFile = mapFiles[0].substr('//# sourceMappingURL='.length);

	var host = nUrl.parse(url);

	var mapUrl = host.protocol + "//" + host.host;

	var urlPaths = host.path.split('/');
	for (var i = 0; i < urlPaths.length - 1; i++) {
		mapUrl += urlPaths[i] + '/';
	}

	mapUrl = nUrl.resolve(mapUrl, mapFile);

	return mapUrl;
}

function GetSourceFileUrl(sourceFile, url) {
	var host = nUrl.parse(url);

	var mapUrl = host.protocol + "//" + host.host;

	var urlPaths = host.path.split('/');
	for (var i = 0; i < urlPaths.length - 1; i++) {
		mapUrl += urlPaths[i] + '/';
	}

	mapUrl = nUrl.resolve(mapUrl, sourceFile);

	return mapUrl;
}

function GetScriptName(url) {
	var bits = url.split('/');
	return bits[bits.length - 1];
}

app.get('/', function(req, res) {
	res.sendFile(__dirname + '/index.html');
});

io.on('connection', function(socket) {
	socket.on('chat message', function(msg) {
		io.emit('chat message', msg);
	});

	socket.on('process', function(url) {
		if(!url){
			socket.emit('log','ERROR: No url');
			return;
		}

		if (url.indexOf("http://") === -1 && url.indexOf('https://') === -1) {
			url = 'http://' + url;
		}

		var errors = [];

		var minifiedJS = null;
		var sourceMap = null;
		var normalJS = null;

		socket.emit('log', "Trying to get " + url);
		request.get(url, function(error, response, body) {
			minifiedJS = body;

			var regex = /\/\/# sourceMappingURL=(.*)/gmi;

			if (error || response.statusCode !== 200) {
				socket.emit('log', "Getting " + url + " failed - " + response.statusCode + error);
				console.log("Getting", url, "failed -", response.statusCode, error);
				return;
			}

			socket.emit('log', "Got " + url + " (" + response.statusCode + ")");
			console.log("Got", url, "(" + response.statusCode + ")");
			var mapFiles = body.match(regex);

			if(!mapFiles){
				socket.emit('log','ERROR: no source map references found');
				return;
			}

			if (mapFiles.length > 1) {
				socket.emit('log', "ERROR: Too many map files");
				return;
			}

			var mapUrl = GetMapFileUrl(mapFiles, url);
			socket.emit('log', 'Trying to get map file ' + mapUrl);

			request.get(mapUrl, function(err, response, body) {
				if (err) {
					socket.emit('log', 'ERROR: Couldn\'t get map file, error: ' + response.statusCodee);
					return;
				}

				socket.emit('log', 'Got the map file');
				sourceMap = body;
{'fraction_non_alphanumeric': 0.1951129211403184, 'fraction_numerical': 0.004442798963346909, 'mean_word_length': 3.528918692372171, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1572744', 'n_tokens_mistral': 2103, 'n_tokens_neox': 1851, 'n_words': 437}