<commit_before>"use strict";

var path = require('path');
var fs = require('fs');
var url = require('url');
var _ = require('underscore');
var zlib = require('zlib');
var winston = require('winston');
var logger = null;

var getToken = function() {
	return Math.random().toString(28).substring(2, 8);
};

exports.parseUrl = function() {
	return url.parse.apply(this, arguments);
}

exports.getRoot = function(filename, root) {
	var parsed = url.parse(filename);
	if(root) {
		parsed = url.parse(root);
		// If we don't have a protocol we need the real path
		if(!parsed.protocol) {
			return url.parse(fs.realpathSync(root));
		}
		return parsed;
	}
	if(!parsed.protocol) {
		return url.parse(process.cwd());
	}
	return parsed;
}

exports.generateToken = function(tokens) {
	var token = getToken();
	while(tokens && tokens.indexOf(token) !== -1) {
		token = getToken();
	}
	return token;
};

exports.getUrl = function(hostUrl, filename, params) {
	var urlData = url.parse(hostUrl, true);
	return url.format(_.extend(_.pick(urlData, 'protocol', 'host', 'port'), {
		query : _.extend({}, urlData.params, params),
		pathname : path.normalize('/' + filename)
	}));
};

exports.getLogger = function() {
	if(logger === null) {
		logger = winston;
		logger.remove(winston.transports.Console);
		logger.add(winston.transports.Console, {
			level : 'error',
			colorize : true
		});
	}

	return logger;
};

exports.parseBrowser = function(str) {
	var parts = str.split(/:|@/);
	var result = { browser : parts[0] };
	if(parts[1]) {
		result.version = parts[1];
	}
	if(parts[2]) {
		result.os = parts[2];
	}
	return result;
};

exports.unzip = function(res, data, callback) {<commit_msg>Fixes #18. Issue with url encoding. All tests passing again.<commit_after>"use strict";

var path = require('path');
var fs = require('fs');
var url = require('url');
var _ = require('underscore');
var zlib = require('zlib');
var winston = require('winston');
var logger = null;

var getToken = function() {
	return Math.random().toString(28).substring(2, 8);
};

exports.parseUrl = function() {
	return url.parse.apply(this, arguments);
}

exports.getRoot = function(filename, root) {
	var parsed = url.parse(filename);
	if(root) {
		parsed = url.parse(root);
		// If we don't have a protocol we need the real path
		if(!parsed.protocol) {
			return url.parse(fs.realpathSync(root));
		}
		return parsed;
	}
	if(!parsed.protocol) {
		return url.parse(process.cwd());
	}
	return parsed;
}

exports.generateToken = function(tokens) {
	var token = getToken();
	while(tokens && tokens.indexOf(token) !== -1) {
		token = getToken();
	}
	return token;
};

exports.getUrl = function(hostUrl, filename, params) {
	var fullPath = hostUrl + path.normalize('/' + filename);
	var urlData = url.parse(fullPath, true);
	return url.format(_.extend(_.pick(urlData, 'protocol', 'host', 'port'), {
		query : _.extend({}, urlData.query, params),
		pathname : urlData.pathname
	}));
};

exports.getLogger = function() {
	if(logger === null) {
		logger = winston;
		logger.remove(winston.transports.Console);
		logger.add(winston.transports.Console, {
			level : 'error',
			colorize : true
		});
	}

	return logger;
};

exports.parseBrowser = function(str) {
	var parts = str.split(/:|@/);
	var result = { browser : parts[0] };
	if(parts[1]) {
		result.version = parts[1];
	}
	if(parts[2]) {
		result.os = parts[2];
	}
	return result;
};

exports.unzip = function(res, data, callback) {
{'fraction_non_alphanumeric': 0.17659018297995935, 'fraction_numerical': 0.006389776357827476, 'mean_word_length': 4.258015267175573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4756342', 'n_tokens_mistral': 1249, 'n_tokens_neox': 1195, 'n_words': 273}