<commit_before>var request   = require('request');

function formatDate(adapterFormatDate, date) {
    if (!date) return date || '';
	if (typeof date !== 'object') date = new Date(date);
    var h = date.getHours();
    var m = date.getMinutes();

    if (h < 10) h = '0' + h.toString();
    if (m < 10) m = '0' + m.toString();

    return adapterFormatDate(date) + ' ' + h + ':' + m;
}

function _getFile(body, cb) {
	var data;
	try {
		if (body.substring(0, 'warnWetter.loadWarnings('.length) == 'warnWetter.loadWarnings(') {
			body = body.substring('warnWetter.loadWarnings('.length);
			while (body[body.length - 1] !== '}') {
				body = body.substring(0, body.length - 1);
			}
		}
		data = JSON.parse(body);
	} catch (e) {
		require('fs').writeFileSync(__dirname + '/problem.json', body);
		cb('Cannot parse JSON file.');
		return;
	}
	cb(null, data);
}

function getFile(url, cb) {
    if (!url.match(/^http:\/\/|^https:\/\//)) {
        _getFile(require('fs').readFileSync(url).toString(), cb);
    } else {
		request(url, function (error, response, body) {
			if (!error && response.statusCode == 200) {
				_getFile(body, cb);
			} else {
				cb('Cannot read JSON file: ' + error || response.statusCode);
			}
		});
	}
}

function sort(a, b) {
	if (a && !b)  			return 1;
	if (b && !a)  			return -1;
	if (!a && !b) 			return 0;
	if (a.type > b.type) 	return 1;
	if (b.type > a.type) 	return -1;
	if (a.start > b.start) 	return 1;
	if (b.start > a.start) 	return -1;
	if (a.end > b.end) 		return 1;
	if (b.end > a.end) 		return -1;
	if (a.level > b.level) 	return 1;
	if (b.level > a.level) 	return -1;
	return 0;
}


module.exports.getFile    = getFile;
module.exports.formatDate = formatDate;
module.exports.sort       = sort;<commit_msg>fixed bug where cb() needs to be called with a 'null' parameter to correctly emit an error message in case the json file couldn't be downloaded.<commit_after>var request   = require('request');

function formatDate(adapterFormatDate, date) {
    if (!date) return date || '';
	if (typeof date !== 'object') date = new Date(date);
    var h = date.getHours();
    var m = date.getMinutes();

    if (h < 10) h = '0' + h.toString();
    if (m < 10) m = '0' + m.toString();

    return adapterFormatDate(date) + ' ' + h + ':' + m;
}

function _getFile(body, cb) {
	var data;
	try {
		if (body.substring(0, 'warnWetter.loadWarnings('.length) == 'warnWetter.loadWarnings(') {
			body = body.substring('warnWetter.loadWarnings('.length);
			while (body[body.length - 1] !== '}') {
				body = body.substring(0, body.length - 1);
			}
		}
		data = JSON.parse(body);
	} catch (e) {
		require('fs').writeFileSync(__dirname + '/problem.json', body);
		cb('Cannot parse JSON file.', null);
		return;
	}
	cb(null, data);
}

function getFile(url, cb) {
    if (!url.match(/^http:\/\/|^https:\/\//)) {
        _getFile(require('fs').readFileSync(url).toString(), cb);
    } else {
		request(url, function (error, response, body) {
			if (!error && response.statusCode == 200) {
				_getFile(body, cb);
			} else {
				cb('Cannot read JSON file: ' + error || response.statusCode);
			}
		});
	}
}

function sort(a, b) {
	if (a && !b)  			return 1;
	if (b && !a)  			return -1;
	if (!a && !b) 			return 0;
	if (a.type > b.type) 	return 1;
	if (b.type > a.type) 	return -1;
	if (a.start > b.start) 	return 1;
	if (b.start > a.start) 	return -1;
	if (a.end > b.end) 		return 1;
	if (b.end > a.end) 		return -1;
	if (a.level > b.level) 	return 1;
	if (b.level > a.level) 	return -1;
	return 0;
}


module.exports.getFile    = getFile;
module.exports.formatDate = formatDate;
module.exports.sort       = sort;

{'fraction_non_alphanumeric': 0.18873626373626373, 'fraction_numerical': 0.013736263736263736, 'mean_word_length': 2.9619151251360174, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6146281', 'n_tokens_mistral': 1495, 'n_tokens_neox': 1384, 'n_words': 353}