<commit_before>var Util = exports || {},
crypto = require('crypto'),
fs = require('fs');

Util.mergeDeep = function (A, B, depth) {
	var forever = depth == null;
	for (var p in B) {
		if (B[p] != null && B[p].constructor==Object && (forever || depth > 0)) {
			A[p] = Util.mergeDeep(
				A.hasOwnProperty(p) ? A[p] : {},
				B[p],
				forever ? null : depth-1
			);
		} else {
			A[p] = B[p];
		}
	}
	return A;
}

Util.merge = function(A, B) {
	return Util.mergeDeep(A, B, 0);
}

Util.mergeCopy = function(A, B, depth) {
	var A_copy = Util.mergeDeep({}, A);
	return Util.mergeDeep(A_copy, B, depth);
}

Util.chop = function(str){
	str = str.toString();
	if(str.length == 0) return str;
	return str.substr(0, str.length - 1);
}

var chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXTZabcdefghiklmnopqrstuvwxyz".split('');
Util.randomString = function(len) {
    var result = [];
    for (var i=0; i<len; i++) {
        var index = Math.floor(Math.random() * chars.length);
        result.push(chars[index]);
    }
    return result.join('');
}

Util.md5 = function(str){
	return crypto.createHash('md5').update(str).digest("hex");
}

Util.sha1 = function(str){
	return crypto.createHash('sha1').update(str).digest("hex");
}<commit_msg>Fix Util.chop for null argument<commit_after>var Util = exports || {},
crypto = require('crypto'),
fs = require('fs');

Util.mergeDeep = function (A, B, depth) {
	var forever = depth == null;
	for (var p in B) {
		if (B[p] != null && B[p].constructor==Object && (forever || depth > 0)) {
			A[p] = Util.mergeDeep(
				A.hasOwnProperty(p) ? A[p] : {},
				B[p],
				forever ? null : depth-1
			);
		} else {
			A[p] = B[p];
		}
	}
	return A;
}

Util.merge = function(A, B) {
	return Util.mergeDeep(A, B, 0);
}

Util.mergeCopy = function(A, B, depth) {
	var A_copy = Util.mergeDeep({}, A);
	return Util.mergeDeep(A_copy, B, depth);
}

Util.chop = function(str){
	if(!str) return str;
	str = str.toString();
	if(str.length == 0) return str;
	return str.substr(0, str.length - 1);
}

var chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXTZabcdefghiklmnopqrstuvwxyz".split('');
Util.randomString = function(len) {
    var result = [];
    for (var i=0; i<len; i++) {
        var index = Math.floor(Math.random() * chars.length);
        result.push(chars[index]);
    }
    return result.join('');
}

Util.md5 = function(str){
	return crypto.createHash('md5').update(str).digest("hex");
}

Util.sha1 = function(str){
	return crypto.createHash('sha1').update(str).digest("hex");
}
{'fraction_non_alphanumeric': 0.1951121794871795, 'fraction_numerical': 0.016826923076923076, 'mean_word_length': 3.6760299625468167, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6018307', 'n_tokens_mistral': 1031, 'n_tokens_neox': 959, 'n_words': 205}