<commit_before>function extend(base) {
	for(var i = 1; i < arguments.length; i++) {
		var extension = arguments[i];
		if(typeof extension === 'object')
			for(var key in extension)
				base[key] = extension[key];
	}
	return base;
}

var regExpPattern= new RegExp(
	'(\\' + [
		'/', '.', '*', '+', '?', '|',
		'(', ')', '[', ']', '{', '}', '\\'
		].join('|\\') + ')', 'g'
);

function trim(text, seq) {
	var pattern = seq.replace(regExpPattern, '\\$1');
	return text
		.replace(new RegExp("^"+pattern), "")
		.replace(new RegExp(pattern+"$"), "")
		.replace(new RegExp("("+pattern+"){2,}", "g"), seq);
}

var setImmediate = function(cb) { cb(); };
if(typeof process !== 'undefined' && typeof process.nextTick === 'function') {
	setImmediate = process.nextTick;
}

var whitespace = /\s/g;
var nonAlphaNum = /[^a-z0-9-]/gi;

var umlautsWithE = {
	'Ä': 'Ae',
	"ä": "ae",
	'Ö': 'Oe',
	"ö": "oe",
	"ü": "ue",
	'Ü': 'Ue'
};

var szToSs = {
	'ß': 'ss'
};

var accents = {
	'À': 'A',
	'Á': 'A',
	'Â': 'A',
	'Ã': 'A',
	'Ä': 'A',
	'Å': 'AA',
	'Æ': 'AE',
	'Ç': 'C',
	'È': 'E',
	'É': 'E',
	'Ê': 'E',
	'Ë': 'E',
	'Ì': 'I',
	'Í': 'I',
	'Î': 'I',
	'Ï': 'I',
	'Ð': 'D',
	'Ł': 'L',
	'Ñ': 'N',
	'Ò': 'O',
	'Ó': 'O',
	'Ô': 'O',
	'Õ': 'O',
	'Ö': 'O',
	'Ø': 'OE',
	'Ù': 'U',
	'Ú': 'U',
	'Ü': 'U',
	'Û': 'U',
	'Ý': 'Y',
	'Þ': 'Th',
	'ß': 'sz',
	'à': 'a',
	'á': 'a',
	'â': 'a',
	'ã': 'a',
	'ä': 'a',
	'å': 'aa',
	'æ': 'ae',
	'ç': 'c',
	'è': 'e',<commit_msg>Corrects accents. Fixes #6.<commit_after>function extend(base) {
	for(var i = 1; i < arguments.length; i++) {
		var extension = arguments[i];
		if(typeof extension === 'object')
			for(var key in extension)
				base[key] = extension[key];
	}
	return base;
}

var regExpPattern= new RegExp(
	'(\\' + [
		'/', '.', '*', '+', '?', '|',
		'(', ')', '[', ']', '{', '}', '\\'
		].join('|\\') + ')', 'g'
);

function trim(text, seq) {
	var pattern = seq.replace(regExpPattern, '\\$1');
	return text
		.replace(new RegExp("^"+pattern), "")
		.replace(new RegExp(pattern+"$"), "")
		.replace(new RegExp("("+pattern+"){2,}", "g"), seq);
}

var setImmediate = function(cb) { cb(); };
if(typeof process !== 'undefined' && typeof process.nextTick === 'function') {
	setImmediate = process.nextTick;
}

var whitespace = /\s/g;
var nonAlphaNum = /[^a-z0-9-]/gi;

var umlautsWithE = {
	'Ä': 'Ae',
	"ä": "ae",
	'Ö': 'Oe',
	"ö": "oe",
	"ü": "ue",
	'Ü': 'Ue'
};

var szToSs = {
	'ß': 'ss'
};

var accents = {
	'À': 'A',
	'Á': 'A',
	'Â': 'A',
	'Ã': 'A',
	'Ä': 'A',
	'Å': 'A',
	'Æ': 'AE',
	'Ç': 'C',
	'È': 'E',
	'É': 'E',
	'Ê': 'E',
	'Ë': 'E',
	'Ì': 'I',
	'Í': 'I',
	'Î': 'I',
	'Ï': 'I',
	'Ð': 'D',
	'Ł': 'L',
	'Ñ': 'N',
	'Ò': 'O',
	'Ó': 'O',
	'Ô': 'O',
	'Õ': 'O',
	'Ö': 'O',
	'Ø': 'OE',
	'Ù': 'U',
	'Ú': 'U',
	'Ü': 'U',
	'Û': 'U',
	'Ý': 'Y',
	'Þ': 'Th',
	'ß': 'sz',
	'à': 'a',
	'á': 'a',
	'â': 'a',
	'ã': 'a',
	'ä': 'a',
	'å': 'a',
	'æ': 'ae',
	'ç': 'c',
	'è': 'e',
{'fraction_non_alphanumeric': 0.3610145934676859, 'fraction_numerical': 0.003822098679638638, 'mean_word_length': 3.3621212121212123, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2073943', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1500, 'n_words': 332}