<commit_before>module.exports = {

	notes:		require('./notes'),



	tokenize: function (name) {
		var result = name
		.toLowerCase()

		// German umlauts
		.replace(/ß/g, 'ss')
		.replace(/ä/g, 'ae')
		.replace(/ö/g, 'oe')
		.replace(/ü/g, 'ue')

		// expand abbreviations
		.replace(/\(pl\)/g, 'polen')
		.replace(/\(berlin\)/g, 'berlin')
		.replace(/\(bln\)/g, 'bln')

		.replace(/^[SU](\+[SU])?\W/, ' ')   // remove `S+U` abbreviations
		.split(/[\s\/\(\)]+/);

		var i;
		for (i = 0; i < result.length; i++) {
			result[i] = result[i]
			// expand abbreviations
			.replace(/str\.?/, 'strasse')
			.replace(/bhf\.?/, 'bahnhof')
			.replace(/hbf\.?/, 'hauptbahnhof');
		}

		return result
		.join(' ')

		// special chars
		.replace(/[^\w]|_/g, ' ')
		.replace(/\s{2,}/g, ' ')
		.trim();
	},



	stringifyId: function (id) {
		return id + '';
	},

	parseId: parseInt

};
<commit_msg>`src/locations/stations/index`: improved `tokenize`<commit_after>module.exports = {

	notes:		require('./notes'),



	tokenize: function (name) {
		var result = name
		.toLowerCase()

		// German umlauts
		.replace(/ß/g, 'ss')
		.replace(/ä/g, 'ae')
		.replace(/ö/g, 'oe')
		.replace(/ü/g, 'ue')

		// expand abbreviations
		.replace(/\(pl\)/g, 'polen')
		.replace(/\(berlin\)/g, 'berlin')
		.replace(/\(bln\)/g, 'berlin')
		.replace(/^s\s/, 'sbahn ')
		.replace(/^u\s/, 'ubahn ')
		.replace(/^s\+u\s/, 'sbahn ubahn ')
		.replace(/str\./, 'strasse')
		.replace(/bhf\./, 'bahnhof')
		.replace(/hbf\./, 'hauptbahnhof')

		.split(/[\s\/\(\)\-,\.]+/);

		for (var i = 0; i < result.length; i++) {
			if (result[i].length === 0){
				result.splice(i--, 1);
				continue;
			}
			result[i] = result[i].trim()

			// expand abbreviations
			.replace(/str\.?$/, 'strasse')
			.replace(/bhf\.?$/, 'bahnhof')
			.replace(/hbf\.?$/, 'hauptbahnhof');
		}

		return result.join(' ')
		.replace(/[^\w\s]|_/g, '');   // special chars
	},



	stringifyId: function (id) {
		return id + '';
	},

	parseId: parseInt

};

{'fraction_non_alphanumeric': 0.26961770623742454, 'fraction_numerical': 0.0025150905432595573, 'mean_word_length': 2.477272727272727, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1056102', 'n_tokens_mistral': 952, 'n_tokens_neox': 828, 'n_words': 145}