<commit_before>#!/usr/bin/env python

#
# file: common.py
#
# author: Copyright (C) 2017-2018 <NAME> http://www.distortec.com http://www.freddiechopin.info
#
# This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
# distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import re

def sanitize(string, pattern = '[^0-9A-Za-z-]', replacement = '_'):
	"""Sanitize and return provided string.

	* `string` is the string that will be sanitized
	* `pattern` is the pattern which will be replaced, default - `'[^0-9A-Za-z-]'`
	* `replacement` is the replacement string, default - `'_'`
	"""
	regex = re.compile(pattern)
	return regex.sub(replacement, string)<commit_msg>Convert sanitize() argument to string<commit_after>#!/usr/bin/env python

#
# file: common.py
#
# author: Copyright (C) 2017-2018 <NAME> http://www.distortec.com http://www.freddiechopin.info
#
# This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
# distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import re

def sanitize(string, pattern = '[^0-9A-Za-z-]', replacement = '_'):
	"""Sanitize and return provided string.

	* `string` is the string that will be sanitized
	* `pattern` is the pattern which will be replaced, default - `'[^0-9A-Za-z-]'`
	* `replacement` is the replacement string, default - `'_'`
	"""
	regex = re.compile(pattern)
	return regex.sub(replacement, str(string))
{'fraction_non_alphanumeric': 0.14481409001956946, 'fraction_numerical': 0.020874103065883887, 'mean_word_length': 3.9644012944983817, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7012260', 'n_tokens_mistral': 545, 'n_tokens_neox': 507, 'n_words': 175}