<commit_before>from __future__ import absolute_import
import copy


_STR_TYPES = [str, bytes]
try:
	_STR_TYPES.append(unicode) #pylint: disable=undefined-variable
except NameError:
	pass


class ConfigError(Exception):
	pass


def _merge_defaults(defaults, confdict):
	for key, value in defaults.items():
		if key not in confdict:
			confdict[key] = value
		elif isinstance(value, dict) and isinstance(confdict[key], dict):
			_merge_defaults(value, confdict[key])


def _validate_value(value, valid_type, key):
	if valid_type is str and not isinstance(value, str):
		for alt_type in _STR_TYPES:
			if isinstance(value, alt_type):
				valid_type = alt_type
				break

	if valid_type is not None and not isinstance(value, valid_type):
		msg = 'Invalid configuration value for {} - expected {}, got {}'.format(
			key, valid_type, type(value))
		raise ConfigError(msg)


def _validate_dict(confdict, types, prefix=None):
	for key, value in confdict.items():
		if key not in types:
			continue

		pkey = ('%s[%s]' % (prefix, key)) if prefix else key

		if isinstance(value, dict) and isinstance(types[key], dict):
			_validate_dict(value, types[key], pkey)
		else:
			_validate_value(value, types[key], pkey)


def _str_to_type(value, valid_type):
	value_l = value.lower()
	if value_l == 'false':
		value = False
	elif value_l == 'true':
		value = True
	elif value_l == 'null' or value_l == 'none':
		value = None
	elif valid_type is int or valid_type is float:
		try:
			value = valid_type(value)
		except ValueError:
			pass

	if valid_type is list:
		value = [v.strip() for v in str(value).split(',')]
	elif valid_type is dict:
		pairs = [v.strip().split(':') for v in str(value).split(',')]
		value = {}
		for key, value in pairs:
			value[key.strip()] = value.strip()
<commit_msg>work on casting strings to bools<commit_after>from __future__ import absolute_import
import copy


_STR_TYPES = [str, bytes]
try:
	_STR_TYPES.append(unicode) #pylint: disable=undefined-variable
except NameError:
	pass


class ConfigError(Exception):
	pass


def _merge_defaults(defaults, confdict):
	for key, value in defaults.items():
		if key not in confdict:
			confdict[key] = value
		elif isinstance(value, dict) and isinstance(confdict[key], dict):
			_merge_defaults(value, confdict[key])


def _validate_value(value, valid_type, key):
	if valid_type is str and not isinstance(value, str):
		for alt_type in _STR_TYPES:
			if isinstance(value, alt_type):
				valid_type = alt_type
				break

	if valid_type is not None and not isinstance(value, valid_type):
		msg = 'Invalid configuration value for {} - expected {}, got {}'.format(
			key, valid_type, type(value))
		raise ConfigError(msg)


def _validate_dict(confdict, types, prefix=None):
	for key, value in confdict.items():
		if key not in types:
			continue

		pkey = ('%s[%s]' % (prefix, key)) if prefix else key

		if isinstance(value, dict) and isinstance(types[key], dict):
			_validate_dict(value, types[key], pkey)
		else:
			_validate_value(value, types[key], pkey)


def _str_to_type(value, valid_type):
	value_l = value.lower()
	if valid_type is int or valid_type is float:
		try:
			value = valid_type(value)
		except ValueError:
			pass
	elif valid_type is bool:
		if value_l in ('1', 'true', 'yes'):
			value = True
		elif value_l in ('', '0', 'false', 'no'):
			value = False
	elif valid_type is list:
		value = [v.strip() for v in str(value).split(',')]
	elif valid_type is dict:
		pairs = [v.strip().split(':') for v in str(value).split(',')]
		value = {}
		for key, value in pairs:
			value[key.strip()] = value.strip()

{'fraction_non_alphanumeric': 0.12339475153545505, 'fraction_numerical': 0.0005583472920156337, 'mean_word_length': 4.420574886535552, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1404331', 'n_tokens_mistral': 1321, 'n_tokens_neox': 1218, 'n_words': 381}