<commit_before>import configparser
import pkg_resources
import os
import sys

config = configparser.ConfigParser()

# if there is a local config.ini file use that
if os.path.exists('config.ini'):
    config.read('config.ini')
# otherwise use the default config.ini file
else:
    config.read(pkg_resources.resource_filename('nalaf.data', 'default_config.ini'))

def print_err(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

print_verbose = print_err if config.getboolean('print', 'verbose', fallback=False) else lambda *a, **k: None
print_debug = print_err if config.getboolean('print', 'debug', fallback=False) else lambda *a, **k: None<commit_msg>add print_warning: always print<commit_after>import configparser
import pkg_resources
import os
import sys

config = configparser.ConfigParser()

# if there is a local config.ini file use that
if os.path.exists('config.ini'):
    config.read('config.ini')
# otherwise use the default config.ini file
else:
    config.read(pkg_resources.resource_filename('nalaf.data', 'default_config.ini'))

def _print_err(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

print_warning = _print_err
print_verbose = _print_err if config.getboolean('print', 'verbose', fallback=False) else lambda *a, **k: None
print_debug = _print_err if config.getboolean('print', 'debug', fallback=False) else lambda *a, **k: None
{'fraction_non_alphanumeric': 0.12790697674418605, 'fraction_numerical': 0.0, 'mean_word_length': 6.061538461538461, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3051644', 'n_tokens_mistral': 423, 'n_tokens_neox': 420, 'n_words': 122}