<commit_before>import os

try:    # pragma: no cover
    _range = xrange
except NameError:    # pragma: no cover
    _range = range


try:    # pragma: no cover
    _unich = unichr
except NameError:    # pragma: no cover
    _unich = chr

try:  # pragma: no cover
    import colorama
    colorama.init()
except ImportError:  # pragma: no cover
    colorama = None


def _is_utf(encoding):
    return ('U8' == encoding) or ('utf' in encoding) or ('UTF' in encoding)


def _supports_unicode(file):
    if not getattr(file, 'encoding', None):
        return False
    return _is_utf(file.encoding)


def _environ_cols_wrapper():  # pragma: no cover
    """
    Return a function which gets width and height of console
    (linux,osx,windows,cygwin).
    """
    import platform
    current_os = platform.system()
    _environ_cols = None
    if current_os == 'Windows':
        _environ_cols = _environ_cols_windows
        if _environ_cols is None:
            _environ_cols = _environ_cols_tput
    if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'):
        _environ_cols = _environ_cols_linux
    return _environ_cols


def _environ_cols_windows(fp):  # pragma: no cover
    try:
        from ctypes import windll, create_string_buffer<commit_msg>Solaris, FreeBSD ncols compatibility<commit_after>import os

try:    # pragma: no cover
    _range = xrange
except NameError:    # pragma: no cover
    _range = range


try:    # pragma: no cover
    _unich = unichr
except NameError:    # pragma: no cover
    _unich = chr

try:  # pragma: no cover
    import colorama
    colorama.init()
except ImportError:  # pragma: no cover
    colorama = None


def _is_utf(encoding):
    return ('U8' == encoding) or ('utf' in encoding) or ('UTF' in encoding)


def _supports_unicode(file):
    if not getattr(file, 'encoding', None):
        return False
    return _is_utf(file.encoding)


def _environ_cols_wrapper():  # pragma: no cover
    """
    Return a function which gets width and height of console
    (linux,osx,windows,cygwin).
    """
    import platform
    current_os = platform.system()
    _environ_cols = None
    if current_os == 'Windows':
        _environ_cols = _environ_cols_windows
        if _environ_cols is None:
            _environ_cols = _environ_cols_tput
    if current_os in ['Linux', 'Darwin', 'SunOS', 'FreeBSD'] or \
            current_os.startswith('CYGWIN'):
        _environ_cols = _environ_cols_linux
    return _environ_cols


def _environ_cols_windows(fp):  # pragma: no cover
    try:
        from ctypes import windll, create_string_buffer
{'fraction_non_alphanumeric': 0.08786127167630058, 'fraction_numerical': 0.0007707129094412332, 'mean_word_length': 3.043613707165109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7507547', 'n_tokens_mistral': 844, 'n_tokens_neox': 847, 'n_words': 246}