<commit_before>from __future__ import (absolute_import, unicode_literals, division,
                        print_function)
import sys
import collections
import numbers
from six import string_types

import warnings
import numpy as np
import scipy

# If numba is installed, import jit. Otherwise, define an empty decorator with
# the same name.

HAS_NUMBA = False
try:
    from numba import jit

    HAS_NUMBA = True
    from numba import njit, prange
except (ImportError, ModuleNotFoundError):
    warnings.warn("Numba not installed. Faking it")

    class jit(object):
        def __init__(self, *args, **kwargs):
            pass

        def __call__(self, func):
            def wrapped_f(*args, **kwargs):
                return func(*args, **kwargs)

            return wrapped_f

    class njit(object):
        def __init__(self, *args, **kwargs):
            pass

        def __call__(self, func):
            def wrapped_f(*args, **kwargs):
                return func(*args, **kwargs)<commit_msg>ModuleNotFoundError not in Python 3.5<commit_after>from __future__ import (absolute_import, unicode_literals, division,
                        print_function)
import sys
import collections
import numbers
from six import string_types

import warnings
import numpy as np
import scipy

# If numba is installed, import jit. Otherwise, define an empty decorator with
# the same name.

HAS_NUMBA = False
try:
    from numba import jit

    HAS_NUMBA = True
    from numba import njit, prange
except ImportError:
    warnings.warn("Numba not installed. Faking it")

    class jit(object):
        def __init__(self, *args, **kwargs):
            pass

        def __call__(self, func):
            def wrapped_f(*args, **kwargs):
                return func(*args, **kwargs)

            return wrapped_f

    class njit(object):
        def __init__(self, *args, **kwargs):
            pass

        def __call__(self, func):
            def wrapped_f(*args, **kwargs):
                return func(*args, **kwargs)
{'fraction_non_alphanumeric': 0.08230044620723848, 'fraction_numerical': 0.0009915716410510659, 'mean_word_length': 2.5218150087260036, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1631305', 'n_tokens_mistral': 571, 'n_tokens_neox': 552, 'n_words': 194}