<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import division

import numpy as np
from numpy.testing import assert_equal, assert_almost_equal, assert_allclose
from ...tests.helper import pytest

from .. import funcs
from ...utils.misc import NumpyRNGContext

try:
    from scipy import stats  # used in testing
    from scipy.integrate import quad  # used in testing
    from scipy.special import erfinv, betainc, betaincinv  # used in funcs
except ImportError:
    HAS_SCIPY = False
else:
    HAS_SCIPY = True
<commit_msg>Fix compatibility with Numpy 1.4.1<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import division

import numpy as np
from numpy.testing import assert_equal, assert_almost_equal
from ...tests.helper import pytest
from ...tests.compat import assert_allclose

from .. import funcs
from ...utils.misc import NumpyRNGContext

try:
    from scipy import stats  # used in testing
    from scipy.integrate import quad  # used in testing
    from scipy.special import erfinv, betainc, betaincinv  # used in funcs
except ImportError:
    HAS_SCIPY = False
else:
    HAS_SCIPY = True

{'fraction_non_alphanumeric': 0.057885906040268456, 'fraction_numerical': 0.0041946308724832215, 'mean_word_length': 3.889344262295082, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5504479', 'n_tokens_mistral': 373, 'n_tokens_neox': 361, 'n_words': 139}