<commit_before>try:
    import numpy
    import scipy
except ImportError:
    raise ImportError("py-sdm requires numpy and scipy to be installed")
    # Don't do this in the setup() requirements, because otherwise pip and
    # friends get too eager about updating numpy/scipy.

try:
    import pyflann
except ImportError:
    msg = """py-sdm requires the python bindings for FLANN. These are available
from http://people.cs.ubc.ca/~mariusm/flann. Make sure to compile them with
OpenMP support (ie using gcc rather than clang).

(If you think you have FLANN installed, try running
    python PREFIX/share/flann/python/setup.py install
to get the bindings installed in your current python environment.)"""
    raise ImportError(msg)

try:
    from setuptools import setup
    from setuptools.extension import Extension
except ImportError:
    from distutils.core import setup
    from distutils.extension import Extension

def cython_ext(name, **k):
    ret = []

    try:
        from Cython.Build import cythonize
    except ImportError:
        import os
        try:
            pyx_time = os.path.getmtime('sdm/{}.pyx'.format(name))
            c_time = os.path.getmtime('sdm/{}.c'.format(name))
            if pyx_time >= c_time:
                raise ValueError
        except (OSError, ValueError):
            msg = "{} extension needs to be compiled but cython isn't available"
            raise ImportError(msg.format(name))
    else:
        ret.extend(cythonize("sdm/{}.pyx".format(name)))
<commit_msg>make extension actually take extra args<commit_after>try:
    import numpy
    import scipy
except ImportError:
    raise ImportError("py-sdm requires numpy and scipy to be installed")
    # Don't do this in the setup() requirements, because otherwise pip and
    # friends get too eager about updating numpy/scipy.

try:
    import pyflann
except ImportError:
    msg = """py-sdm requires the python bindings for FLANN. These are available
from http://people.cs.ubc.ca/~mariusm/flann. Make sure to compile them with
OpenMP support (ie using gcc rather than clang).

(If you think you have FLANN installed, try running
    python PREFIX/share/flann/python/setup.py install
to get the bindings installed in your current python environment.)"""
    raise ImportError(msg)

try:
    from setuptools import setup
    from setuptools.extension import Extension
except ImportError:
    from distutils.core import setup
    from distutils.extension import Extension

def cython_ext(name, **k):
    ret = []

    try:
        from Cython.Build import cythonize
    except ImportError:
        import os
        try:
            pyx_time = os.path.getmtime('sdm/{}.pyx'.format(name))
            c_time = os.path.getmtime('sdm/{}.c'.format(name))
            if pyx_time >= c_time:
                raise ValueError
        except (OSError, ValueError):
            msg = "{} extension needs to be compiled but cython isn't available"
            raise ImportError(msg.format(name))
    else:
        cythonize("sdm/{}.pyx".format(name))

{'fraction_non_alphanumeric': 0.08579881656804733, 'fraction_numerical': 0.0, 'mean_word_length': 2.9062901155327343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2571027', 'n_tokens_mistral': 876, 'n_tokens_neox': 857, 'n_words': 303}