<commit_before>
from __future__ import print_function
import os
import sys
import tempfile
import cffi


DIRNAME = os.path.dirname(__file__)
PATH = os.path.join(DIRNAME, "src")

TMPDIR = os.path.join(tempfile.gettempdir(), "__pycache__")


def wrap_lib(fptype):

    with open(os.path.join(PATH, "libtupan.h"), "r") as fobj:
        src = fobj.read()
        source = "typedef {} REAL, *pREAL;".format(fptype) + src

    ffi = cffi.FFI()

    ffi.cdef(source)

    define_macros = []
    modulename = "libTupanSPcffi"
    if fptype == "double":
        define_macros.append(("DOUBLE", 1))
        modulename = modulename.replace("SP", "DP")

    clib = ffi.verify("""
    #include<common.h>
    #include<libtupan.h>
    """,
                      tmpdir=TMPDIR,
                      define_macros=define_macros,
                      include_dirs=[PATH],
                      libraries=['m'],
                      sources=[os.path.join(PATH, 'libtupan.c')],
                      force_generic_engine=hasattr(sys, '_force_generic_engine_'),
                      modulename=modulename,
                     )
    return ffi, clib

try:
    ffi_sp, clib_sp = wrap_lib("float")
    ffi_dp, clib_dp = wrap_lib("double")
except Exception as exc:
    print(str(exc), file=sys.stderr)


def get_extensions():
    return [ffi_sp.verifier.get_extension(), ffi_dp.verifier.get_extension()]
<commit_msg>try to build the docs on readthedocs (IV)<commit_after>
from __future__ import print_function
import os
import sys
import tempfile
import cffi


DIRNAME = os.path.dirname(__file__)
PATH = os.path.join(DIRNAME, "src")

TMPDIR = os.path.join(tempfile.gettempdir(), "__pycache__")


def wrap_lib(fptype):

    with open(os.path.join(PATH, "libtupan.h"), "r") as fobj:
        src = fobj.read()
        source = "typedef {} REAL, *pREAL;".format(fptype) + src

    ffi = cffi.FFI()

    ffi.cdef(source)

    define_macros = []
    modulename = "libTupanSPcffi"
    if fptype == "double":
        define_macros.append(("DOUBLE", 1))
        modulename = modulename.replace("SP", "DP")

    clib = ffi.verify("""
    #include<common.h>
    #include<libtupan.h>
    """,
                      tmpdir=TMPDIR,
                      define_macros=define_macros,
                      include_dirs=[PATH],
                      libraries=['m'],
                      sources=[os.path.join(PATH, 'libtupan.c')],
                      force_generic_engine=hasattr(sys, '_force_generic_engine_'),
                      modulename=modulename,
                     )
    return ffi, clib

try:
    ffi_sp, clib_sp = wrap_lib("float")
    ffi_dp, clib_dp = wrap_lib("double")
except Exception as exc:
    print(str(exc), file=sys.stderr)
    ffi_sp, clib_sp = None, None
    ffi_dp, clib_dp = None, None


def get_extensions():
    return [ffi_sp.verifier.get_extension(), ffi_dp.verifier.get_extension()]

{'fraction_non_alphanumeric': 0.13097713097713098, 'fraction_numerical': 0.000693000693000693, 'mean_word_length': 2.420616113744076, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1514114', 'n_tokens_mistral': 1029, 'n_tokens_neox': 995, 'n_words': 183}