<commit_before>include "config.pxi"
import os

cdef extern from 'dlfcn.h' nogil:
    void* dlopen(const char *filename, int flag)
    char *dlerror()
    void *dlsym(void *handle, const char *symbol)
    int dlclose(void *handle)

    unsigned int RTLD_LAZY
    unsigned int RTLD_NOW
    unsigned int RTLD_GLOBAL
    unsigned int RTLD_LOCAL
    unsigned int RTLD_NODELETE
    unsigned int RTLD_NOLOAD
    unsigned int RTLD_DEEPBIND

    unsigned int RTLD_DEFAULT
    long unsigned int RTLD_NEXT


cdef extern from "jni.h":
    int JNI_VERSION_1_6
    int JNI_OK
    jboolean JNI_FALSE
    ctypedef struct JavaVMInitArgs:
        jint version
        jint nOptions
        jboolean ignoreUnrecognized
        JavaVMOption *options
    ctypedef struct JavaVMOption:
        char *optionString
        void *extraInfo

cdef JNIEnv *_platform_default_env = NULL

cdef void create_jnienv() except *:
    cdef JavaVM* jvm
    cdef JavaVMInitArgs args
    cdef JavaVMOption *options
    cdef int ret
    cdef bytes py_bytes
    import jnius_config

    JAVA_HOME = os.environ['JAVA_HOME']
    if JAVA_HOME is None or JAVA_HOME == '':
       raise SystemError("JAVA_HOME is not set.")
    IF JNIUS_PYTHON3:
       lib_path = str_for_c(os.path.join(JAVA_HOME, JNIUS_LIB_SUFFIX.decode("utf-8")))
    ELSE:
       lib_path = str_for_c(os.path.join(JAVA_HOME, JNIUS_LIB_SUFFIX))

    cdef void *handle = dlopen(lib_path, RTLD_NOW | RTLD_GLOBAL)
    if handle == NULL:
        raise SystemError("Error calling dlopen({0}: {1}".format(lib_path, dlerror()))

    cdef void *jniCreateJVM = dlsym(handle, b"JNI_CreateJavaVM")

    if jniCreateJVM == NULL:
       raise SystemError("Error calling dlfcn for JNI_CreateJavaVM: {0}".format(dlerror()))
<commit_msg>fix python3 issue for decoding str (it might come from an updated Cython.). Closes #218<commit_after>include "config.pxi"
import os

cdef extern from 'dlfcn.h' nogil:
    void* dlopen(const char *filename, int flag)
    char *dlerror()
    void *dlsym(void *handle, const char *symbol)
    int dlclose(void *handle)

    unsigned int RTLD_LAZY
    unsigned int RTLD_NOW
    unsigned int RTLD_GLOBAL
    unsigned int RTLD_LOCAL
    unsigned int RTLD_NODELETE
    unsigned int RTLD_NOLOAD
    unsigned int RTLD_DEEPBIND

    unsigned int RTLD_DEFAULT
    long unsigned int RTLD_NEXT


cdef extern from "jni.h":
    int JNI_VERSION_1_6
    int JNI_OK
    jboolean JNI_FALSE
    ctypedef struct JavaVMInitArgs:
        jint version
        jint nOptions
        jboolean ignoreUnrecognized
        JavaVMOption *options
    ctypedef struct JavaVMOption:
        char *optionString
        void *extraInfo

cdef JNIEnv *_platform_default_env = NULL

cdef void create_jnienv() except *:
    cdef JavaVM* jvm
    cdef JavaVMInitArgs args
    cdef JavaVMOption *options
    cdef int ret
    cdef bytes py_bytes
    import jnius_config

    JAVA_HOME = os.environ['JAVA_HOME']
    if JAVA_HOME is None or JAVA_HOME == '':
        raise SystemError("JAVA_HOME is not set.")
    IF JNIUS_PYTHON3:
        try:
            jnius_lib_suffix = JNIUS_LIB_SUFFIX.decode("utf-8")
        except AttributeError:
            jnius_lib_suffix = JNIUS_LIB_SUFFIX
        lib_path = str_for_c(os.path.join(JAVA_HOME, jnius_lib_suffix))
    ELSE:
        lib_path = str_for_c(os.path.join(JAVA_HOME, JNIUS_LIB_SUFFIX))

    cdef void *handle = dlopen(lib_path, RTLD_NOW | RTLD_GLOBAL)
    if handle == NULL:
        raise SystemError("Error calling dlopen({0}: {1}".format(lib_path, dlerror()))

    cdef void *jniCreateJVM = dlsym(handle, b"JNI_CreateJavaVM")

    if jniCreateJVM == NULL:
       raise SystemError("Error calling dlfcn for JNI_CreateJavaVM: {0}".format(dlerror()))

{'fraction_non_alphanumeric': 0.07318390494193897, 'fraction_numerical': 0.004860923575479341, 'mean_word_length': 3.1947904869762174, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6856589', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1278, 'n_words': 339}