<commit_before># cython: language_level=2
from __future__ import absolute_import, division, print_function

import ctypes
from importlib import import_module
import os
import sys
import traceback

from cpython.module cimport PyImport_ImportModule
from cpython.object cimport PyObject
from cpython.ref cimport Py_INCREF, Py_DECREF
cdef extern from "Python.h":
    void Py_Initialize()
    void PyEval_SaveThread()

from libc.errno cimport errno
from libc.stdint cimport uintptr_t
from libc.stdlib cimport getenv, malloc
from libc.stdio cimport printf, snprintf
from libc.string cimport strerror, strlen
from posix.fcntl cimport open as c_open, O_APPEND, O_CREAT, O_WRONLY
from posix.stdlib cimport putenv
from posix.unistd cimport dup2, getpid, STDERR_FILENO, STDOUT_FILENO, write

import java
from java._vendor import six

from java.jni cimport *
from java.chaquopy cimport *
cdef extern from "chaquopy_java_extra.h":
    void PyInit_chaquopy_java() except *


cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved):
    return JNI_VERSION_1_6


# === com.chaquo.python.Python ================================================

cdef public void Java_com_chaquo_python_Python_startNative \
    (JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path):
    if getenv("CHAQUOPY_PROCESS_TYPE") == NULL:  # See jvm.pxi
        startNativeJava(env, j_platform, j_python_path)
    else:
        init_module(env)


# We're running in a Java process, so start the Python VM.
cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path):
    # Enable the following lines to help debug low-level startup failures. Requires
    # WRITE_EXTERNAL_STORAGE permission, and on API 23 and higher, this permission must have
    # been activated by the user. /sdcard is not guaranteed to be the correct path for
    # external storage, so this may have to be changed for some devices. Any failure in the
    # redirection process will be silent.
    # redirect(STDOUT_FILENO, "/sdcard/stdout.txt")
    # redirect(STDERR_FILENO, "/sdcard/stderr.txt")

    cdef const char *python_path
    if j_python_path != NULL:
        python_path = env[0].GetStringUTFChars(env, j_python_path, NULL)
        if python_path == NULL:
            throw_simple_exception(env, "GetStringUTFChars failed in startNativeJava")
            return
        try:
            if not set_path(env, python_path):
                return
            if not set_env(env, "CHAQUOPY_PROCESS_TYPE", "java"):  # See chaquopy.pyx
                return
        finally:  # Yes, this does compile to pure C, with the help of "goto".
            env[0].ReleaseStringUTFChars(env, j_python_path, python_path)

    Py_Initialize()  # Calls abort() on failure

    # All code run before Py_Initialize must compile to pure C. After Py_Initialize returns we
    # can start using the Python VM, but many things will still cause a native crash until our
    # module initialization function succeeds. In particular, global and built-in names won't
    # be bound, and "import" won't work either, even locally.
    if not init_module(env):
        return  # A Java exception should already have been thrown.

    cdef SavedException se = None
    cdef JavaVM *jvm = NULL
    try:
        ret = env[0].GetJavaVM(env, &jvm)
        if ret != 0:
             raise Exception(f"GetJavaVM failed: {ret}")
        set_jvm(jvm)
        check_license(j2p(env, LocalRef.create(env, j_platform)))
    except BaseException:
        se = SavedException()
    if se:
        se.throw(env)

    # We imported java.chaquopy above, which has called PyEval_InitThreads during its own
    # module intialization, so the GIL now exists and we have it. We must release the GIL
    # before we return to Java so that the methods below can be called from any thread.
    # (http://bugs.python.org/issue1720250)
    PyEval_SaveThread();


# WARNING: This function (specifically PyInit_chaquopy_java) will crash if called
# more than once.
cdef bint init_module(JNIEnv *env) with gil:
    try:
        PyInit_chaquopy_java()
        return True
    except BaseException:
        throw_simple_exception(env, format_exception().encode("utf-8"))
        return False


# This runs before Py_Initialize, so it must compile to pure C.
cdef public bint set_path(JNIEnv *env, const char *python_path):
    return set_env(env, "PYTHONPATH", python_path)


# The POSIX setenv function is not available on MSYS2.
# This runs before Py_Initialize, so it must compile to pure C.
cdef bint set_env(JNIEnv *env, const char *name, const char *value):
    cdef int putenvArgLen = strlen(name) + 1 + strlen(value) + 1
    cdef char *putenvArg = <char*>malloc(putenvArgLen)
    if snprintf(putenvArg, putenvArgLen, "%s=%s", name, value) != (putenvArgLen - 1):
        throw_simple_exception(env, "snprintf failed in set_env")
        return False

    # putenv takes ownership of the string passed to it.
    if putenv(putenvArg) != 0:
        throw_simple_exception(env, "putenv failed in set_env")
        return False

    return True


cdef int redirect(int stream_fd, char *filename):
    file_fd = c_open(filename, O_WRONLY|O_CREAT|O_APPEND)<commit_msg>Add runtime function to redirect native stdout and stderr<commit_after># cython: language_level=2
from __future__ import absolute_import, division, print_function

import ctypes
from importlib import import_module
import os
import sys
import traceback

from cpython.module cimport PyImport_ImportModule
from cpython.object cimport PyObject
from cpython.ref cimport Py_INCREF, Py_DECREF
cdef extern from "Python.h":
    void Py_Initialize()
    void PyEval_SaveThread()

from libc.errno cimport errno
from libc.stdint cimport uintptr_t
from libc.stdlib cimport getenv, malloc
from libc.stdio cimport printf, snprintf
from libc.string cimport strerror, strlen
from posix.fcntl cimport open as c_open, O_APPEND, O_CREAT, O_WRONLY
from posix.stdlib cimport putenv
from posix.unistd cimport dup2, getpid, STDERR_FILENO, STDOUT_FILENO, write

import java
from java._vendor import six

from java.jni cimport *
from java.chaquopy cimport *
cdef extern from "chaquopy_java_extra.h":
    void PyInit_chaquopy_java() except *


cdef public jint JNI_OnLoad(JavaVM *jvm, void *reserved):
    return JNI_VERSION_1_6


# === com.chaquo.python.Python ================================================

cdef public void Java_com_chaquo_python_Python_startNative \
    (JNIEnv *env, jobject klass, jobject j_platform, jobject j_python_path):
    if getenv("CHAQUOPY_PROCESS_TYPE") == NULL:  # See jvm.pxi
        startNativeJava(env, j_platform, j_python_path)
    else:
        init_module(env)


# We're running in a Java process, so start the Python VM.
cdef void startNativeJava(JNIEnv *env, jobject j_platform, jobject j_python_path):
    # Enable the following lines to help debug low-level startup failures. Requires
    # WRITE_EXTERNAL_STORAGE permission, and on API 23 and higher, this permission must have
    # been activated by the user. /sdcard is not guaranteed to be the correct path for
    # external storage, so this may have to be changed for some devices. Any failure in the
    # redirection process will be silent.
    # redirect(STDOUT_FILENO, "/sdcard/stdout.txt")
    # redirect(STDERR_FILENO, "/sdcard/stderr.txt")

    cdef const char *python_path
    if j_python_path != NULL:
        python_path = env[0].GetStringUTFChars(env, j_python_path, NULL)
        if python_path == NULL:
            throw_simple_exception(env, "GetStringUTFChars failed in startNativeJava")
            return
        try:
            if not set_path(env, python_path):
                return
            if not set_env(env, "CHAQUOPY_PROCESS_TYPE", "java"):  # See chaquopy.pyx
                return
        finally:  # Yes, this does compile to pure C, with the help of "goto".
            env[0].ReleaseStringUTFChars(env, j_python_path, python_path)

    Py_Initialize()  # Calls abort() on failure

    # All code run before Py_Initialize must compile to pure C. After Py_Initialize returns we
    # can start using the Python VM, but many things will still cause a native crash until our
    # module initialization function succeeds. In particular, global and built-in names won't
    # be bound, and "import" won't work either, even locally.
    if not init_module(env):
        return  # A Java exception should already have been thrown.

    cdef SavedException se = None
    cdef JavaVM *jvm = NULL
    try:
        ret = env[0].GetJavaVM(env, &jvm)
        if ret != 0:
             raise Exception(f"GetJavaVM failed: {ret}")
        set_jvm(jvm)
        check_license(j2p(env, LocalRef.create(env, j_platform)))
    except BaseException:
        se = SavedException()
    if se:
        se.throw(env)

    # We imported java.chaquopy above, which has called PyEval_InitThreads during its own
    # module intialization, so the GIL now exists and we have it. We must release the GIL
    # before we return to Java so that the methods below can be called from any thread.
    # (http://bugs.python.org/issue1720250)
    PyEval_SaveThread();


# WARNING: This function (specifically PyInit_chaquopy_java) will crash if called
# more than once.
cdef bint init_module(JNIEnv *env) with gil:
    try:
        PyInit_chaquopy_java()
        return True
    except BaseException:
        throw_simple_exception(env, format_exception().encode("utf-8"))
        return False


# This runs before Py_Initialize, so it must compile to pure C.
cdef public bint set_path(JNIEnv *env, const char *python_path):
    return set_env(env, "PYTHONPATH", python_path)


# The POSIX setenv function is not available on MSYS2.
# This runs before Py_Initialize, so it must compile to pure C.
cdef bint set_env(JNIEnv *env, const char *name, const char *value):
    cdef int putenvArgLen = strlen(name) + 1 + strlen(value) + 1
    cdef char *putenvArg = <char*>malloc(putenvArgLen)
    if snprintf(putenvArg, putenvArgLen, "%s=%s", name, value) != (putenvArgLen - 1):
        throw_simple_exception(env, "snprintf failed in set_env")
        return False

    # putenv takes ownership of the string passed to it.
    if putenv(putenvArg) != 0:
        throw_simple_exception(env, "putenv failed in set_env")
        return False

    return True


# For debugging packages which write to native stdout and stderr.
def redirect_streams():
    tmpdir = os.environ["TMPDIR"]
    redirect(STDOUT_FILENO, f"{tmpdir}/stdout.txt".encode())
    redirect(STDERR_FILENO, f"{tmpdir}/stderr.txt".encode())


cdef int redirect(int stream_fd, char *filename):
    file_fd = c_open(filename, O_WRONLY|O_CREAT|O_APPEND)
{'fraction_non_alphanumeric': 0.08331771321462043, 'fraction_numerical': 0.004498594189315839, 'mean_word_length': 3.5975872468763463, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4326225', 'n_tokens_mistral': 3342, 'n_tokens_neox': 3250, 'n_words': 1123}