<commit_before>import sys
import os
import copy
import distutils
import distutils.sysconfig
import re
import warnings

from subprocess \
    import \
        Popen, PIPE, STDOUT

from yaku.task_manager \
    import \
        topo_sort, build_dag, \
        CompiledTaskGen, set_extension_hook
from yaku.sysconfig \
    import \
        get_configuration, detect_distutils_cc
from yaku.compiled_fun \
    import \
        compile_fun
from yaku.task \
    import \
        task_factory
from yaku.utils \
    import \
        ensure_dir
from yaku.environment \
    import \
        Environment
from yaku.conftests \
    import \
        check_compiler, check_header
from yaku.tools.ctasks \
    import \
        apply_define
from yaku.scheduler \
    import \
        run_tasks
from yaku.conf \
    import \
        with_conf_blddir, create_file, write_log
from yaku.errors \
    import \
        TaskRunFailure
import yaku.tools

pylink, pylink_vars = compile_fun("pylink", "${PYEXT_SHLINK} ${PYEXT_LINK_TGT_F}${TGT[0].abspath()} ${PYEXT_LINK_SRC_F}${SRC} ${PYEXT_APP_LIBDIR} ${PYEXT_APP_LIBS} ${PYEXT_APP_FRAMEWORKS} ${PYEXT_SHLINKFLAGS}", False)

pycc, pycc_vars = compile_fun("pycc", "${PYEXT_CC} ${PYEXT_CFLAGS} ${PYEXT_INCPATH} ${PYEXT_CC_TGT_F}${TGT[0].abspath()} ${PYEXT_CC_SRC_F}${SRC}", False)

pycxx, pycxx_vars = compile_fun("pycxx", "${PYEXT_CXX} ${PYEXT_CXXFLAGS} ${PYEXT_INCPATH} ${PYEXT_CXX_TGT_F}${TGT[0].abspath()} ${PYEXT_CXX_SRC_F}${SRC}", False)

pycxxlink, pycxxlink_vars = compile_fun("pycxxlink", "${PYEXT_CXXSHLINK} ${PYEXT_LINK_TGT_F}${TGT[0].abspath()} ${PYEXT_LINK_SRC_F}${SRC} ${PYEXT_APP_LIBDIR} ${PYEXT_APP_LIBS} ${PYEXT_APP_FRAMEWORKS} ${PYEXT_SHLINKFLAGS}", False)

# pyext env <-> sysconfig env conversion

_SYS_TO_PYENV = {
        "PYEXT_SHCC": "CC",
        "PYEXT_CCSHARED": "CCSHARED",
        "PYEXT_SHLINK": "LDSHARED",
        "PYEXT_SUFFIX": "SO",
        "PYEXT_CFLAGS": "CFLAGS",
        "PYEXT_OPT": "OPT",
        "PYEXT_LIBDIR": "LIBDIR",
}

_PYENV_REQUIRED = [
        "LIBDIR_FMT",
        "LIBS",
        "LIB_FMT",
        "CPPPATH_FMT",
        "CC_TGT_F",
        "CC_SRC_F",
        "LINK_TGT_F",
        "LINK_SRC_F",
]

_SYS_TO_CCENV = {
        "CC": "CC",
        "SHCC": "CCSHARED",
        "SHLINK": "LDSHARED",
        "SO": "SO",
        "CFLAGS": "CFLAGS",
        "OPT": "OPT",
        "LIBDIR": "LIBDIR",
        "LIBDIR_FMT": "LIBDIR_FMT",
        "LIBS": "LIBS",
        "LIB_FMT": "LIB_FMT",
        "CPPPATH_FMT": "CPPPATH_FMT",
        "CC_TGT_F": "CC_TGT_F",
        "CC_SRC_F": "CC_SRC_F",
        "CXX": "CXX",
        "CXXSHLINK": "CXXSHLINK",
}

def setup_pyext_env(ctx, cc_type="default", use_distutils=True):
    pyenv = Environment()
    if use_distutils:
        if cc_type == "default":
            dist_env = get_configuration()
        else:
            dist_env = get_configuration(cc_type)
        for name, value in dist_env.items():
            pyenv["PYEXT_%s" % name] = value
        pyenv["PYEXT_FMT"] = "%%s%s" % dist_env["SO"]
        pyenv["PYEXT_CFLAGS"] = pyenv["PYEXT_BASE_CFLAGS"] + \
                pyenv["PYEXT_OPT"] + \
                pyenv["PYEXT_SHARED"]
        pyenv["PYEXT_SHLINKFLAGS"] = dist_env["LDFLAGS"]
    else:
        old_env = ctx.env
        ctx.env = Environment()
        cc_env = None
        sys.path.insert(0, os.path.dirname(yaku.tools.__file__))
        try:
            try:
                mod = __import__(cc_type)
                mod.setup(ctx)
            except ImportError:
                raise RuntimeError("No tool %s is available (import failed)" \
                                % cc_type)
            cc_env = ctx.env
        finally:
            sys.path.pop(0)
            ctx.env = old_env
        pyenv["PYEXT_CC"] = cc_env["CC"]
        pyenv["PYEXT_CFLAGS"] = cc_env["CFLAGS"]
        pyenv["PYEXT_LIBDIR"] = cc_env["LIBDIR"]
        pyenv["PYEXT_LIBS"] = cc_env["LIBS"]
        pyenv["PYEXT_FMT"] = "%s.so"
        pyenv["PYEXT_SHLINK"] = cc_env["MODLINK"]
        pyenv["PYEXT_SHLINKFLAGS"] = cc_env["MODLINKFLAGS"]
        pyenv.append("PYEXT_CPPPATH", distutils.sysconfig.get_python_inc(), create=True)

    return pyenv<commit_msg>ENH: add required cpppath/libdir for pyext in non-distutils mode.<commit_after>import sys
import os
import copy
import distutils
import distutils.sysconfig
import re
import warnings

from subprocess \
    import \
        Popen, PIPE, STDOUT

from yaku.task_manager \
    import \
        topo_sort, build_dag, \
        CompiledTaskGen, set_extension_hook
from yaku.sysconfig \
    import \
        get_configuration, detect_distutils_cc
from yaku.compiled_fun \
    import \
        compile_fun
from yaku.task \
    import \
        task_factory
from yaku.utils \
    import \
        ensure_dir
from yaku.environment \
    import \
        Environment
from yaku.conftests \
    import \
        check_compiler, check_header
from yaku.tools.ctasks \
    import \
        apply_define
from yaku.scheduler \
    import \
        run_tasks
from yaku.conf \
    import \
        with_conf_blddir, create_file, write_log
from yaku.errors \
    import \
        TaskRunFailure
import yaku.tools

pylink, pylink_vars = compile_fun("pylink", "${PYEXT_SHLINK} ${PYEXT_LINK_TGT_F}${TGT[0].abspath()} ${PYEXT_LINK_SRC_F}${SRC} ${PYEXT_APP_LIBDIR} ${PYEXT_APP_LIBS} ${PYEXT_APP_FRAMEWORKS} ${PYEXT_SHLINKFLAGS}", False)

pycc, pycc_vars = compile_fun("pycc", "${PYEXT_CC} ${PYEXT_CFLAGS} ${PYEXT_INCPATH} ${PYEXT_CC_TGT_F}${TGT[0].abspath()} ${PYEXT_CC_SRC_F}${SRC}", False)

pycxx, pycxx_vars = compile_fun("pycxx", "${PYEXT_CXX} ${PYEXT_CXXFLAGS} ${PYEXT_INCPATH} ${PYEXT_CXX_TGT_F}${TGT[0].abspath()} ${PYEXT_CXX_SRC_F}${SRC}", False)

pycxxlink, pycxxlink_vars = compile_fun("pycxxlink", "${PYEXT_CXXSHLINK} ${PYEXT_LINK_TGT_F}${TGT[0].abspath()} ${PYEXT_LINK_SRC_F}${SRC} ${PYEXT_APP_LIBDIR} ${PYEXT_APP_LIBS} ${PYEXT_APP_FRAMEWORKS} ${PYEXT_SHLINKFLAGS}", False)

# pyext env <-> sysconfig env conversion

_SYS_TO_PYENV = {
        "PYEXT_SHCC": "CC",
        "PYEXT_CCSHARED": "CCSHARED",
        "PYEXT_SHLINK": "LDSHARED",
        "PYEXT_SUFFIX": "SO",
        "PYEXT_CFLAGS": "CFLAGS",
        "PYEXT_OPT": "OPT",
        "PYEXT_LIBDIR": "LIBDIR",
}

_PYENV_REQUIRED = [
        "LIBDIR_FMT",
        "LIBS",
        "LIB_FMT",
        "CPPPATH_FMT",
        "CC_TGT_F",
        "CC_SRC_F",
        "LINK_TGT_F",
        "LINK_SRC_F",
]

_SYS_TO_CCENV = {
        "CC": "CC",
        "SHCC": "CCSHARED",
        "SHLINK": "LDSHARED",
        "SO": "SO",
        "CFLAGS": "CFLAGS",
        "OPT": "OPT",
        "LIBDIR": "LIBDIR",
        "LIBDIR_FMT": "LIBDIR_FMT",
        "LIBS": "LIBS",
        "LIB_FMT": "LIB_FMT",
        "CPPPATH_FMT": "CPPPATH_FMT",
        "CC_TGT_F": "CC_TGT_F",
        "CC_SRC_F": "CC_SRC_F",
        "CXX": "CXX",
        "CXXSHLINK": "CXXSHLINK",
}

def setup_pyext_env(ctx, cc_type="default", use_distutils=True):
    pyenv = Environment()
    if use_distutils:
        if cc_type == "default":
            dist_env = get_configuration()
        else:
            dist_env = get_configuration(cc_type)
        for name, value in dist_env.items():
            pyenv["PYEXT_%s" % name] = value
        pyenv["PYEXT_FMT"] = "%%s%s" % dist_env["SO"]
        pyenv["PYEXT_CFLAGS"] = pyenv["PYEXT_BASE_CFLAGS"] + \
                pyenv["PYEXT_OPT"] + \
                pyenv["PYEXT_SHARED"]
        pyenv["PYEXT_SHLINKFLAGS"] = dist_env["LDFLAGS"]
    else:
        old_env = ctx.env
        ctx.env = Environment()
        cc_env = None
        sys.path.insert(0, os.path.dirname(yaku.tools.__file__))
        try:
            try:
                mod = __import__(cc_type)
                mod.setup(ctx)
            except ImportError:
                raise RuntimeError("No tool %s is available (import failed)" \
                                % cc_type)
            cc_env = ctx.env
        finally:
            sys.path.pop(0)
            ctx.env = old_env
        pyenv["PYEXT_CC"] = cc_env["CC"]
        pyenv["PYEXT_CFLAGS"] = cc_env["CFLAGS"]
        pyenv["PYEXT_LIBDIR"] = cc_env["LIBDIR"]
        pyenv["PYEXT_LIBS"] = cc_env["LIBS"]
        pyenv["PYEXT_FMT"] = "%s.so"
        pyenv["PYEXT_SHLINK"] = cc_env["MODLINK"]
        pyenv["PYEXT_SHLINKFLAGS"] = cc_env["MODLINKFLAGS"]
        pyenv["PYEXT_CPPPATH"] = cc_env["CPPPATH"]
        pyenv.append("PYEXT_CPPPATH", distutils.sysconfig.get_python_inc(), create=True)
        if sys.platform == "win32":
            pyenv.append("PYEXT_LIBDIR", os.path.join(sys.exec_prefix, "libs"))

    return pyenv
{'fraction_non_alphanumeric': 0.13889212827988337, 'fraction_numerical': 0.0016326530612244899, 'mean_word_length': 2.559983395599834, 'pattern_counts': {'":': 47, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2443357', 'n_tokens_mistral': 3180, 'n_tokens_neox': 3169, 'n_words': 527}