<commit_before>
# Configures LLVM dependency
#
# This function handles everything needed to setup LLVM project.
# By default it downloads and builds LLVM from source.
# In case LLVM_DIR variable is set it tries to use the pointed pre-built
# LLVM package. LLVM_DIR should point LLVM's shared cmake files to be used
# by find_package(... CONFIG) function.
#
# Creates a target representing all required LLVM libraries and include path.
function(configure_llvm_project TARGET_NAME)
    if (LLVM_DIR)
        find_package(LLVM REQUIRED CONFIG)
        llvm_map_components_to_libnames(LIBS mcjit ipo x86codegen)

        # To create a fake imported library later on we need to know the
        # location of some library
        list(GET LIBS 0 MAIN_LIB)
        get_property(CONFIGS TARGET ${MAIN_LIB} PROPERTY IMPORTED_CONFIGURATIONS)
        list(GET CONFIGS 0 CONFIG)  # Just get the first one. Usually there is only one.
        if (CONFIG)
            get_property(MAIN_LIB TARGET ${MAIN_LIB} PROPERTY IMPORTED_LOCATION_${CONFIG})
        else()
            set(CONFIG Unknown)
            get_property(MAIN_LIB TARGET ${MAIN_LIB} PROPERTY IMPORTED_LOCATION)
        endif()
        message(STATUS "LLVM ${LLVM_VERSION} (${CONFIG}; ${LLVM_ENABLE_ASSERTIONS}; ${LLVM_DIR})")
        if (NOT EXISTS ${MAIN_LIB})
            # Add some diagnostics to detect issues before building.
            message(FATAL_ERROR "LLVM library not found: ${MAIN_LIB}")
        endif()
    else()
        # List of required LLVM libs.
        # Generated with `llvm-config --libs mcjit ipo x86codegen`
        # Only used here locally to setup the "llvm-libs" imported target
        set(LIBS
            LLVMMCJIT
            LLVMX86CodeGen LLVMX86Desc LLVMX86Info LLVMMCDisassembler LLVMX86AsmPrinter
            LLVMX86Utils LLVMSelectionDAG LLVMAsmPrinter LLVMCodeGen
            LLVMInstrumentation LLVMBitWriter LLVMipo LLVMVectorize LLVMScalarOpts
            LLVMProfileData LLVMIRReader LLVMAsmParser LLVMInstCombine
            LLVMTransformUtils LLVMExecutionEngine LLVMTarget LLVMAnalysis
            LLVMRuntimeDyld LLVMObject LLVMMCParser LLVMBitReader LLVMMC
            LLVMCore LLVMSupport
        )

        # System libs that LLVM depend on.
        # See `llvm-config --system-libs`
        if (APPLE)
            set(SYSTEM_LIBS pthread z m curses)
        elseif (UNIX)
            set(SYSTEM_LIBS pthread z m tinfo dl)
        endif()

        include(ExternalProject)
        ExternalProject_Add(llvm-project
            PREFIX llvm
            BINARY_DIR llvm  # Build directly to install dir to avoid copy.
            SOURCE_DIR llvm/src/llvm
            URL http://llvm.org/releases/3.8.0/llvm-3.8.0.src.tar.xz
            URL_HASH SHA256=555b028e9ee0f6445ff8f949ea10e9cd8be0d084840e21fbbe1d31d51fc06e46
            CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
                       -DLLVM_TARGETS_TO_BUILD=X86
                       -DLLVM_INCLUDE_TOOLS=OFF -DLLVM_INCLUDE_EXAMPLES=OFF
                       -DLLVM_INCLUDE_TESTS=OFF
            BUILD_COMMAND   cmake --build <BINARY_DIR> --config Release
            INSTALL_COMMAND cmake --build <BINARY_DIR> --config Release --target install
            EXCLUDE_FROM_ALL TRUE
        )

        ExternalProject_Get_Property(llvm-project INSTALL_DIR)
        set(LLVM_LIBRARY_DIRS ${INSTALL_DIR}/lib)
        set(LLVM_INCLUDE_DIRS ${INSTALL_DIR}/include)
        file(MAKE_DIRECTORY ${LLVM_INCLUDE_DIRS})  # Must exists.

        foreach(LIB ${LIBS})<commit_msg>LLVM: Disable support for unused features to reduce system deps<commit_after>
# Configures LLVM dependency
#
# This function handles everything needed to setup LLVM project.
# By default it downloads and builds LLVM from source.
# In case LLVM_DIR variable is set it tries to use the pointed pre-built
# LLVM package. LLVM_DIR should point LLVM's shared cmake files to be used
# by find_package(... CONFIG) function.
#
# Creates a target representing all required LLVM libraries and include path.
function(configure_llvm_project TARGET_NAME)
    if (LLVM_DIR)
        find_package(LLVM REQUIRED CONFIG)
        llvm_map_components_to_libnames(LIBS mcjit ipo x86codegen)

        # To create a fake imported library later on we need to know the
        # location of some library
        list(GET LIBS 0 MAIN_LIB)
        get_property(CONFIGS TARGET ${MAIN_LIB} PROPERTY IMPORTED_CONFIGURATIONS)
        list(GET CONFIGS 0 CONFIG)  # Just get the first one. Usually there is only one.
        if (CONFIG)
            get_property(MAIN_LIB TARGET ${MAIN_LIB} PROPERTY IMPORTED_LOCATION_${CONFIG})
        else()
            set(CONFIG Unknown)
            get_property(MAIN_LIB TARGET ${MAIN_LIB} PROPERTY IMPORTED_LOCATION)
        endif()
        message(STATUS "LLVM ${LLVM_VERSION} (${CONFIG}; ${LLVM_ENABLE_ASSERTIONS}; ${LLVM_DIR})")
        if (NOT EXISTS ${MAIN_LIB})
            # Add some diagnostics to detect issues before building.
            message(FATAL_ERROR "LLVM library not found: ${MAIN_LIB}")
        endif()
    else()
        # List of required LLVM libs.
        # Generated with `llvm-config --libs mcjit ipo x86codegen`
        # Only used here locally to setup the "llvm-libs" imported target
        set(LIBS
            LLVMMCJIT
            LLVMX86CodeGen LLVMX86Desc LLVMX86Info LLVMMCDisassembler LLVMX86AsmPrinter
            LLVMX86Utils LLVMSelectionDAG LLVMAsmPrinter LLVMCodeGen
            LLVMInstrumentation LLVMBitWriter LLVMipo LLVMVectorize LLVMScalarOpts
            LLVMProfileData LLVMIRReader LLVMAsmParser LLVMInstCombine
            LLVMTransformUtils LLVMExecutionEngine LLVMTarget LLVMAnalysis
            LLVMRuntimeDyld LLVMObject LLVMMCParser LLVMBitReader LLVMMC
            LLVMCore LLVMSupport
        )

        # System libs that LLVM depend on.
        # See `llvm-config --system-libs`
        if (APPLE)
            set(SYSTEM_LIBS pthread)
        elseif (UNIX)
            set(SYSTEM_LIBS pthread dl)
        endif()

        include(ExternalProject)
        ExternalProject_Add(llvm-project
            PREFIX llvm
            BINARY_DIR llvm  # Build directly to install dir to avoid copy.
            SOURCE_DIR llvm/src/llvm
            URL http://llvm.org/releases/3.8.0/llvm-3.8.0.src.tar.xz
            URL_HASH SHA256=555b028e9ee0f6445ff8f949ea10e9cd8be0d084840e21fbbe1d31d51fc06e46
            CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
                       -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
                       -DLLVM_ENABLE_TERMINFO=OFF  # Disable terminal color support
                       -DLLVM_ENABLE_ZLIB=OFF      # Disable compression support -- not needed at all
                       -DLLVM_TARGETS_TO_BUILD=X86
                       -DLLVM_INCLUDE_TOOLS=OFF -DLLVM_INCLUDE_EXAMPLES=OFF
                       -DLLVM_INCLUDE_TESTS=OFF
            BUILD_COMMAND   cmake --build <BINARY_DIR> --config Release
            INSTALL_COMMAND cmake --build <BINARY_DIR> --config Release --target install
            EXCLUDE_FROM_ALL TRUE
        )

        ExternalProject_Get_Property(llvm-project INSTALL_DIR)
        set(LLVM_LIBRARY_DIRS ${INSTALL_DIR}/lib)
        set(LLVM_INCLUDE_DIRS ${INSTALL_DIR}/include)
        file(MAKE_DIRECTORY ${LLVM_INCLUDE_DIRS})  # Must exists.

        foreach(LIB ${LIBS})
{'fraction_non_alphanumeric': 0.060816326530612246, 'fraction_numerical': 0.017687074829931974, 'mean_word_length': 2.529044647143543, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4070465', 'n_tokens_mistral': 2350, 'n_tokens_neox': 2184, 'n_words': 591}