<commit_before>#include "Reader.h"
#include "Config.h"

#include "../../llvm_LLVMContext.h"
#include "../../llvm_Module.h"
#include "../../llvm_Linker.h"
#include "../../llvm_Function.h"
#include "../../llvm_CallingConv.h"
#include "../../llvm_AssemblyPrintModulePass.h"
#include "../../llvm_ValueSymbolTable.h"
#include "../../llvm_AnalysisVerifier.h"

#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/CodeGen/LinkAllAsmWriterComponents.h"
#include "llvm/CodeGen/LinkAllCodegenComponents.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/Interpreter.h"
#include "llvm/LinkAllPasses.h"
#include "llvm/PassManager.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/DynamicLibrary.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/Debug.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Transforms/Utils/Cloning.h"

#if D_LLVM_VERSION_MINOR <= 4
#include "llvm/Support/system_error.h"
#else
#include "llvm/Object/Error.h"
#endif
#if D_LLVM_VERSION_MINOR >= 3
#include "llvm/Support/SourceMgr.h"
#include "llvm/IRReader/IRReader.h"
#endif

#include "../../Serialise/Serialise.h"
#include "../../Utils/Utils.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

using namespace dale::ErrorInst;

static const char *bc_suffix    = ".bc";
static const char *bc_nm_suffix = "-nomacros.bc";
static const char *so_suffix    = ".so";

namespace dale
{
namespace Module
{
Reader::Reader(std::vector<const char *> *module_directory_paths,
               std::vector<std::string> *so_paths,
               std::vector<const char *> *include_directory_paths)
{
    cwd = getcwd(NULL, 0);
    this->module_directory_paths.push_back(cwd);
    std::copy(module_directory_paths->begin(),
              module_directory_paths->end(),
              back_inserter(this->module_directory_paths));
    this->module_directory_paths.push_back(DALE_MODULE_PATH);

    this->include_directory_paths.push_back("");
    this->include_directory_paths.push_back("./include/");
    std::copy(include_directory_paths->begin(),
              include_directory_paths->end(),
              back_inserter(this->include_directory_paths));
    std::string standard_include_path(DALE_INCLUDE_PATH);
    standard_include_path.append("/");
    this->include_directory_paths.push_back(standard_include_path.c_str());

    this->so_paths = so_paths;
}

Reader::~Reader()
{
    free(cwd);
}

llvm::Module *
Reader::loadModule(std::string *path)
{
#if D_LLVM_VERSION_MINOR <= 4
    llvm::OwningPtr<llvm::MemoryBuffer> buffer;
#endif

#if D_LLVM_VERSION_MINOR <= 3
    const llvm::sys::Path sys_path(*path);
    llvm::MemoryBuffer::getFileOrSTDIN(sys_path.c_str(), buffer);
#elif D_LLVM_VERSION_MINOR <= 4
    llvm::MemoryBuffer::getFileOrSTDIN(*path, buffer);
#else
    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> eo_buffer =
        llvm::MemoryBuffer::getFileOrSTDIN(*path);
    assert(!eo_buffer.getError() && "cannot load module");
    std::unique_ptr<llvm::MemoryBuffer> buffer = std::move(eo_buffer.get());
#endif

#if D_LLVM_VERSION_MINOR <= 4
    std::string error_msg;
    llvm::Module *module =
        llvm::getLazyBitcodeModule(buffer.get(),
                                   llvm::getGlobalContext(),
                                   &error_msg);
#else
    std::string error_msg;
    llvm::ErrorOr<llvm::Module *> eo_module =
        llvm::getLazyBitcodeModule(buffer.get(),
                                   llvm::getGlobalContext());
    llvm::Module *module = eo_module.get();
    if (!module) {
        error_msg = eo_module.getError().message();
    }
    buffer.release();
#endif

    assert(module && "cannot load module");

#if D_LLVM_VERSION_MINOR <= 4
    bool materialized = module->MaterializeAll(&error_msg);
#else
    std::error_code ec = module->materializeAllPermanently();
    bool materialized = (bool) ec;
    if (ec) {
        error_msg = ec.message();
    }
#endif
    assert(!materialized && "failed to materialize module");
    _unused(materialized);

    return module;
}

bool
Reader::addDynamicLibrary(const char *path, bool add_to_so_paths,
                          bool add_nm_to_so_paths)
{
    std::string error_msg;
    bool res =
        llvm::sys::DynamicLibrary::LoadLibraryPermanently(
            path,
            &error_msg
        );

    int len = strlen(path);
    bool is_so = (len >= 3) && !strcmp((path + len - 3), so_suffix);

    if (res) {
        /* If this is Darwin, and .so is at the end of path, try
         * replacing it with dylib.  Note that dylib is only for
         * internal Mac libraries, not for Dale modules. */
        if (!strcmp(SYSTEM_NAME, "Darwin") && is_so) {
            std::string dylib_path;
            dylib_path.append(path);
            dylib_path.erase((dylib_path.size() - 3), 3);
            dylib_path.append(so_suffix);
            return addDynamicLibrary(dylib_path.c_str(), add_to_so_paths,
                                     add_nm_to_so_paths);
        }
        assert(true && "unable to load library");
        return false;
    }

    if (add_nm_to_so_paths) {
        std::string nm_path;
        nm_path.append(path);
        nm_path.erase((nm_path.size() - 3), 3);
        nm_path.append("-nomacros.so");
        so_paths->push_back(nm_path);
    } else if (add_to_so_paths) {
        std::string m_path;
        m_path.append(path);
        so_paths->push_back(m_path);
    }

    return true;
}<commit_msg>remove attempted dylib handling<commit_after>#include "Reader.h"
#include "Config.h"

#include "../../llvm_LLVMContext.h"
#include "../../llvm_Module.h"
#include "../../llvm_Linker.h"
#include "../../llvm_Function.h"
#include "../../llvm_CallingConv.h"
#include "../../llvm_AssemblyPrintModulePass.h"
#include "../../llvm_ValueSymbolTable.h"
#include "../../llvm_AnalysisVerifier.h"

#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/Passes.h"
#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/CodeGen/LinkAllAsmWriterComponents.h"
#include "llvm/CodeGen/LinkAllCodegenComponents.h"
#include "llvm/ExecutionEngine/ExecutionEngine.h"
#include "llvm/ExecutionEngine/JIT.h"
#include "llvm/ExecutionEngine/Interpreter.h"
#include "llvm/LinkAllPasses.h"
#include "llvm/PassManager.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/DynamicLibrary.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/Debug.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include "llvm/Transforms/Utils/Cloning.h"

#if D_LLVM_VERSION_MINOR <= 4
#include "llvm/Support/system_error.h"
#else
#include "llvm/Object/Error.h"
#endif
#if D_LLVM_VERSION_MINOR >= 3
#include "llvm/Support/SourceMgr.h"
#include "llvm/IRReader/IRReader.h"
#endif

#include "../../Serialise/Serialise.h"
#include "../../Utils/Utils.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

using namespace dale::ErrorInst;

static const char *bc_suffix    = ".bc";
static const char *bc_nm_suffix = "-nomacros.bc";
static const char *so_suffix    = ".so";

namespace dale
{
namespace Module
{
Reader::Reader(std::vector<const char *> *module_directory_paths,
               std::vector<std::string> *so_paths,
               std::vector<const char *> *include_directory_paths)
{
    cwd = getcwd(NULL, 0);
    this->module_directory_paths.push_back(cwd);
    std::copy(module_directory_paths->begin(),
              module_directory_paths->end(),
              back_inserter(this->module_directory_paths));
    this->module_directory_paths.push_back(DALE_MODULE_PATH);

    this->include_directory_paths.push_back("");
    this->include_directory_paths.push_back("./include/");
    std::copy(include_directory_paths->begin(),
              include_directory_paths->end(),
              back_inserter(this->include_directory_paths));
    std::string standard_include_path(DALE_INCLUDE_PATH);
    standard_include_path.append("/");
    this->include_directory_paths.push_back(standard_include_path.c_str());

    this->so_paths = so_paths;
}

Reader::~Reader()
{
    free(cwd);
}

llvm::Module *
Reader::loadModule(std::string *path)
{
#if D_LLVM_VERSION_MINOR <= 4
    llvm::OwningPtr<llvm::MemoryBuffer> buffer;
#endif

#if D_LLVM_VERSION_MINOR <= 3
    const llvm::sys::Path sys_path(*path);
    llvm::MemoryBuffer::getFileOrSTDIN(sys_path.c_str(), buffer);
#elif D_LLVM_VERSION_MINOR <= 4
    llvm::MemoryBuffer::getFileOrSTDIN(*path, buffer);
#else
    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> eo_buffer =
        llvm::MemoryBuffer::getFileOrSTDIN(*path);
    assert(!eo_buffer.getError() && "cannot load module");
    std::unique_ptr<llvm::MemoryBuffer> buffer = std::move(eo_buffer.get());
#endif

#if D_LLVM_VERSION_MINOR <= 4
    std::string error_msg;
    llvm::Module *module =
        llvm::getLazyBitcodeModule(buffer.get(),
                                   llvm::getGlobalContext(),
                                   &error_msg);
#else
    std::string error_msg;
    llvm::ErrorOr<llvm::Module *> eo_module =
        llvm::getLazyBitcodeModule(buffer.get(),
                                   llvm::getGlobalContext());
    llvm::Module *module = eo_module.get();
    if (!module) {
        error_msg = eo_module.getError().message();
    }
    buffer.release();
#endif

    assert(module && "cannot load module");

#if D_LLVM_VERSION_MINOR <= 4
    bool materialized = module->MaterializeAll(&error_msg);
#else
    std::error_code ec = module->materializeAllPermanently();
    bool materialized = (bool) ec;
    if (ec) {
        error_msg = ec.message();
    }
#endif
    assert(!materialized && "failed to materialize module");
    _unused(materialized);

    return module;
}

bool
Reader::addDynamicLibrary(const char *path, bool add_to_so_paths,
                          bool add_nm_to_so_paths)
{
    std::string error_msg;
    bool res =
        llvm::sys::DynamicLibrary::LoadLibraryPermanently(
            path,
            &error_msg
        );

    if (res) {
        fprintf(stderr, "%s\n", error_msg.c_str());
        error("unable to load dynamic library", false);
        return false;
    }

    if (add_nm_to_so_paths) {
        std::string nm_path;
        nm_path.append(path);
        nm_path.erase((nm_path.size() - 3), 3);
        nm_path.append("-nomacros.so");
        so_paths->push_back(nm_path);
    } else if (add_to_so_paths) {
        std::string m_path;
        m_path.append(path);
        so_paths->push_back(m_path);
    }

    return true;
}
{'fraction_non_alphanumeric': 0.14801412180052956, 'fraction_numerical': 0.002118270079435128, 'mean_word_length': 3.503577106518283, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3290642', 'n_tokens_mistral': 3819, 'n_tokens_neox': 3782, 'n_words': 635}