<commit_before>// This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

/** plugin_resource.cc
    <NAME>, 18 fevrier 2015
    Copyright (c) 2015 mldb.ai inc.  All rights reserved.

*/

#include "mldb/server/plugin_resource.h"

#include "mldb/ext/libgit2/include/git2.h"
#include "mldb/ext/libgit2/include/git2/clone.h"
#include "mldb/types/structure_description.h"
#include "mldb/vfs/filter_streams.h"
#include "mldb/utils/tmpdir.h"

using namespace std;

namespace fs = std::filesystem;


namespace MLDB {


/*****************************************************************************/
/* PLUGIN STRUCTURES                                                         */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(PackageElementSources);

PackageElementSourcesDescription::
PackageElementSourcesDescription()
{
    addField("main", &PackageElementSources::main,
             "source for the main element of the plugin");
    addField("routes", &PackageElementSources::routes,
             "source for the routes element of the plugin");
    addField("status", &PackageElementSources::status,
             "source for the status element of the plugin");
}


DEFINE_STRUCTURE_DESCRIPTION(PluginResource);

PluginResourceDescription::
PluginResourceDescription()
{
    addField("address", &PluginResource::address,
             "URI or location of script to run (use this parameter OR 'source')");
    PackageElementSources elem;
    addField("source", &PluginResource::source,
             "source of script to run (use this parameter OR 'address')", elem);
    addField("args", &PluginResource::args,
             "Arguments to pass to script");
}


DEFINE_STRUCTURE_DESCRIPTION(ScriptResource);

ScriptResourceDescription::
ScriptResourceDescription()
{
    addField("address", &ScriptResource::address,
             "URI or location of script to run (use this parameter OR 'source')");
    addField("source", &ScriptResource::source,
             "source of script to run (use this parameter OR 'address')");
    addField("args", &ScriptResource::args,
             "Arguments to pass to script");
    addField("writeSourceToFile", &ScriptResource::writeSourceToFile,
             "If passing in the source as a string, write it to disk to a temporary "
              "location as if it was downloaded remotely. This should usually not be "
              "necessary.", false);
}


DEFINE_STRUCTURE_DESCRIPTION(PluginVersion);

PluginVersionDescription::
PluginVersionDescription()
{
    addField("branch", &PluginVersion::branch,
             "branch currently checkout");
    addField("revision", &PluginVersion::revision,
             "revision currently checked out");
    addField("address", &PluginVersion::address,
             "address");
    addField("message", &PluginVersion::message,
             "message");
}

ScriptLanguage parseScriptLanguage(const std::string lang)
{
    if(lang == "python")        return PYTHON;
    if(lang == "javascript")    return JAVASCRIPT;
    throw MLDB::Exception("Unknown language");
}

/*****************************************************************************/
/* PLUGIN RESSOURCE                                                          */
/*****************************************************************************/
    
LoadedPluginResource::
LoadedPluginResource(ScriptLanguage lang, ScriptType type,
                     Utf8String pluginId,
                     const PluginResource & resource) :
    url(resource.address), args(resource.args), MLDB_ROOT("/mldb_data/")
{
    pluginLanguage = lang;
    scriptType = type;

    if(!url.empty() && !resource.source.empty())
        throw MLDB::Exception("Cannot specify both address and source for plugin");

    // if this is a plugin, we need to create a folder in the plugins folder
    // that we'll be keeping around
    if(scriptType == PLUGIN) {
        // if we are running inside of docker. TODO this check is a little flimsy
        if(fs::exists(MLDB_ROOT)) {
            fs::create_directories(MLDB_ROOT / fs::path("plugins"));
            plugin_working_dir = MLDB_ROOT / fs::path("plugins") / fs::path(pluginId.rawString());
            cerr << " PLUGIN INIT. Inside docker. Plugin working dir: " << plugin_working_dir << endl;
        }
        // if not, create a /mldb_data in the working directory to mimic
        // the behaviour of when we'll be dockyfied
        else {
            fs::create_directories(fs::path(".") / MLDB_ROOT / "plugins");
            plugin_working_dir = fs::path(".") / MLDB_ROOT / "plugins" / fs::path(pluginId.rawString());
            cerr << " PLUGIN INIT. NOT inside docker. Plugin working dir: " << plugin_working_dir << endl;
        }
    }
    // if we're a script, create a temporary folder so we can do the checkout if required
    else {
        plugin_working_dir = make_unique_directory(fs::temp_directory_path());
    }

    // if we're passing in the source
    if(!resource.source.empty() && !resource.source.writeSourceToFile) {
        source = resource.source;
        pluginLocation = SOURCE;
        return;
    }

    // TODO what is the folder already exists?! we could create two different plugins
    // with the same id, which would clash... ?
    if(fs::exists(plugin_working_dir)) {
        try {
            MLDB_TRACE_EXCEPTIONS(false);
            fs::remove_all(plugin_working_dir);<commit_msg>Plugin resource filesystem fix<commit_after>// This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

/** plugin_resource.cc
    <NAME>, 18 fevrier 2015
    Copyright (c) 2015 mldb.ai inc.  All rights reserved.

*/

#include "mldb/server/plugin_resource.h"

#include "mldb/ext/libgit2/include/git2.h"
#include "mldb/ext/libgit2/include/git2/clone.h"
#include "mldb/types/structure_description.h"
#include "mldb/vfs/filter_streams.h"
#include "mldb/utils/tmpdir.h"

using namespace std;

namespace fs = std::filesystem;


namespace MLDB {


/*****************************************************************************/
/* PLUGIN STRUCTURES                                                         */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(PackageElementSources);

PackageElementSourcesDescription::
PackageElementSourcesDescription()
{
    addField("main", &PackageElementSources::main,
             "source for the main element of the plugin");
    addField("routes", &PackageElementSources::routes,
             "source for the routes element of the plugin");
    addField("status", &PackageElementSources::status,
             "source for the status element of the plugin");
}


DEFINE_STRUCTURE_DESCRIPTION(PluginResource);

PluginResourceDescription::
PluginResourceDescription()
{
    addField("address", &PluginResource::address,
             "URI or location of script to run (use this parameter OR 'source')");
    PackageElementSources elem;
    addField("source", &PluginResource::source,
             "source of script to run (use this parameter OR 'address')", elem);
    addField("args", &PluginResource::args,
             "Arguments to pass to script");
}


DEFINE_STRUCTURE_DESCRIPTION(ScriptResource);

ScriptResourceDescription::
ScriptResourceDescription()
{
    addField("address", &ScriptResource::address,
             "URI or location of script to run (use this parameter OR 'source')");
    addField("source", &ScriptResource::source,
             "source of script to run (use this parameter OR 'address')");
    addField("args", &ScriptResource::args,
             "Arguments to pass to script");
    addField("writeSourceToFile", &ScriptResource::writeSourceToFile,
             "If passing in the source as a string, write it to disk to a temporary "
              "location as if it was downloaded remotely. This should usually not be "
              "necessary.", false);
}


DEFINE_STRUCTURE_DESCRIPTION(PluginVersion);

PluginVersionDescription::
PluginVersionDescription()
{
    addField("branch", &PluginVersion::branch,
             "branch currently checkout");
    addField("revision", &PluginVersion::revision,
             "revision currently checked out");
    addField("address", &PluginVersion::address,
             "address");
    addField("message", &PluginVersion::message,
             "message");
}

ScriptLanguage parseScriptLanguage(const std::string lang)
{
    if(lang == "python")        return PYTHON;
    if(lang == "javascript")    return JAVASCRIPT;
    throw MLDB::Exception("Unknown language");
}

/*****************************************************************************/
/* PLUGIN RESSOURCE                                                          */
/*****************************************************************************/
    
LoadedPluginResource::
LoadedPluginResource(ScriptLanguage lang, ScriptType type,
                     Utf8String pluginId,
                     const PluginResource & resource) :
    url(resource.address), args(resource.args), MLDB_ROOT("/mldb_data/")
{
    pluginLanguage = lang;
    scriptType = type;

    if(!url.empty() && !resource.source.empty())
        throw MLDB::Exception("Cannot specify both address and source for plugin");

    // if this is a plugin, we need to create a folder in the plugins folder
    // that we'll be keeping around
    if(scriptType == PLUGIN) {
        // if we are running inside of docker. TODO this check is a little flimsy
        if(fs::exists(MLDB_ROOT)) {
            fs::create_directories(MLDB_ROOT / fs::path("plugins"));
            plugin_working_dir = MLDB_ROOT / fs::path("plugins") / fs::path(pluginId.rawString());
            cerr << " PLUGIN INIT. Inside docker. Plugin working dir: " << plugin_working_dir << endl;
        }
        // if not, create a /mldb_data in the working directory to mimic
        // the behaviour of when we'll be dockyfied
        else {
            fs::create_directories(fs::path("./mldb_data/plugins"));
            plugin_working_dir = fs::path("./mldb_data/plugins") / fs::path(pluginId.rawString());
            cerr << " PLUGIN INIT. NOT inside docker. Plugin working dir: " << plugin_working_dir << endl;
        }
    }
    // if we're a script, create a temporary folder so we can do the checkout if required
    else {
        plugin_working_dir = make_unique_directory(fs::temp_directory_path());
    }

    // if we're passing in the source
    if(!resource.source.empty() && !resource.source.writeSourceToFile) {
        source = resource.source;
        pluginLocation = SOURCE;
        return;
    }

    // TODO what is the folder already exists?! we could create two different plugins
    // with the same id, which would clash... ?
    if(fs::exists(plugin_working_dir)) {
        try {
            MLDB_TRACE_EXCEPTIONS(false);
            fs::remove_all(plugin_working_dir);
{'fraction_non_alphanumeric': 0.16132273682287385, 'fraction_numerical': 0.0034712706677628574, 'mean_word_length': 3.1096096096096097, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5254469', 'n_tokens_mistral': 2923, 'n_tokens_neox': 2799, 'n_words': 798}