<commit_before>/*****************************************************************************************
 *                                                                                       *
 * OpenSpace                                                                             *
 *                                                                                       *
 * Copyright (c) 2014-2020                                                               *
 *                                                                                       *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this  *
 * software and associated documentation files (the "Software"), to deal in the Software *
 * without restriction, including without limitation the rights to use, copy, modify,    *
 * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to    *
 * permit persons to whom the Software is furnished to do so, subject to the following   *
 * conditions:                                                                           *
 *                                                                                       *
 * The above copyright notice and this permission notice shall be included in all copies *
 * or substantial portions of the Software.                                              *
 *                                                                                       *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,   *
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A         *
 * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT    *
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF  *
 * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE  *
 * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                                         *
 ****************************************************************************************/

#include <modules/exoplanets/tasks/exoplanetsdatapreparationtask.h>

#include <modules/exoplanets/exoplanetshelper.h>
#include <openspace/documentation/documentation.h>
#include <openspace/documentation/verifier.h>
#include <openspace/json.h>
#include <ghoul/filesystem/filesystem.h>
#include <ghoul/fmt.h>
#include <ghoul/glm.h>
#include <ghoul/logging/logmanager.h>
#include <ghoul/misc/dictionary.h>
#include <charconv>
#include <fstream>

 // TEST: do not optimise for now
#pragma optimize("", off)

namespace {
    constexpr const char* KeyInputDataFile = "InputDataFile";
    constexpr const char* KeyInputSpeck = "InputSPECK";
    constexpr const char* KeyOutputBin = "OutputBIN";
    constexpr const char* KeyOutputLut = "OutputLUT";
    constexpr const char* KeyTeffToBv = "TeffToBvFile";

    constexpr const char* _loggerCat = "ExoplanetsDataPreparationTask";
} // namespace

namespace openspace::exoplanets {

using json = nlohmann::json;

ExoplanetsDataPreparationTask::ExoplanetsDataPreparationTask(const ghoul::Dictionary& dictionary) {
    openspace::documentation::testSpecificationAndThrow(
        documentation(),
        dictionary,
        "ExoplanetsDataPreparationTask"
    );

    _inputDataPath = absPath(dictionary.value<std::string>(KeyInputDataFile));
    _inputSpeckPath = absPath(dictionary.value<std::string>(KeyInputSpeck));
    _outputBinPath = absPath(dictionary.value<std::string>(KeyOutputBin));
    _outputLutPath = absPath(dictionary.value<std::string>(KeyOutputLut));
    _teffToBvFilePath = absPath(dictionary.value<std::string>(KeyTeffToBv));
}

std::string ExoplanetsDataPreparationTask::description() {
    return fmt::format(
        "Extract metadata from file '{}' and write as bin to '{}'", // TODO: update description
        _inputDataPath,
        _outputBinPath
    );
}

void ExoplanetsDataPreparationTask::perform(const Task::ProgressCallback& progressCallback) {
    std::ifstream inputDataFile(_inputDataPath);
    if (!inputDataFile.good()) {
        LERROR(fmt::format("Failed to open input file '{}'", _inputDataPath));
        return;
    }

    std::ofstream binFile(_outputBinPath, std::ios::out | std::ios::binary);
    std::ofstream lutFile(_outputLutPath);

    int version = 1;
    binFile.write(reinterpret_cast<char*>(&version), sizeof(int));

    std::string planetRow;
    getline(inputDataFile, planetRow); // The first line, containing the data names

    // read column names into a vector, for access later
    std::vector<std::string> columnNames;
    std::stringstream sStream(planetRow);
    std::string colName;
    while (getline(sStream, colName, ',')) {
        columnNames.push_back(colName);
    }

    // read total number of items
    int total = 0;
    while (getline(inputDataFile, planetRow)) {
        ++total;
    }
    inputDataFile.clear();
    inputDataFile.seekg(0);
    getline(inputDataFile, planetRow); // The first line, containing the data names

    LINFO(fmt::format("Loading {} exoplanets", total));

    auto readFloatData = [](const std::string& str) -> float {
        float result;
        auto [p, ec] = std::from_chars(str.data(), str.data() + str.size(), result);
        if (ec == std::errc()) {
            return result;
        }
        return NAN;
    };

    auto readDoubleData = [](const std::string& str) -> double {
        double result;
        auto [p, ec] = std::from_chars(str.data(), str.data() + str.size(), result);
        if (ec == std::errc()) {
            return result;
        }
        return NAN;
    };

    auto readIntegerData = [](const std::string& str) -> int {
        int result;
        auto [p, ec] = std::from_chars(str.data(), str.data() + str.size(), result);
        if (ec == std::errc()) {
            return result;
        }
        return -1;
    };

    auto readStringData = [](const std::string& str) -> std::string {
        std::string result = str;
        // remove quotes, if any
        result.erase(std::remove(result.begin(), result.end(), '\"'), result.end());
        return result;
    };

    Exoplanet p;
    std::string data;
    int exoplanetCount = 0;
    while (getline(inputDataFile, planetRow)) {<commit_msg>Add fix for std::from_chars not working on Linux<commit_after>/*****************************************************************************************
 *                                                                                       *
 * OpenSpace                                                                             *
 *                                                                                       *
 * Copyright (c) 2014-2020                                                               *
 *                                                                                       *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this  *
 * software and associated documentation files (the "Software"), to deal in the Software *
 * without restriction, including without limitation the rights to use, copy, modify,    *
 * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to    *
 * permit persons to whom the Software is furnished to do so, subject to the following   *
 * conditions:                                                                           *
 *                                                                                       *
 * The above copyright notice and this permission notice shall be included in all copies *
 * or substantial portions of the Software.                                              *
 *                                                                                       *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,   *
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A         *
 * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT    *
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF  *
 * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE  *
 * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                                         *
 ****************************************************************************************/

#include <modules/exoplanets/tasks/exoplanetsdatapreparationtask.h>

#include <modules/exoplanets/exoplanetshelper.h>
#include <openspace/documentation/documentation.h>
#include <openspace/documentation/verifier.h>
#include <openspace/json.h>
#include <ghoul/filesystem/filesystem.h>
#include <ghoul/fmt.h>
#include <ghoul/glm.h>
#include <ghoul/logging/logmanager.h>
#include <ghoul/misc/dictionary.h>
#include <charconv>
#include <fstream>

 // TEST: do not optimise for now
#pragma optimize("", off)

namespace {
    constexpr const char* KeyInputDataFile = "InputDataFile";
    constexpr const char* KeyInputSpeck = "InputSPECK";
    constexpr const char* KeyOutputBin = "OutputBIN";
    constexpr const char* KeyOutputLut = "OutputLUT";
    constexpr const char* KeyTeffToBv = "TeffToBvFile";

    constexpr const char* _loggerCat = "ExoplanetsDataPreparationTask";
} // namespace

namespace openspace::exoplanets {

using json = nlohmann::json;

ExoplanetsDataPreparationTask::ExoplanetsDataPreparationTask(const ghoul::Dictionary& dictionary) {
    openspace::documentation::testSpecificationAndThrow(
        documentation(),
        dictionary,
        "ExoplanetsDataPreparationTask"
    );

    _inputDataPath = absPath(dictionary.value<std::string>(KeyInputDataFile));
    _inputSpeckPath = absPath(dictionary.value<std::string>(KeyInputSpeck));
    _outputBinPath = absPath(dictionary.value<std::string>(KeyOutputBin));
    _outputLutPath = absPath(dictionary.value<std::string>(KeyOutputLut));
    _teffToBvFilePath = absPath(dictionary.value<std::string>(KeyTeffToBv));
}

std::string ExoplanetsDataPreparationTask::description() {
    return fmt::format(
        "Extract metadata from file '{}' and write as bin to '{}'", // TODO: update description
        _inputDataPath,
        _outputBinPath
    );
}

void ExoplanetsDataPreparationTask::perform(const Task::ProgressCallback& progressCallback) {
    std::ifstream inputDataFile(_inputDataPath);
    if (!inputDataFile.good()) {
        LERROR(fmt::format("Failed to open input file '{}'", _inputDataPath));
        return;
    }

    std::ofstream binFile(_outputBinPath, std::ios::out | std::ios::binary);
    std::ofstream lutFile(_outputLutPath);

    int version = 1;
    binFile.write(reinterpret_cast<char*>(&version), sizeof(int));

    std::string planetRow;
    getline(inputDataFile, planetRow); // The first line, containing the data names

    // read column names into a vector, for access later
    std::vector<std::string> columnNames;
    std::stringstream sStream(planetRow);
    std::string colName;
    while (getline(sStream, colName, ',')) {
        columnNames.push_back(colName);
    }

    // read total number of items
    int total = 0;
    while (getline(inputDataFile, planetRow)) {
        ++total;
    }
    inputDataFile.clear();
    inputDataFile.seekg(0);
    getline(inputDataFile, planetRow); // The first line, containing the data names

    LINFO(fmt::format("Loading {} exoplanets", total));

    auto readFloatData = [](const std::string& str) -> float {
    #ifdef WIN32
        float result;
        auto [p, ec] = std::from_chars(str.data(), str.data() + str.size(), result);
        if (ec == std::errc()) {
            return result;
        }
        return NAN;
    #else
        // clang is missing float support for std::from_chars
        return !str.empty() ? std::stof(str.c_str(), nullptr) : NAN;
    #endif
    };

    auto readDoubleData = [](const std::string& str) -> double {
    #ifdef WIN32
        double result;
        auto [p, ec] = std::from_chars(str.data(), str.data() + str.size(), result);
        if (ec == std::errc()) {
            return result;
        }
        return NAN;
    #else
        // clang is missing double support for std::from_chars
        return !str.empty() ? std::stod(str.c_str(), nullptr) : NAN;
    #endif
    };

    auto readIntegerData = [](const std::string& str) -> int {
        int result;
        auto [p, ec] = std::from_chars(str.data(), str.data() + str.size(), result);
        if (ec == std::errc()) {
            return result;
        }
        return -1;
    };

    auto readStringData = [](const std::string& str) -> std::string {
        std::string result = str;
        // remove quotes, if any
        result.erase(std::remove(result.begin(), result.end(), '\"'), result.end());
        return result;
    };

    Exoplanet p;
    std::string data;
    int exoplanetCount = 0;
    while (getline(inputDataFile, planetRow)) {
{'fraction_non_alphanumeric': 0.13236655353286023, 'fraction_numerical': 0.0023141005862388152, 'mean_word_length': 2.2939532520325203, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 62, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7148097', 'n_tokens_mistral': 3424, 'n_tokens_neox': 3267, 'n_words': 935}