<commit_before>#include "LanguageTranslator.h"

#include <sstream>

#include "Logger.h"
#include "StringUtils.h"

std::vector<std::wstring> LanguageTranslator::CurrentLocale() {
    wchar_t buf[LOCALE_NAME_MAX_LENGTH];
    GetUserDefaultLocaleName(buf, LOCALE_NAME_MAX_LENGTH);
    CLOG(L"Locale: %s", buf);

    std::wstring locale(buf);

    /* Remove custom supplemental locales */
    int custom = locale.find(L"-x-");
    if (custom > 0) {
        locale.erase(custom, locale.size() - 1);
    }

    /* Split up parts of the locale string */
    std::vector<std::wstring> parts;
    std::wstring sub;
    std::wistringstream ss(locale);
    while (std::getline(ss, sub, L'-')) {
        parts.push_back(sub);
    }

    std::vector<std::wstring> finalLocale;
    for (std::wstring sub : parts) {
        /* Remove sort orders */
        int sort = sub.find(L"_");
        if (sort > 0) {
            sub.erase(sort, sub.size() - 1);
        }

        finalLocale.push_back(sub);
    }

    return finalLocale;
}

LanguageTranslator::LanguageTranslator() {

}

LanguageTranslator::LanguageTranslator(std::wstring langFileName) {
    CLOG(L"Loading language XML: %s", langFileName.c_str());
    std::string u8FileName = StringUtils::Narrow(langFileName);
    tinyxml2::XMLError result = _xml.LoadFile(u8FileName.c_str());
    if (result != tinyxml2::XMLError::XML_SUCCESS) {
        if (result == tinyxml2::XMLError::XML_ERROR_FILE_NOT_FOUND) {
            CLOG(L"File not found");
        }
        return;
    }

    _root = _xml.GetDocument()->FirstChildElement("translation");
    if (_root == NULL) {
        throw std::runtime_error("Could not find root XML element");
    }

    /*
    _author = StringUtils::Widen(_root->Attribute("author", ""));
    */

    tinyxml2::XMLElement *trans = _root->FirstChildElement("language");
    if (trans == NULL) {
        CLOG(L"No <language> tag.");<commit_msg>Fix UTF-8 language name load bug<commit_after>#include "LanguageTranslator.h"

#include <sstream>

#include "Logger.h"
#include "StringUtils.h"

std::vector<std::wstring> LanguageTranslator::CurrentLocale() {
    wchar_t buf[LOCALE_NAME_MAX_LENGTH];
    GetUserDefaultLocaleName(buf, LOCALE_NAME_MAX_LENGTH);
    CLOG(L"Locale: %s", buf);

    std::wstring locale(buf);

    /* Remove custom supplemental locales */
    int custom = locale.find(L"-x-");
    if (custom > 0) {
        locale.erase(custom, locale.size() - 1);
    }

    /* Split up parts of the locale string */
    std::vector<std::wstring> parts;
    std::wstring sub;
    std::wistringstream ss(locale);
    while (std::getline(ss, sub, L'-')) {
        parts.push_back(sub);
    }

    std::vector<std::wstring> finalLocale;
    for (std::wstring sub : parts) {
        /* Remove sort orders */
        int sort = sub.find(L"_");
        if (sort > 0) {
            sub.erase(sort, sub.size() - 1);
        }

        finalLocale.push_back(sub);
    }

    return finalLocale;
}

LanguageTranslator::LanguageTranslator() {

}

LanguageTranslator::LanguageTranslator(std::wstring langFileName) {
    CLOG(L"Loading language XML: %s", langFileName.c_str());

    FILE *fp;
    _wfopen_s(&fp, langFileName.c_str(), L"rb");
    if (fp == NULL) {
        CLOG(L"Failed to open file!");
        return;
    }

    tinyxml2::XMLError result = _xml.LoadFile(fp);
    if (result != tinyxml2::XMLError::XML_SUCCESS) {
        CLOG(L"Failed to read language file!");
        return;
    }

    _root = _xml.GetDocument()->FirstChildElement("translation");
    if (_root == NULL) {
        throw std::runtime_error("Could not find root XML element");
    }

    /*
    _author = StringUtils::Widen(_root->Attribute("author", ""));
    */

    tinyxml2::XMLElement *trans = _root->FirstChildElement("language");
    if (trans == NULL) {
        CLOG(L"No <language> tag.");
{'fraction_non_alphanumeric': 0.15382613222280062, 'fraction_numerical': 0.004685059864653826, 'mean_word_length': 3.1278195488721803, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7119969', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1259, 'n_words': 249}