<commit_before>#include <locale>
#include <codecvt>
#include "NOD/NOD.hpp"

namespace NOD
{

std::string WideToUTF8(const std::wstring& src)
{
    std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
    return conv.to_bytes(src);
}

std::wstring UTF8ToWide(const std::string& src)
{
    std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
    return conv.from_bytes(src);
}

}<commit_msg>Removed codecvt<commit_after>#include "NOD/NOD.hpp"

namespace NOD
{

std::string WideToUTF8(const std::wstring& src)
{
    std::string retval;
    retval.reserve(src.length());
    for (wchar_t ch : src)
    {
        char mb[4];
        int c = std::wctomb(mb, ch);
        retval.append(mb, c);
    }
    return retval;
}

std::wstring UTF8ToWide(const std::string& src)
{
    std::wstring retval;
    retval.reserve(src.length());
    const char* buf = src.c_str();
    while (*buf)
    {
        wchar_t wc;
        buf += std::mbtowc(&wc, buf, MB_CUR_MAX);
        retval += wc;
    }
    return retval;
}

}
{'fraction_non_alphanumeric': 0.15376984126984128, 'fraction_numerical': 0.006944444444444444, 'mean_word_length': 3.406113537117904, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4088755', 'n_tokens_mistral': 399, 'n_tokens_neox': 397, 'n_words': 73}