<commit_before>// NanoWin32
// -----------------------------------------------------------------------
// Simple library to subset Win32(64) API functions implemenation on POSIX
// This software distributed by MIT license

// INTERNAL: wchat_t to char conversion functions

#include "NanoWinStrConvert.h"

#include <string.h>
#include <wchar.h>

namespace NanoWin
{
  const char *StrConverter::Error::what() const
  {
    return "string conversion failed";
  }

  std::string  StrConverter::Convert(const std::wstring &src)
  {
    mbstate_t state;

    memset(&state, 0, sizeof(state));

    const wchar_t *srcChars = src.c_str();

    size_t requiredLen = wcsrtombs(NULL, &srcChars, 0, &state);

    if (requiredLen == (size_t)-1)
    {
      throw Error();
    }

    std::string result;

    result.reserve(requiredLen);

    char charBuffer[MB_CUR_MAX];

    memset(&state, 0, sizeof(state));

    for (std::wstring::const_iterator pos = src.begin(); pos != src.end(); ++pos)
    {
      size_t charCount = wcrtomb(charBuffer, *pos, &state);

      if (charCount == (size_t)-1)
      {
        throw Error();
      }

      result.append(charBuffer, charCount);
    }

    return result;
  }

  std::wstring StrConverter::Convert(const std::string &src)
  {
    mbstate_t state;

    memset(&state, 0, sizeof(state));

    const char *srcChars = src.c_str();

    size_t requiredLen = mbsrtowcs(NULL, &srcChars, 0, &state);

    if (requiredLen == (size_t)-1)
    {
      throw Error();
    }

    std::wstring result;

    result.reserve(requiredLen);

    memset(&state, 0, sizeof(state));

    wchar_t currChar;

    const char *srcPtr = src.c_str();
    size_t      tailLen = src.length() + 1;

    size_t charLen = mbrtowc(&currChar, srcPtr, tailLen, &state);

    while ((ssize_t)charLen > 0)
    {
      result += currChar;

      srcPtr += charLen;
      tailLen -= charLen;

      charLen = mbrtowc(&currChar, srcPtr, tailLen, &state);
    }
<commit_msg>Visual C++ compatibility fix for NanoWinStrConvert.cpp<commit_after>// NanoWin32
// -----------------------------------------------------------------------
// Simple library to subset Win32(64) API functions implemenation on POSIX
// This software distributed by MIT license

// INTERNAL: wchat_t to char conversion functions

#include "NanoWinStrConvert.h"

#include <string.h>
#include <wchar.h>

namespace NanoWin
{
  const char *StrConverter::Error::what() const
  {
    return "string conversion failed";
  }

  std::string  StrConverter::Convert(const std::wstring &src)
  {
    mbstate_t state;

    memset(&state, 0, sizeof(state));

    const wchar_t *srcChars = src.c_str();

    size_t requiredLen = wcsrtombs(NULL, &srcChars, 0, &state);

    if (requiredLen == (size_t)-1)
    {
      throw Error();
    }

    std::string result;

    result.reserve(requiredLen);

    #ifdef __linux
    char charBuffer[MB_CUR_MAX];
    #else
    char charBuffer[6];
    #endif

    memset(&state, 0, sizeof(state));

    for (std::wstring::const_iterator pos = src.begin(); pos != src.end(); ++pos)
    {
      size_t charCount = wcrtomb(charBuffer, *pos, &state);

      if (charCount == (size_t)-1)
      {
        throw Error();
      }

      result.append(charBuffer, charCount);
    }

    return result;
  }

  std::wstring StrConverter::Convert(const std::string &src)
  {
    mbstate_t state;

    memset(&state, 0, sizeof(state));

    const char *srcChars = src.c_str();

    size_t requiredLen = mbsrtowcs(NULL, &srcChars, 0, &state);

    if (requiredLen == (size_t)-1)
    {
      throw Error();
    }

    std::wstring result;

    result.reserve(requiredLen);

    memset(&state, 0, sizeof(state));

    wchar_t currChar;

    const char *srcPtr = src.c_str();
    size_t      tailLen = src.length() + 1;

    size_t charLen = mbrtowc(&currChar, srcPtr, tailLen, &state);

    while (charLen > 0 && charLen != (size_t)-1  && charLen != (size_t)-2)
    {
      result += currChar;

      srcPtr += charLen;
      tailLen -= charLen;

      charLen = mbrtowc(&currChar, srcPtr, tailLen, &state);
    }

{'fraction_non_alphanumeric': 0.1677687974528533, 'fraction_numerical': 0.009061964241978937, 'mean_word_length': 2.5887521968365554, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4195479', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1335, 'n_words': 321}