<commit_before>#include "MiniEngine.h"
using namespace std;

#include <windows.h>

//GBK编码转换到UTF8编码
int _GBKToUTF8(unsigned char * lpGBKStr,unsigned char * lpUTF8Str,int nUTF8StrLen)
{
    wchar_t * lpUnicodeStr = NULL;
    int nRetLen = 0;

    if(!lpGBKStr)  //如果GBK字符串为NULL则出错退出
        return 0;

    nRetLen = MultiByteToWideChar(CP_ACP,0,(char *)lpGBKStr,-1,NULL,0);  //获取转换到Unicode编码后所需要的字符空间长度
    lpUnicodeStr = new WCHAR[nRetLen + 1];  //为Unicode字符串空间
    nRetLen = ::MultiByteToWideChar(CP_ACP,0,(char *)lpGBKStr,-1,lpUnicodeStr,nRetLen);  //转换到Unicode编码
    if(!nRetLen)  //转换失败则出错退出
        return 0;

    nRetLen = ::WideCharToMultiByte(CP_UTF8,0,lpUnicodeStr,-1,NULL,0,NULL,0);  //获取转换到UTF8编码后所需要的字符空间长度

    if(!lpUTF8Str)  //输出缓冲区为空则返回转换后需要的空间大小
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return nRetLen;
    }

    if(nUTF8StrLen < nRetLen)  //如果输出缓冲区长度不够则退出
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return 0;
    }

    nRetLen = ::WideCharToMultiByte(CP_UTF8,0,lpUnicodeStr,-1,(char *)lpUTF8Str,nUTF8StrLen,NULL,NULL);  //转换到UTF8编码

    if(lpUnicodeStr)
        delete []lpUnicodeStr;

    return nRetLen;
}

// UTF8编码转换到GBK编码
int _UTF8ToGBK(unsigned char * lpUTF8Str,unsigned char * lpGBKStr,int nGBKStrLen)
{
    wchar_t * lpUnicodeStr = NULL;
    int nRetLen = 0;

    if(!lpUTF8Str)  //如果UTF8字符串为NULL则出错退出
        return 0;

    nRetLen = ::MultiByteToWideChar(CP_UTF8,0,(char *)lpUTF8Str,-1,NULL,0);  //获取转换到Unicode编码后所需要的字符空间长度
    lpUnicodeStr = new WCHAR[nRetLen + 1];  //为Unicode字符串空间
    nRetLen = ::MultiByteToWideChar(CP_UTF8,0,(char *)lpUTF8Str,-1,lpUnicodeStr,nRetLen);  //转换到Unicode编码
    if(!nRetLen)  //转换失败则出错退出
        return 0;

    nRetLen = ::WideCharToMultiByte(CP_ACP,0,lpUnicodeStr,-1,NULL,0,NULL,NULL);  //获取转换到GBK编码后所需要的字符空间长度

    if(!lpGBKStr)  //输出缓冲区为空则返回转换后需要的空间大小
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return nRetLen;
    }

    if(nGBKStrLen < nRetLen)  //如果输出缓冲区长度不够则退出
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return 0;
    }

    nRetLen = ::WideCharToMultiByte(CP_ACP,0,lpUnicodeStr,-1,(char *)lpGBKStr,nRetLen,NULL,NULL);  //转换到GBK编码

    if(lpUnicodeStr)
        delete []lpUnicodeStr;

    return nRetLen;
}

string UTF8ToGBK(string UTF8String)
{
    int sz=UTF8String.size()+32;
    auto utf8str=new unsigned char[sz];
    memset(utf8str,0,sz);
    memcpy(utf8str,UTF8String.c_str(),UTF8String.size());

    auto gbkstr=new unsigned char[sz];
    memset(gbkstr,0,sz);

    _UTF8ToGBK(utf8str,gbkstr,sz);

    string s((char*)gbkstr);

    delete[] gbkstr;
    delete[] utf8str;
    return s;
}

string GBKToUTF8(string GBKString)
{
    int sz=GBKString.size()+32;
    auto gbkstr=new unsigned char[sz];
    memset(gbkstr,0,sz);
    memcpy(gbkstr,GBKString.c_str(),GBKString.size());
<commit_msg>Fix bugs in UTF8ToGBK and GBKToUTF8 GBK->UTF8 x/2*3 UTF8->GBK x/3*2<commit_after>#include "MiniEngine.h"
using namespace std;

#include <windows.h>

//GBK编码转换到UTF8编码
int _GBKToUTF8(unsigned char * lpGBKStr,unsigned char * lpUTF8Str,int nUTF8StrLen)
{
    wchar_t * lpUnicodeStr = NULL;
    int nRetLen = 0;

    if(!lpGBKStr)  //如果GBK字符串为NULL则出错退出
        return 0;

    nRetLen = MultiByteToWideChar(CP_ACP,0,(char *)lpGBKStr,-1,NULL,0);  //获取转换到Unicode编码后所需要的字符空间长度
    lpUnicodeStr = new WCHAR[nRetLen + 1];  //为Unicode字符串空间
    nRetLen = ::MultiByteToWideChar(CP_ACP,0,(char *)lpGBKStr,-1,lpUnicodeStr,nRetLen);  //转换到Unicode编码
    if(!nRetLen)  //转换失败则出错退出
        return 0;

    nRetLen = ::WideCharToMultiByte(CP_UTF8,0,lpUnicodeStr,-1,NULL,0,NULL,0);  //获取转换到UTF8编码后所需要的字符空间长度

    if(!lpUTF8Str)  //输出缓冲区为空则返回转换后需要的空间大小
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return nRetLen;
    }

    if(nUTF8StrLen < nRetLen)  //如果输出缓冲区长度不够则退出
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return 0;
    }

    nRetLen = ::WideCharToMultiByte(CP_UTF8,0,lpUnicodeStr,-1,(char *)lpUTF8Str,nUTF8StrLen,NULL,NULL);  //转换到UTF8编码

    if(lpUnicodeStr)
        delete []lpUnicodeStr;

    return nRetLen;
}

// UTF8编码转换到GBK编码
int _UTF8ToGBK(unsigned char * lpUTF8Str,unsigned char * lpGBKStr,int nGBKStrLen)
{
    wchar_t * lpUnicodeStr = NULL;
    int nRetLen = 0;

    if(!lpUTF8Str)  //如果UTF8字符串为NULL则出错退出
        return 0;

    nRetLen = ::MultiByteToWideChar(CP_UTF8,0,(char *)lpUTF8Str,-1,NULL,0);  //获取转换到Unicode编码后所需要的字符空间长度
    lpUnicodeStr = new WCHAR[nRetLen + 1];  //为Unicode字符串空间
    nRetLen = ::MultiByteToWideChar(CP_UTF8,0,(char *)lpUTF8Str,-1,lpUnicodeStr,nRetLen);  //转换到Unicode编码
    if(!nRetLen)  //转换失败则出错退出
        return 0;

    nRetLen = ::WideCharToMultiByte(CP_ACP,0,lpUnicodeStr,-1,NULL,0,NULL,NULL);  //获取转换到GBK编码后所需要的字符空间长度

    if(!lpGBKStr)  //输出缓冲区为空则返回转换后需要的空间大小
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return nRetLen;
    }

    if(nGBKStrLen < nRetLen)  //如果输出缓冲区长度不够则退出
    {
        if(lpUnicodeStr)
            delete []lpUnicodeStr;
        return 0;
    }

    nRetLen = ::WideCharToMultiByte(CP_ACP,0,lpUnicodeStr,-1,(char *)lpGBKStr,nRetLen,NULL,NULL);  //转换到GBK编码

    if(lpUnicodeStr)
        delete []lpUnicodeStr;

    return nRetLen;
}

string UTF8ToGBK(string UTF8String)
{
    int sz=UTF8String.size()*2/3+256;
    auto utf8str=new unsigned char[sz];
    memset(utf8str,0,sz);
    memcpy(utf8str,UTF8String.c_str(),UTF8String.size());

    auto gbkstr=new unsigned char[sz];
    memset(gbkstr,0,sz);

    _UTF8ToGBK(utf8str,gbkstr,sz);

    string s((char*)gbkstr);

    delete[] gbkstr;
    delete[] utf8str;
    return s;
}

string GBKToUTF8(string GBKString)
{
    int sz=GBKString.size()*3/2+32;
    auto gbkstr=new unsigned char[sz];
    memset(gbkstr,0,sz);
    memcpy(gbkstr,GBKString.c_str(),GBKString.size());

{'fraction_non_alphanumeric': 0.12733185007701522, 'fraction_numerical': 0.02652746876604484, 'mean_word_length': 3.361194029850746, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3476387', 'n_tokens_mistral': 2649, 'n_tokens_neox': 2809, 'n_words': 343}