<commit_before>#include "ScriptDecoder.h"
#include "utf8_decode.h"

#ifdef ENABLE_KOREAN
extern unsigned short convKOR2UTF16(unsigned short code);
#endif

#ifdef ENABLE_CHINESE
extern unsigned short convGBK2UTF16(unsigned short code);
#endif

extern unsigned short convSJIS2UTF16( unsigned short in );

const char* ScriptDecoder::name = "One byte";

bool ScriptDecoder::canConvertNextChar(char* buffer, int* outNumBytes)
{
    *outNumBytes = 1;
    return isOneByte(*buffer);
}

unsigned short ScriptDecoder::convertNextChar(char* buffer)
{
    if ((buffer[0] & 0xe0) == 0xa0 || (buffer[0] & 0xe0) == 0xc0) {
        return ((unsigned char*) buffer)[0] - 0xa0 + 0xff60;
    } else {
        return *buffer;
    }
}

bool ScriptDecoder::isMonospaced()
{
    return false;
}

ScriptDecoder* ScriptDecoder::detectAndAllocateScriptDecoder(char* buffer, size_t size)
{
    const int TOTAL_COUNT_FINISHED = 20;
    ScriptDecoder* decoders[] = { new JapaneseDecoder()
#ifdef ENABLE_KOREAN
                                , new KoreanDecoder()
#endif
#ifdef ENABLE_CHINESE
                                , new ChineseDecoder()
#endif
                                };

    size_t numDecoders = sizeof(decoders) / sizeof(decoders[0]);
    int* langCounter = new int[numDecoders]();
    ScriptDecoder* foundDecoder = 0;
    int lineNum = 1;
    size_t i = 0;

    // Read line routine
    while(i < size && !foundDecoder) {
        // Skip the spaces at the beginning of each line
        while(i < size && buffer[i] == ' ') i++;
        if (i >= size) break;

        // Skip square brackets if they exist
        if (buffer[i] == '[') {
            do i++; while(i < size && buffer[i] != ']' && buffer[i] != '\n');
            if (++i >= size) break;

            // Incorrect format, skip this line
            if (buffer[i] == '\n') {
                lineNum++;
                continue;
            }
        }

        char c = buffer[i];
        if (c == '`') {
            // This says that the text is guarenteed to be English/UTF8
            foundDecoder = new UTF8Decoder();
            break;
        } else if (!ScriptDecoder::isOneByte(c)) {
            // We need to detect Asian language
            for (size_t j = 0; j < numDecoders && !foundDecoder; j++) {
                // Get line length
                char* start = buffer + i;
                char* ptr = start;
                while(*ptr != '\n' && *ptr != '\0') ptr++;
                size_t lineLength = ptr - start;
                if (*ptr == '\0' || !lineLength) break;

                // Parse through each character on this line and see which decoders work
                ScriptDecoder* decoder = decoders[j];
                int k = 0, outNumBytes = 0;
                while (k < lineLength && !foundDecoder) {
                    if (!decoder->canConvertNextChar(buffer + k + i, &outNumBytes)) {
                        break;
                    }
                    k += outNumBytes;

                    // Successfully read the entire line
                    if (k == lineLength) {
                        langCounter[j]++;

                        // Found our decoder
                        if (langCounter[j] >= TOTAL_COUNT_FINISHED) {
                            foundDecoder = decoders[j];
                        }
                    }
                }
            }
        }
        lineNum++;

        // Skip to the next line
        while(i < size && buffer[i] != '\n') i++;
        if (++i >= size) break;
    }

    delete langCounter;
<commit_msg>Corrects script detection when 2byte characters mix with single byte<commit_after>#include "ScriptDecoder.h"
#include "utf8_decode.h"

#ifdef ENABLE_KOREAN
extern unsigned short convKOR2UTF16(unsigned short code);
#endif

#ifdef ENABLE_CHINESE
extern unsigned short convGBK2UTF16(unsigned short code);
#endif

extern unsigned short convSJIS2UTF16( unsigned short in );

const char* ScriptDecoder::name = "One byte";

bool ScriptDecoder::canConvertNextChar(char* buffer, int* outNumBytes)
{
    *outNumBytes = 1;
    return isOneByte(*buffer);
}

unsigned short ScriptDecoder::convertNextChar(char* buffer)
{
    if ((buffer[0] & 0xe0) == 0xa0 || (buffer[0] & 0xe0) == 0xc0) {
        return ((unsigned char*) buffer)[0] - 0xa0 + 0xff60;
    } else {
        return *buffer;
    }
}

bool ScriptDecoder::isMonospaced()
{
    return false;
}

ScriptDecoder* ScriptDecoder::detectAndAllocateScriptDecoder(char* buffer, size_t size)
{
    const int TOTAL_COUNT_FINISHED = 20;
    ScriptDecoder* decoders[] = { new JapaneseDecoder()
#ifdef ENABLE_KOREAN
                                , new KoreanDecoder()
#endif
#ifdef ENABLE_CHINESE
                                , new ChineseDecoder()
#endif
                                };

    size_t numDecoders = sizeof(decoders) / sizeof(decoders[0]);
    int* langCounter = new int[numDecoders]();
    ScriptDecoder* foundDecoder = 0;
    int lineNum = 1;
    size_t i = 0;

    // Read line routine
    while(i < size && !foundDecoder) {
        // Skip the spaces at the beginning of each line
        while(i < size && buffer[i] == ' ') i++;
        if (i >= size) break;

        // Skip square brackets if they exist
        if (buffer[i] == '[') {
            do i++; while(i < size && buffer[i] != ']' && buffer[i] != '\n');
            if (++i >= size) break;

            // Incorrect format, skip this line
            if (buffer[i] == '\n') {
                lineNum++;
                continue;
            }
        }

        char c = buffer[i];
        if (c == '`') {
            // This says that the text is guarenteed to be English/UTF8
            foundDecoder = new UTF8Decoder();
            break;
        } else if (c != ';' && c != '*') {
            // We need to detect Asian language
            for (size_t j = 0; j < numDecoders && !foundDecoder; j++) {
                // Get line length
                char* start = buffer + i;
                char* ptr = start;
                int numOneBytes = 0;
                while(*ptr != '\n' && *ptr != '\0') {
                    if (ScriptDecoder::isOneByte(*ptr)) numOneBytes++;
                    ptr++;
                }
                size_t lineLength = ptr - start;
                if (*ptr == '\0' || !lineLength) break;

                // If more of the line is multibyte, then we should continue
                if (numOneBytes * 1.0 / lineLength >= 0.5) break;

                // Parse through each character on this line and see which decoders work
                ScriptDecoder* decoder = decoders[j];
                int k = 0, outNumBytes = 0;
                while (k < lineLength && !foundDecoder) {
                    if (!decoder->canConvertNextChar(buffer + k + i, &outNumBytes)) {
                        break;
                    }
                    k += outNumBytes;

                    // Successfully read the entire line
                    if (k == lineLength) {
                        langCounter[j]++;

                        // Found our decoder
                        if (langCounter[j] >= TOTAL_COUNT_FINISHED) {
                            foundDecoder = decoders[j];
                        }
                    }
                }
            }
        }
        lineNum++;

        // Skip to the next line
        while(i < size && buffer[i] != '\n') i++;
        if (++i >= size) break;
    }

    delete langCounter;

{'fraction_non_alphanumeric': 0.11732294046499127, 'fraction_numerical': 0.011557586345921246, 'mean_word_length': 1.602097902097902, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2640729', 'n_tokens_mistral': 2211, 'n_tokens_neox': 2061, 'n_words': 603}