<commit_before>#include "compress_rle.h"

#include <stdexcept>

bool CompressRLE::CompressImpl(const ByteBuffer& source, ByteBuffer& output)
{
    throw std::exception("The method or operation is not implemented.");
}

bool CompressRLE::DecompressImpl(const ByteBuffer& input, ByteBuffer& result)
{
    throw std::exception("The method or operation is not implemented.");
}<commit_msg>Implement RLE<commit_after>#include "compress_rle.h"
#include "data_input.h"

#include "bytebuffer.h"

#include <cctype>
#include <stdexcept>

#define MARKER_CHAR '*'
#define MIN_REPEATED 4

bool ValidateInput(const ByteBuffer& source)
{
    const char* in = (const char*)source.Data();

    for (size_t i = 0; i < source.Size() - 1; ++i)
        if (in[i] == MARKER_CHAR && isdigit(in[i + 1])) // example: *1A
            return false;

    return true;
}

bool CompressRLE::CompressImpl(const ByteBuffer& input, ByteBuffer& output)
{
    if (!ValidateInput(input))
        return false;

    const char* in = (const char*)input.Data();

    for (size_t i = 0; i < input.Size(); ++i)
    {
        char c = in[i];

        if (i != input.Size() - 1) // can't be last char
        {
            size_t j = i + 1;
            int count = 1;
            while (in[j++] == c)
                count++;

            if (count >= MIN_REPEATED)
            {
                output.WriteUInt8(MARKER_CHAR); // especial marker

                char numStr[11];
                _itoa_s(count, numStr, 10);
                output.WriteBuffer(numStr, strlen(numStr)); // number of repeated 

                i += count - 1;
            }
        }

        output.WriteUInt8(c);
    }

    return true;
}

bool CompressRLE::DecompressImpl(const ByteBuffer& input, ByteBuffer& output)
{
    const char* in = (const char*)input.Data();

    for (size_t i = 0; i < input.Size(); ++i)
    {
        if (in[i] == MARKER_CHAR)
        {
            size_t digitCount = 0, j = i + 1;
            while (isdigit(in[j++]))
                digitCount++;

            int count = atoi(&in[i + 1]);
            for (int k = 0; k < count; ++k)
                output.WriteUInt8(in[i + 1 + digitCount]);

            i += digitCount + 1;
        }
        else
            output.WriteUInt8(in[i]);
    }

    return true;
}
{'fraction_non_alphanumeric': 0.13704028021015763, 'fraction_numerical': 0.010945709281961471, 'mean_word_length': 2.0836707152496627, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6872578', 'n_tokens_mistral': 775, 'n_tokens_neox': 727, 'n_words': 193}