<commit_before>#include "Srl/Srl.h"
#include "Srl/Lib.h"

using namespace std;
using namespace Srl;
using namespace Lib;

namespace {

    const function<void()> error = [] {
        throw Exception("Unable to parse JSON document. Index out of bounds.");
    };

    const In::Substitute uc_literal = [](In& src, vector<uint8_t>& buf, size_t idx) -> size_t {
        uint8_t hex_buffer[2];
        /* skip "\u" and read 4 hex digits */
        auto block = src.read_block(2 + 4, error);

        Tools::hex_to_bytes(hex_buffer, 2, block.ptr + 2, 4);

        return Tools::conv_charset(Encoding::UTF8, String(hex_buffer, 2, Encoding::UTF16), buf, true, idx);
    };

    constexpr array<const char, 2> ar(const char c)
    {
        return array<const char, 2> { { '\\', c } };
    }

    void write_escape(const MemBlock& str, Out& out)
    {
        out.write_substitute(str,
            '\"', ar('\"'), '\'', ar('\''), '\\', ar('\\'), '/', ar('/'), '\n', ar('n'),
            '\t', ar('t'), '\r',  ar('r'), '\b',  ar('b'), '\f', ar('f')
        );
    }

    MemBlock read_unescape(In& in, vector<uint8_t>& buffer)
    {
        auto len = in.read_substitue(error, '\"', buffer,
            '\"', ar('\"'), '\'', ar('\''), '\\', ar('\\'), '/', ar('/'), '\n', ar('n'),
            '\t', ar('t'), '\r',  ar('r'), '\b',  ar('b'), '\f', ar('f'), uc_literal, ar('u')
        );

        return { buffer.data(), len };
    }

    void insert_in_quotes(Out& out, const MemBlock& txt, bool escape = false)
    {
        out.write('\"');
        if(!escape) {
            out.write(txt);
        } else {
            write_escape(txt, out);
        }
        out.write('\"');
    }<commit_msg>remove single quote escape in json parser<commit_after>#include "Srl/Srl.h"
#include "Srl/Lib.h"

using namespace std;
using namespace Srl;
using namespace Lib;

namespace {

    const function<void()> error = [] {
        throw Exception("Unable to parse JSON document. Index out of bounds.");
    };

    const In::Substitute uc_literal = [](In& src, vector<uint8_t>& buf, size_t idx) -> size_t {
        uint8_t hex_buffer[2];
        /* skip "\u" and read 4 hex digits */
        auto block = src.read_block(2 + 4, error);

        Tools::hex_to_bytes(hex_buffer, 2, block.ptr + 2, 4);

        return Tools::conv_charset(Encoding::UTF8, String(hex_buffer, 2, Encoding::UTF16), buf, true, idx);
    };

    constexpr array<const char, 2> ar(const char c)
    {
        return array<const char, 2> { { '\\', c } };
    }

    void write_escape(const MemBlock& str, Out& out)
    {
        out.write_substitute(str,
            '\"', ar('\"'), '\\', ar('\\'), '/', ar('/'), '\n', ar('n'),
            '\t', ar('t'), '\r',  ar('r'), '\b',  ar('b'), '\f', ar('f')
        );
    }

    MemBlock read_unescape(In& in, vector<uint8_t>& buffer)
    {
        auto len = in.read_substitue(error, '\"', buffer,
            '\"', ar('\"'), '\'', ar('\''), '\\', ar('\\'), '/', ar('/'), '\n', ar('n'),
            '\t', ar('t'), '\r',  ar('r'), '\b',  ar('b'), '\f', ar('f'), uc_literal, ar('u')
        );

        return { buffer.data(), len };
    }

    void insert_in_quotes(Out& out, const MemBlock& txt, bool escape = false)
    {
        out.write('\"');
        if(!escape) {
            out.write(txt);
        } else {
            write_escape(txt, out);
        }
        out.write('\"');
    }
{'fraction_non_alphanumeric': 0.22209096278795037, 'fraction_numerical': 0.009450679267572357, 'mean_word_length': 2.2104265402843604, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7512219', 'n_tokens_mistral': 1214, 'n_tokens_neox': 1164, 'n_words': 287}