<commit_before>
#include <cppexpose/json/JSON.h>

#include <iostream>

#include <cppassist/string/conversion.h>
#include <cppassist/logging/logging.h>

#include <cppexpose/base/Tokenizer.h>
#include <cppexpose/variant/Variant.h>


using namespace cppassist;
using namespace cppexpose;


namespace
{


bool readValue(Variant & value, Tokenizer::Token & token, Tokenizer & tokenizer);
bool readArray(Variant & root, Tokenizer & tokenizer);
bool readObject(Variant & root, Tokenizer & tokenizer);


const char * const g_hexdig = "0123456789ABCDEF";


std::string escapeString(const std::string & in)
{
    std::string out = "";

    for (unsigned char c : in)
    {
        if (c >= ' ' && c <= '~' && c != '\\' && c != '"')
        {
            out.append(1, c);
        }
        else
        {
            out = out + '\\';
            switch(c) {
                case '"':  out = out + "\"";  break;
                case '\\': out = out + "\\"; break;
                case '\t': out = out + "t";  break;
                case '\r': out = out + "r";  break;
                case '\n': out = out + "n";  break;
                default:
                    out = out + "x";
                    out.append(1, g_hexdig[c >> 4]);
                    out.append(1, g_hexdig[c & 0xF]);
                    break;
            }
        }
    }

    return out;
}

std::ostream & jsonStringify(std::ostream & stream, const Variant & root, bool beautify, const std::string & indent)
{
    // Variant is an object
    if (root.isVariantMap())
    {
        // Quick output: {} if empty
        if (root.asMap()->empty()){
            stream << "{}";
            return stream;
        }

        // Begin output
        stream << "{";

        if (beautify)
            stream << "\n";

        // Add all variables
        bool first = true;
        for (auto it : *root.asMap())
        {
            // Add separator (",")
            if (!first)
                stream << (beautify ? ",\n" : ",");
            else
                first = false;

            // Get variable
            const std::string & name       = it.first;
            const cppexpose::Variant & var = it.second;

            // Get value
            std::stringstream value;
            if (var.isVariantMap() || var.isVariantArray())
            {
                jsonStringify(value, var, beautify, indent + "    ");
            }
            else if (var.isNull())
            {
                value << "null";
            }
            else
            {
                std::stringstream unescaped;
                jsonStringify(unescaped, var, beautify, "");
                auto escaped = escapeString(unescaped.str());

                if (var.hasType<std::string>())
                    value << "\"" << escaped << "\"";
                else
                    value << escaped;
            }

            // Add value to JSON
            stream << (beautify ? (indent + "    \"" + name + "\": " + value.str()) : ("\"" + name + "\":" + value.str()));
        }

        // Finish JSON
        stream << (beautify ? "\n" + indent + "}" : "}");

        return stream;
    }

    // Variant is an array
    else if (root.isVariantArray())
    {
        // Quick output: [] if empty
        if (root.asArray()->empty())
        {
            stream << "[]";
            return stream;
        }

        // Begin output
        stream << "[";
        if (beautify) stream << "\n";

        // Add all elements
        bool first = true;
        for (const cppexpose::Variant & var : *root.asArray())
        {
            // Add separator (",")
            if (!first)
                stream << (beautify ? ",\n" : ",");
            else
                first = false;

            // Get value
            std::stringstream value;
            if (var.isVariantMap() || var.isVariantArray())
            {
                jsonStringify(value, var, beautify, indent + "    ");
            }
            else if (var.isNull())
            {
                value << "null";
            }
            else
            {
                std::stringstream unescaped;
                jsonStringify(unescaped, var, beautify, "");
                auto escaped = escapeString(unescaped.str());

                if (var.hasType<std::string>())
                    value << "\"" << escaped << "\"";
                else
                    value << escaped;
            }

            // Add value to JSON
            stream << (beautify ? (indent + "    " + value.str()) : value.str());
        }

        // Finish JSON
        stream << (beautify ? "\n" + indent + "]" : "]");

        return stream;
    }

    // Primitive data types
    else if (root.canConvert<std::string>())
    {
        stream << root.toString();
        return stream;
    }

    // Invalid type for JSON output
    else
    {
        stream << "null";
        return stream;
    }
}
<commit_msg>Refactor JSON stringification<commit_after>
#include <cppexpose/json/JSON.h>

#include <iostream>

#include <cppassist/string/conversion.h>
#include <cppassist/logging/logging.h>

#include <cppexpose/base/Tokenizer.h>
#include <cppexpose/variant/Variant.h>


using namespace cppassist;
using namespace cppexpose;


namespace
{


bool readValue(Variant & value, Tokenizer::Token & token, Tokenizer & tokenizer);
bool readArray(Variant & root, Tokenizer & tokenizer);
bool readObject(Variant & root, Tokenizer & tokenizer);


const char * const g_hexdig = "0123456789ABCDEF";


std::string escapeString(const std::string & in)
{
    std::string out = "";

    for (unsigned char c : in)
    {
        if (c >= ' ' && c <= '~' && c != '\\' && c != '"')
        {
            out.append(1, c);
        }
        else
        {
            out = out + '\\';
            switch(c) {
                case '"':  out = out + "\"";  break;
                case '\\': out = out + "\\"; break;
                case '\t': out = out + "t";  break;
                case '\r': out = out + "r";  break;
                case '\n': out = out + "n";  break;
                default:
                    out = out + "x";
                    out.append(1, g_hexdig[c >> 4]);
                    out.append(1, g_hexdig[c & 0xF]);
                    break;
            }
        }
    }

    return out;
}

std::string jsonStringifyPrimitive(const Variant & value)
{
    if (value.canConvert<std::string>())
    {
        return value.toString();
    }

    // Invalid type for JSON output
    return "null";
}

std::ostream & jsonStringify(std::ostream & stream, const Variant & root, bool beautify, const std::string & indent)
{
    // Variant is an object
    if (root.isVariantMap())
    {
        // Quick output: {} if empty
        if (root.asMap()->empty()){
            stream << "{}";
            return stream;
        }

        // Begin output
        stream << "{";

        if (beautify)
            stream << "\n";

        // Add all variables
        bool first = true;
        for (auto it : *root.asMap())
        {
            // Add separator (",")
            if (!first)
                stream << (beautify ? ",\n" : ",");
            else
                first = false;

            // Get variable
            const std::string & name       = it.first;
            const cppexpose::Variant & var = it.second;

            // Add variable name
            if (beautify)
                stream << indent << "    \"" << name << "\": ";
            else
                stream << "\"" << name << "\":";

            // Add variable value
            if (var.isVariantMap() || var.isVariantArray())
            {
                jsonStringify(stream, var, beautify, indent + "    ");
            }
            else
            {
                auto escaped = escapeString(jsonStringifyPrimitive(var));

                if (var.hasType<std::string>())
                    stream << "\"" << escaped << "\"";
                else
                    stream << escaped;
            }
        }

        // Finish JSON
        if (beautify)
            stream << "\n" << indent;

        stream << "}";

        return stream;
    }

    // Variant is an array
    else if (root.isVariantArray())
    {
        // Quick output: [] if empty
        if (root.asArray()->empty())
        {
            stream << "[]";
            return stream;
        }

        // Begin output
        stream << "[";

        if (beautify)
            stream << "\n";

        // Add all elements
        bool first = true;
        for (const cppexpose::Variant & var : *root.asArray())
        {
            // Add separator (",")
            if (!first)
                stream << (beautify ? ",\n" : ",");
            else
                first = false;

            // Add indent
            if (beautify)
                stream << indent << "    ";

            // Add next value
            if (var.isVariantMap() || var.isVariantArray())
            {
                jsonStringify(stream, var, beautify, indent + "    ");
            }
            else
            {
                auto escaped = escapeString(jsonStringifyPrimitive(var));

                if (var.hasType<std::string>())
                    stream << "\"" << escaped << "\"";
                else
                    stream << escaped;
            }
        }

        // Finish JSON
        if (beautify)
            stream << "\n" << indent;

        stream << "]";

        return stream;
    }

    // Primitive data types
    stream << jsonStringifyPrimitive(root);
    return stream;
}

{'fraction_non_alphanumeric': 0.14534641068447413, 'fraction_numerical': 0.0031302170283806345, 'mean_word_length': 1.2832301095759886, 'pattern_counts': {'":': 4, '<': 135, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2633166', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2625, 'n_words': 643}