<commit_before>/* Lua JSON routines
 */

/* Caveats:
 * - Assumes strings are valid UTF-8 and mostly treats them as opaque
 *   binary data. Will not throw an exception on bad data.
 * - Will decode \uXXXX escapes, but leaves high codepoints as UTF-8
 *   when encoding.
 * - JSON "null" values are represented as lightuserdata. Compare with
 *   json.null.
 * - Parsing comments is not supported. According to json.org, this isn't
 *   part of the spec.
 * - Parser accepts number formats beyond the JSON spec.
 *
 * Note: lua_json_decode() probably spends significant time rehashing
 *       tables since it is difficult to know their size ahead of time.
 *       Earlier JSON libaries didn't have this problem but the intermediate
 *       storage (and their implementations) were much slower anyway..
 */

/* FIXME:
 * - Option to encode non-printable characters? Only \" \\ are required
 * - Protect against cycles when encoding JSON from a data structure
 *   - Max depth? Notice cycles?
 * - Handle huge sparse arrays?
 */

#include <assert.h>
#include <string.h>
#include <math.h>

#include <pthread.h>

#include <lua.h>
#include <lauxlib.h>

#include "strbuf.h"

/* ===== ENCODING ===== */

static void json_encode_exception(lua_State *l, strbuf_t *json,
                                  char *location, int lindex)
                                  
{
    strbuf_free(json);

    luaL_error(l, "Cannot serialise %s: %s", location,
               lua_typename(l, lua_type(l, lindex)));
}

/* JSON escape a character if required, or return NULL */
static inline char *json_escape_char(int c)
{
    switch(c) {
    case 0:
        return "\\u0000";
    case '\\':
        return "\\\\";
    case '"':
        return "\\\"";
    case '\b':
        return "\\b";
    case '\t':
        return "\\t";
    case '\n':
        return "\\n";
    case '\f':
        return "\\f";
    case '\r':
        return "\\r";
    }

    return NULL;
}

/* json_append_string args:
 * - lua_State
 * - JSON strbuf
 * - String (Lua stack index)
 *
 * Returns nothing. Doesn't remove string from Lua stack */
static void json_append_string(lua_State *l, strbuf_t *json, int lindex)
{
    char *p;
    int i;
    const char *str;
    size_t len;

    str = lua_tolstring(l, lindex, &len);

    /* Worst case is len * 6 (all unicode escapes).
     * This buffer is reused constantly for small strings
     * If there are any excess pages, they won't be hit anyway.
     * This gains ~5% speedup. */
    strbuf_ensure_empty_length(json, len * 6);

    strbuf_append_char(json, '\"');
    for (i = 0; i < len; i++) {
        p = json_escape_char(str[i]);
        if (p)
            strbuf_append_string(json, p);
        else
            strbuf_append_char_unsafe(json, str[i]);
    }
    strbuf_append_char(json, '\"');
}

/* Find the size of the array on the top of the Lua stack
 * -1   object (not a pure array)
 * >=0  elements in array
 */
static int lua_array_length(lua_State *l)
{
    double k;
    int max;

    max = 0;

    lua_pushnil(l);
    /* table, startkey */
    while (lua_next(l, -2) != 0) {
        /* table, key, value */
        if (lua_isnumber(l, -2) &&
            (k = lua_tonumber(l, -2))) {
            /* Integer >= 1 ? */
            if (floor(k) == k && k >= 1) {
                if (k > max)
                    max = k;
                lua_pop(l, 1);
                continue;
            }
        }

        /* Must not be an array (non integer key) */
        lua_pop(l, 2);
        return -1;
    }

    return max;
}

static void json_append_data(lua_State *l, strbuf_t *json);

/* json_append_array args:
 * - lua_State
 * - JSON strbuf
 * - Size of passwd Lua array (top of stack) */
static void json_append_array(lua_State *l, strbuf_t *json, int array_length)
{
    int comma, i;

    strbuf_append_string(json, "[ ");

    comma = 0;
    for (i = 1; i <= array_length; i++) {
        if (comma)
            strbuf_append_string(json, ", ");<commit_msg>Encode very sparse arrays as objects<commit_after>/* Lua JSON routines
 */

/* Caveats:
 * - Assumes strings are valid UTF-8 and mostly treats them as opaque
 *   binary data. Will not throw an exception on bad data.
 * - Will decode \uXXXX escapes, but leaves high codepoints as UTF-8
 *   when encoding.
 * - JSON "null" values are represented as lightuserdata. Compare with
 *   json.null.
 * - Parsing comments is not supported. According to json.org, this isn't
 *   part of the spec.
 * - Parser accepts number formats beyond the JSON spec.
 *
 * Note: lua_json_decode() probably spends significant time rehashing
 *       tables since it is difficult to know their size ahead of time.
 *       Earlier JSON libaries didn't have this problem but the intermediate
 *       storage (and their implementations) were much slower anyway..
 */

/* FIXME:
 * - Option to encode non-printable characters? Only \" \\ are required
 * - Protect against cycles when encoding JSON from a data structure
 *   - Max depth? Notice cycles?
 */

#include <assert.h>
#include <string.h>
#include <math.h>

#include <pthread.h>

#include <lua.h>
#include <lauxlib.h>

#include "strbuf.h"

/* Encode very sparse arrays as objects */
#ifndef VERY_SPARSE_ARRAY_RATIO
#define VERY_SPARSE_ARRAY_RATIO 2
#endif

/* ===== ENCODING ===== */

static void json_encode_exception(lua_State *l, strbuf_t *json,
                                  char *location, int lindex)
                                  
{
    strbuf_free(json);

    luaL_error(l, "Cannot serialise %s: %s", location,
               lua_typename(l, lua_type(l, lindex)));
}

/* JSON escape a character if required, or return NULL */
static inline char *json_escape_char(int c)
{
    switch(c) {
    case 0:
        return "\\u0000";
    case '\\':
        return "\\\\";
    case '"':
        return "\\\"";
    case '\b':
        return "\\b";
    case '\t':
        return "\\t";
    case '\n':
        return "\\n";
    case '\f':
        return "\\f";
    case '\r':
        return "\\r";
    }

    return NULL;
}

/* json_append_string args:
 * - lua_State
 * - JSON strbuf
 * - String (Lua stack index)
 *
 * Returns nothing. Doesn't remove string from Lua stack */
static void json_append_string(lua_State *l, strbuf_t *json, int lindex)
{
    char *p;
    int i;
    const char *str;
    size_t len;

    str = lua_tolstring(l, lindex, &len);

    /* Worst case is len * 6 (all unicode escapes).
     * This buffer is reused constantly for small strings
     * If there are any excess pages, they won't be hit anyway.
     * This gains ~5% speedup. */
    strbuf_ensure_empty_length(json, len * 6);

    strbuf_append_char(json, '\"');
    for (i = 0; i < len; i++) {
        p = json_escape_char(str[i]);
        if (p)
            strbuf_append_string(json, p);
        else
            strbuf_append_char_unsafe(json, str[i]);
    }
    strbuf_append_char(json, '\"');
}

/* Find the size of the array on the top of the Lua stack
 * -1   object (not a pure array)
 * >=0  elements in array
 */
static int lua_array_length(lua_State *l)
{
    double k;
    int max;
    int items;

    max = 0;
    items = 0;

    lua_pushnil(l);
    /* table, startkey */
    while (lua_next(l, -2) != 0) {
        /* table, key, value */
        if (lua_isnumber(l, -2) &&
            (k = lua_tonumber(l, -2))) {
            /* Integer >= 1 ? */
            if (floor(k) == k && k >= 1) {
                if (k > max)
                    max = k;
                items++;
                lua_pop(l, 1);
                continue;
            }
        }

        /* Must not be an array (non integer key) */
        lua_pop(l, 2);
        return -1;
    }

#ifdef VERY_SPARSE_ARRAY_RATIO
    /* Encode very sparse arrays as objects */
    if (max > items * VERY_SPARSE_ARRAY_RATIO)
        return -1;
#endif

    return max;
}

static void json_append_data(lua_State *l, strbuf_t *json);

/* json_append_array args:
 * - lua_State
 * - JSON strbuf
 * - Size of passwd Lua array (top of stack) */
static void json_append_array(lua_State *l, strbuf_t *json, int array_length)
{
    int comma, i;

    strbuf_append_string(json, "[ ");

    comma = 0;
    for (i = 1; i <= array_length; i++) {
        if (comma)
            strbuf_append_string(json, ", ");
{'fraction_non_alphanumeric': 0.12444122266521687, 'fraction_numerical': 0.00640328621481213, 'mean_word_length': 2.1356060606060607, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3454797', 'n_tokens_mistral': 2784, 'n_tokens_neox': 2632, 'n_words': 881}