<commit_before>/* Copyright (C) 2012-2015 <NAME>.
 *
 * See included LICENSE file for license details.
 */

#include "jsonsl.h"
#include <assert.h>
#include <limits.h>
#include <ctype.h>

#ifdef JSONSL_USE_METRICS
#define XMETRICS \
    X(STRINGY_INSIGNIFICANT) \
    X(STRINGY_SLOWPATH) \
    X(ALLOWED_WHITESPACE) \
    X(QUOTE_FASTPATH) \
    X(SPECIAL_FASTPATH) \
    X(SPECIAL_WSPOP) \
    X(SPECIAL_SLOWPATH) \
    X(GENERIC) \
    X(STRUCTURAL_TOKEN) \
    X(SPECIAL_SWITCHFIRST) \
    X(STRINGY_CATCH) \
    X(NUMBER_FASTPATH) \
    X(ESCAPES) \
    X(TOTAL) \

struct jsonsl_metrics_st {
#define X(m) \
    unsigned long metric_##m;
    XMETRICS
#undef X
};

static struct jsonsl_metrics_st GlobalMetrics = { 0 };
static unsigned long GenericCounter[0x100] = { 0 };
static unsigned long StringyCatchCounter[0x100] = { 0 };

#define INCR_METRIC(m) \
    GlobalMetrics.metric_##m++;

#define INCR_GENERIC(c) \
        INCR_METRIC(GENERIC); \
        GenericCounter[c]++; \

#define INCR_STRINGY_CATCH(c) \
    INCR_METRIC(STRINGY_CATCH); \
    StringyCatchCounter[c]++;

JSONSL_API
void jsonsl_dump_global_metrics(void)
{
    int ii;
    printf("JSONSL Metrics:\n");
#define X(m) \
    printf("\t%-30s %20lu (%0.2f%%)\n", #m, GlobalMetrics.metric_##m, \
           (float)((float)(GlobalMetrics.metric_##m/(float)GlobalMetrics.metric_TOTAL)) * 100);
    XMETRICS
#undef X
    printf("Generic Characters:\n");
    for (ii = 0; ii < 0xff; ii++) {
        if (GenericCounter[ii]) {
            printf("\t[ %c ] %lu\n", ii, GenericCounter[ii]);
        }
    }
    printf("Weird string loop\n");
    for (ii = 0; ii < 0xff; ii++) {
        if (StringyCatchCounter[ii]) {
            printf("\t[ %c ] %lu\n", ii, StringyCatchCounter[ii]);
        }
    }
}

#else
#define INCR_METRIC(m)
#define INCR_GENERIC(c)
#define INCR_STRINGY_CATCH(c)
JSONSL_API
void jsonsl_dump_global_metrics(void) { }
#endif /* JSONSL_USE_METRICS */

#define CASE_DIGITS \
case '1': \
case '2': \
case '3': \
case '4': \
case '5': \
case '6': \
case '7': \
case '8': \
case '9': \
case '0':

static unsigned extract_special(unsigned);
static int is_special_end(unsigned);
static int is_allowed_whitespace(unsigned);
static int is_allowed_escape(unsigned);
static int is_simple_char(unsigned);
static char get_escape_equiv(unsigned);

JSONSL_API
jsonsl_t jsonsl_new(int nlevels)
{
    struct jsonsl_st *jsn = (struct jsonsl_st *)
            calloc(1, sizeof (*jsn) +
                    ( (nlevels-1) * sizeof (struct jsonsl_state_st) )
            );

    jsn->levels_max = nlevels;
    jsn->max_callback_level = -1;
    jsonsl_reset(jsn);
    return jsn;
}

JSONSL_API
void jsonsl_reset(jsonsl_t jsn)
{
    unsigned int ii;
    jsn->tok_last = 0;
    jsn->can_insert = 1;
    jsn->pos = 0;
    jsn->level = 0;
    jsn->stopfl = 0;
    jsn->in_escape = 0;
    jsn->expecting = 0;

    memset(jsn->stack, 0, (jsn->levels_max * sizeof (struct jsonsl_state_st)));

    for (ii = 0; ii < jsn->levels_max; ii++) {
        jsn->stack[ii].level = ii;
    }
}

JSONSL_API
void jsonsl_destroy(jsonsl_t jsn)
{
    if (jsn) {
        free(jsn);
    }
}


#define FASTPARSE_EXHAUSTED 1
#define FASTPARSE_BREAK 0

/*
 * This function is meant to accelerate string parsing, reducing the main loop's
 * check if we are indeed a string.
 *
 * @param jsn the parser
 * @param[in,out] bytes_p A pointer to the current buffer (i.e. current position)
 * @param[in,out] nbytes_p A pointer to the current size of the buffer
 * @return true if all bytes have been exhausted (and thus the main loop can
 * return), false if a special character was examined which requires greater
 * examination.
 */
static int
jsonsl__str_fastparse(jsonsl_t jsn,<commit_msg>Optimized jsonsl_reset by removing code to reinitialize the stack<commit_after>/* Copyright (C) 2012-2015 <NAME>.
 *
 * See included LICENSE file for license details.
 */

#include "jsonsl.h"
#include <assert.h>
#include <limits.h>
#include <ctype.h>

#ifdef JSONSL_USE_METRICS
#define XMETRICS \
    X(STRINGY_INSIGNIFICANT) \
    X(STRINGY_SLOWPATH) \
    X(ALLOWED_WHITESPACE) \
    X(QUOTE_FASTPATH) \
    X(SPECIAL_FASTPATH) \
    X(SPECIAL_WSPOP) \
    X(SPECIAL_SLOWPATH) \
    X(GENERIC) \
    X(STRUCTURAL_TOKEN) \
    X(SPECIAL_SWITCHFIRST) \
    X(STRINGY_CATCH) \
    X(NUMBER_FASTPATH) \
    X(ESCAPES) \
    X(TOTAL) \

struct jsonsl_metrics_st {
#define X(m) \
    unsigned long metric_##m;
    XMETRICS
#undef X
};

static struct jsonsl_metrics_st GlobalMetrics = { 0 };
static unsigned long GenericCounter[0x100] = { 0 };
static unsigned long StringyCatchCounter[0x100] = { 0 };

#define INCR_METRIC(m) \
    GlobalMetrics.metric_##m++;

#define INCR_GENERIC(c) \
        INCR_METRIC(GENERIC); \
        GenericCounter[c]++; \

#define INCR_STRINGY_CATCH(c) \
    INCR_METRIC(STRINGY_CATCH); \
    StringyCatchCounter[c]++;

JSONSL_API
void jsonsl_dump_global_metrics(void)
{
    int ii;
    printf("JSONSL Metrics:\n");
#define X(m) \
    printf("\t%-30s %20lu (%0.2f%%)\n", #m, GlobalMetrics.metric_##m, \
           (float)((float)(GlobalMetrics.metric_##m/(float)GlobalMetrics.metric_TOTAL)) * 100);
    XMETRICS
#undef X
    printf("Generic Characters:\n");
    for (ii = 0; ii < 0xff; ii++) {
        if (GenericCounter[ii]) {
            printf("\t[ %c ] %lu\n", ii, GenericCounter[ii]);
        }
    }
    printf("Weird string loop\n");
    for (ii = 0; ii < 0xff; ii++) {
        if (StringyCatchCounter[ii]) {
            printf("\t[ %c ] %lu\n", ii, StringyCatchCounter[ii]);
        }
    }
}

#else
#define INCR_METRIC(m)
#define INCR_GENERIC(c)
#define INCR_STRINGY_CATCH(c)
JSONSL_API
void jsonsl_dump_global_metrics(void) { }
#endif /* JSONSL_USE_METRICS */

#define CASE_DIGITS \
case '1': \
case '2': \
case '3': \
case '4': \
case '5': \
case '6': \
case '7': \
case '8': \
case '9': \
case '0':

static unsigned extract_special(unsigned);
static int is_special_end(unsigned);
static int is_allowed_whitespace(unsigned);
static int is_allowed_escape(unsigned);
static int is_simple_char(unsigned);
static char get_escape_equiv(unsigned);

JSONSL_API
jsonsl_t jsonsl_new(int nlevels)
{
    unsigned int ii;
    struct jsonsl_st *jsn = (struct jsonsl_st *)
            calloc(1, sizeof (*jsn) +
                    ( (nlevels-1) * sizeof (struct jsonsl_state_st) )
            );

    jsn->levels_max = nlevels;
    jsn->max_callback_level = -1;
    jsonsl_reset(jsn);
    for (ii = 0; ii < jsn->levels_max; ii++) {
        jsn->stack[ii].level = ii;
    }
    return jsn;
}

JSONSL_API
void jsonsl_reset(jsonsl_t jsn)
{
    jsn->tok_last = 0;
    jsn->can_insert = 1;
    jsn->pos = 0;
    jsn->level = 0;
    jsn->stopfl = 0;
    jsn->in_escape = 0;
    jsn->expecting = 0;
}

JSONSL_API
void jsonsl_destroy(jsonsl_t jsn)
{
    if (jsn) {
        free(jsn);
    }
}


#define FASTPARSE_EXHAUSTED 1
#define FASTPARSE_BREAK 0

/*
 * This function is meant to accelerate string parsing, reducing the main loop's
 * check if we are indeed a string.
 *
 * @param jsn the parser
 * @param[in,out] bytes_p A pointer to the current buffer (i.e. current position)
 * @param[in,out] nbytes_p A pointer to the current size of the buffer
 * @return true if all bytes have been exhausted (and thus the main loop can
 * return), false if a special character was examined which requires greater
 * examination.
 */
static int
jsonsl__str_fastparse(jsonsl_t jsn,
{'fraction_non_alphanumeric': 0.1343969713358572, 'fraction_numerical': 0.015008112493239589, 'mean_word_length': 3.226857142857143, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2286106', 'n_tokens_mistral': 2923, 'n_tokens_neox': 2808, 'n_words': 645}