<commit_before>#include "yajl_ext.h"

// Helpers for building objects
inline void yajl_check_and_fire_callback(void * ctx) {
    struct yajl_parser_wrapper * wrapper;
    GetParser((VALUE)ctx, wrapper);
    
    // No need to do any of this if the callback isn't even setup
    if (wrapper->parse_complete_callback != Qnil) {
        int len = RARRAY_LEN(wrapper->builderStack);
        if (len == 1 && wrapper->nestedArrayLevel == 0 && wrapper->nestedHashLevel == 0) {
            rb_funcall(wrapper->parse_complete_callback, intern_call, 1, rb_ary_pop(wrapper->builderStack));
        }
    } else {
        int len = RARRAY_LEN(wrapper->builderStack);
        if (len == 1 && wrapper->nestedArrayLevel == 0 && wrapper->nestedHashLevel == 0) {
            wrapper->objectsFound++;
            if (wrapper->objectsFound > 1) {
                rb_raise(cParseError, "%s", "Found multiple JSON objects in the stream but no block or the on_parse_complete callback was assigned to handle them.");
            }
        }
    }
}

inline void yajl_set_static_value(void * ctx, VALUE val) {
    struct yajl_parser_wrapper * wrapper;
    VALUE lastEntry, hash;
    int len;
    
    GetParser((VALUE)ctx, wrapper);
    
    len = RARRAY_LEN(wrapper->builderStack);
    if (len > 0) {
        lastEntry = rb_ary_entry(wrapper->builderStack, len-1);
        switch (TYPE(lastEntry)) {
            case T_ARRAY:
                rb_ary_push(lastEntry, val);
                if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
                    rb_ary_push(wrapper->builderStack, val);
                }
                break;
            case T_HASH:
                rb_hash_aset(lastEntry, val, Qnil);
                rb_ary_push(wrapper->builderStack, val);
                break;
            case T_STRING:
            case T_SYMBOL:
                hash = rb_ary_entry(wrapper->builderStack, len-2);
                if (TYPE(hash) == T_HASH) {
                    rb_hash_aset(hash, lastEntry, val);
                    rb_ary_pop(wrapper->builderStack);
                    if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
                        rb_ary_push(wrapper->builderStack, val);
                    }
                }
                break;
        }
    } else {
        rb_ary_push(wrapper->builderStack, val);
    }
}

void yajl_encode_part(yajl_gen hand, VALUE obj, VALUE io) {
    VALUE str, outBuff, otherObj;
    int objLen;
    int idx = 0;
    const unsigned char * buffer;
    unsigned int len;
    
    if (io != Qnil) {
        yajl_gen_get_buf(hand, &buffer, &len);
        if (len >= WRITE_BUFSIZE) {
            outBuff = rb_str_new((const char *)buffer, len);
            rb_io_write(io, outBuff);
            yajl_gen_clear(hand);
        }
    }
    
    switch (TYPE(obj)) {
        case T_HASH:
            yajl_gen_map_open(hand);
            
            // TODO: itterate through keys in the hash
            VALUE keys = rb_funcall(obj, intern_keys, 0);
            VALUE entry;
            for(idx=0; idx<RARRAY_LEN(keys); idx++) {
                entry = rb_ary_entry(keys, idx);
                // the key
                yajl_encode_part(hand, entry, io);
                // the value
                yajl_encode_part(hand, rb_hash_aref(obj, entry), io);
            }
            
            yajl_gen_map_close(hand);
            break;
        case T_ARRAY:
            yajl_gen_array_open(hand);
            for(idx=0; idx<RARRAY_LEN(obj); idx++) {
                otherObj = rb_ary_entry(obj, idx);
                yajl_encode_part(hand, otherObj, io);
            }
            yajl_gen_array_close(hand);
            break;
        case T_NIL:
            yajl_gen_null(hand);
            break;
        case T_TRUE:
            yajl_gen_bool(hand, 1);
            break;
        case T_FALSE:
            yajl_gen_bool(hand, 0);
            break;
        case T_FIXNUM:
        case T_FLOAT:
        case T_BIGNUM:
            str = rb_funcall(obj, intern_to_s, 0);
            objLen = RSTRING_LEN(str);
            yajl_gen_number(hand, RSTRING_PTR(str), (unsigned int)objLen);
            break;
        default:
            str = rb_funcall(obj, intern_to_s, 0);
            objLen = RSTRING_LEN(str);
            yajl_gen_string(hand, (const unsigned char *)RSTRING_PTR(str), (unsigned int)objLen);
            break;
    }
}

void yajl_parser_wrapper_free(void * wrapper) {
    struct yajl_parser_wrapper * w = wrapper;
    yajl_free(w->parser);
    free(w);
}

void yajl_parser_wrapper_mark(void * wrapper) {
    struct yajl_parser_wrapper * w = wrapper;
    rb_gc_mark(w->builderStack);
    rb_gc_mark(w->parse_complete_callback);
}

void yajl_parse_chunk(const unsigned char * chunk, unsigned int len, yajl_handle parser) {
    yajl_status stat;
    
    stat = yajl_parse(parser, chunk, len);
    
    if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) {
        unsigned char * str = yajl_get_error(parser, 1, chunk, len);
        rb_raise(cParseError, "%s", (const char *) str);
        yajl_free_error(parser, str);
    }
}

// YAJL Callbacks
static int yajl_found_null(void * ctx) {
    yajl_set_static_value(ctx, Qnil);
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_boolean(void * ctx, int boolean) {
    yajl_set_static_value(ctx, boolean ? Qtrue : Qfalse);
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_number(void * ctx, const char * numberVal, unsigned int numberLen) {
    VALUE subString = rb_str_new(numberVal, numberLen);
    char * cSubString = RSTRING_PTR(subString);
    
    if (strstr(cSubString, ".") != NULL ||
        strstr(cSubString, "e") != NULL ||
        strstr(cSubString, "E") != NULL) {
            yajl_set_static_value(ctx, rb_Float(subString));
    } else {
        yajl_set_static_value(ctx, rb_Integer(subString));
    }
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
    yajl_set_static_value(ctx, rb_str_new((const char *)stringVal, stringLen));
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_hash_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
    struct yajl_parser_wrapper * wrapper;
    GetParser((VALUE)ctx, wrapper);
    
    if (wrapper->symbolizeKeys) {
        char keyStr[stringLen];
        ID key;
        sprintf(keyStr, "%.*s", stringLen, stringVal);
        key = rb_intern(keyStr);
        yajl_set_static_value(ctx, ID2SYM(key));
    } else {
        yajl_set_static_value(ctx, rb_str_new((const char *)stringVal, stringLen));
    }
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_start_hash(void * ctx) {
    struct yajl_parser_wrapper * wrapper;
    GetParser((VALUE)ctx, wrapper);
    wrapper->nestedHashLevel++;<commit_msg>remove use of sprintf for a little speed improvement in parsing<commit_after>#include "yajl_ext.h"

// Helpers for building objects
inline void yajl_check_and_fire_callback(void * ctx) {
    struct yajl_parser_wrapper * wrapper;
    GetParser((VALUE)ctx, wrapper);
    
    // No need to do any of this if the callback isn't even setup
    if (wrapper->parse_complete_callback != Qnil) {
        int len = RARRAY_LEN(wrapper->builderStack);
        if (len == 1 && wrapper->nestedArrayLevel == 0 && wrapper->nestedHashLevel == 0) {
            rb_funcall(wrapper->parse_complete_callback, intern_call, 1, rb_ary_pop(wrapper->builderStack));
        }
    } else {
        int len = RARRAY_LEN(wrapper->builderStack);
        if (len == 1 && wrapper->nestedArrayLevel == 0 && wrapper->nestedHashLevel == 0) {
            wrapper->objectsFound++;
            if (wrapper->objectsFound > 1) {
                rb_raise(cParseError, "%s", "Found multiple JSON objects in the stream but no block or the on_parse_complete callback was assigned to handle them.");
            }
        }
    }
}

inline void yajl_set_static_value(void * ctx, VALUE val) {
    struct yajl_parser_wrapper * wrapper;
    VALUE lastEntry, hash;
    int len;
    
    GetParser((VALUE)ctx, wrapper);
    
    len = RARRAY_LEN(wrapper->builderStack);
    if (len > 0) {
        lastEntry = rb_ary_entry(wrapper->builderStack, len-1);
        switch (TYPE(lastEntry)) {
            case T_ARRAY:
                rb_ary_push(lastEntry, val);
                if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
                    rb_ary_push(wrapper->builderStack, val);
                }
                break;
            case T_HASH:
                rb_hash_aset(lastEntry, val, Qnil);
                rb_ary_push(wrapper->builderStack, val);
                break;
            case T_STRING:
            case T_SYMBOL:
                hash = rb_ary_entry(wrapper->builderStack, len-2);
                if (TYPE(hash) == T_HASH) {
                    rb_hash_aset(hash, lastEntry, val);
                    rb_ary_pop(wrapper->builderStack);
                    if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
                        rb_ary_push(wrapper->builderStack, val);
                    }
                }
                break;
        }
    } else {
        rb_ary_push(wrapper->builderStack, val);
    }
}

void yajl_encode_part(yajl_gen hand, VALUE obj, VALUE io) {
    VALUE str, outBuff, otherObj;
    int objLen;
    int idx = 0;
    const unsigned char * buffer;
    unsigned int len;
    
    if (io != Qnil) {
        yajl_gen_get_buf(hand, &buffer, &len);
        if (len >= WRITE_BUFSIZE) {
            outBuff = rb_str_new((const char *)buffer, len);
            rb_io_write(io, outBuff);
            yajl_gen_clear(hand);
        }
    }
    
    switch (TYPE(obj)) {
        case T_HASH:
            yajl_gen_map_open(hand);
            
            // TODO: itterate through keys in the hash
            VALUE keys = rb_funcall(obj, intern_keys, 0);
            VALUE entry;
            for(idx=0; idx<RARRAY_LEN(keys); idx++) {
                entry = rb_ary_entry(keys, idx);
                // the key
                yajl_encode_part(hand, entry, io);
                // the value
                yajl_encode_part(hand, rb_hash_aref(obj, entry), io);
            }
            
            yajl_gen_map_close(hand);
            break;
        case T_ARRAY:
            yajl_gen_array_open(hand);
            for(idx=0; idx<RARRAY_LEN(obj); idx++) {
                otherObj = rb_ary_entry(obj, idx);
                yajl_encode_part(hand, otherObj, io);
            }
            yajl_gen_array_close(hand);
            break;
        case T_NIL:
            yajl_gen_null(hand);
            break;
        case T_TRUE:
            yajl_gen_bool(hand, 1);
            break;
        case T_FALSE:
            yajl_gen_bool(hand, 0);
            break;
        case T_FIXNUM:
        case T_FLOAT:
        case T_BIGNUM:
            str = rb_funcall(obj, intern_to_s, 0);
            objLen = RSTRING_LEN(str);
            yajl_gen_number(hand, RSTRING_PTR(str), (unsigned int)objLen);
            break;
        default:
            str = rb_funcall(obj, intern_to_s, 0);
            objLen = RSTRING_LEN(str);
            yajl_gen_string(hand, (const unsigned char *)RSTRING_PTR(str), (unsigned int)objLen);
            break;
    }
}

void yajl_parser_wrapper_free(void * wrapper) {
    struct yajl_parser_wrapper * w = wrapper;
    yajl_free(w->parser);
    free(w);
}

void yajl_parser_wrapper_mark(void * wrapper) {
    struct yajl_parser_wrapper * w = wrapper;
    rb_gc_mark(w->builderStack);
    rb_gc_mark(w->parse_complete_callback);
}

void yajl_parse_chunk(const unsigned char * chunk, unsigned int len, yajl_handle parser) {
    yajl_status stat;
    
    stat = yajl_parse(parser, chunk, len);
    
    if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) {
        unsigned char * str = yajl_get_error(parser, 1, chunk, len);
        rb_raise(cParseError, "%s", (const char *) str);
        yajl_free_error(parser, str);
    }
}

// YAJL Callbacks
static int yajl_found_null(void * ctx) {
    yajl_set_static_value(ctx, Qnil);
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_boolean(void * ctx, int boolean) {
    yajl_set_static_value(ctx, boolean ? Qtrue : Qfalse);
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_number(void * ctx, const char * numberVal, unsigned int numberLen) {
    VALUE subString = rb_str_new(numberVal, numberLen);
    char * cSubString = RSTRING_PTR(subString);
    
    if (strstr(cSubString, ".") != NULL || strstr(cSubString, "e") != NULL || strstr(cSubString, "E") != NULL) {
            yajl_set_static_value(ctx, rb_Float(subString));
    } else {
        yajl_set_static_value(ctx, rb_Integer(subString));
    }
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
    yajl_set_static_value(ctx, rb_str_new((const char *)stringVal, stringLen));
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_hash_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
    struct yajl_parser_wrapper * wrapper;
    GetParser((VALUE)ctx, wrapper);
    VALUE keyStr = rb_str_new((const char *)stringVal, stringLen);
    
    if (wrapper->symbolizeKeys) {
        ID key = rb_intern(RSTRING_PTR(keyStr));
        yajl_set_static_value(ctx, ID2SYM(key));
    } else {
        yajl_set_static_value(ctx, keyStr);
    }
    yajl_check_and_fire_callback(ctx);
    return 1;
}

static int yajl_found_start_hash(void * ctx) {
    struct yajl_parser_wrapper * wrapper;
    GetParser((VALUE)ctx, wrapper);
    wrapper->nestedHashLevel++;
{'fraction_non_alphanumeric': 0.10596841952774157, 'fraction_numerical': 0.003766478342749529, 'mean_word_length': 2.238798967863007, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4773308', 'n_tokens_mistral': 4577, 'n_tokens_neox': 4472, 'n_words': 943}