<commit_before>#include "kjson.h"
#include <unistd.h>
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include "benchmark.h"

#define SIZE 1024*16
static char *loadLine(FILE *fp, char *buf)
{
    bzero(buf, SIZE);
    return fgets(buf, SIZE, fp);
}
#define JSON_FILE "./test/twitter_public.json"
static int filesize = 0;
static void test0()
{
    size_t len;
    char buf[SIZE], *str;
    FILE *fp = fopen(JSON_FILE, "r");
    reset_timer();
    while ((str = loadLine(fp, buf)) != NULL) {
        len = strlen(str);
        filesize += len;
    }
    show_timer("nop");
    fclose(fp);
}

static void test1()
{
    size_t len;
    char buf[SIZE], *str;
    FILE *fp = fopen(JSON_FILE, "r");
    reset_timer();
    JSONMemoryPool jm;
    JSONMemoryPool_Init(&jm);
    while ((str = loadLine(fp, buf)) != NULL) {
        len = strlen(str);
        JSON json = parseJSON(&jm, str, str+len);
        assert(JSON_type(json) == JSON_Object);
        JSON_free(json);
    }
    JSONMemoryPool_Delete(&jm);
    _show_timer("parse", filesize);
    fclose(fp);
}

static void test2()
{
    size_t len;
    char buf[SIZE], *str;
    FILE *fp = fopen(JSON_FILE, "r");
    reset_timer();
    JSONMemoryPool jm;
    JSONMemoryPool_Init(&jm);
    while ((str = loadLine(fp, buf)) != NULL) {
        len = strlen(str);
        JSON json = parseJSON(&jm, str, str+len);
        char *p = JSON_toStringWithLength(json, &len);
        assert(JSON_type(json) == JSON_Object);
        assert(p);
        JSON_free(json);
        free(p);
    }
    JSONMemoryPool_Delete(&jm);
    _show_timer("pack/unpack", filesize);
    fclose(fp);
}

int main(int argc, char const* argv[])
{
    int i, size = 1;
    fprintf(stderr, "benchmark3\n");
    for (i = 0; i < size; i++) {
        test0();
        test1();
        test2();
    }
    return 0;
}<commit_msg>update benchmark3.c<commit_after>#include "kjson.h"
#include <unistd.h>
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include "benchmark.h"

#define SIZE 1024*128
static char *loadLine(FILE *fp, char *buf)
{
    bzero(buf, SIZE);
    return fgets(buf, SIZE, fp);
}
#define JSON_FILE "./test/twitter_public.json"
static int filesize = 0;
static void test0()
{
    size_t len;
    char buf[SIZE], *str;
    FILE *fp = fopen(JSON_FILE, "r");
    reset_timer();
    while ((str = loadLine(fp, buf)) != NULL) {
        len = strlen(str);
        filesize += len;
    }
    show_timer("nop");
    fclose(fp);
}

static void test1()
{
    size_t len;
    char buf[SIZE], *str;
    FILE *fp = fopen(JSON_FILE, "r");
    reset_timer();
    JSONMemoryPool jm;
    JSONMemoryPool_Init(&jm);
    while ((str = loadLine(fp, buf)) != NULL) {
        len = strlen(str);
        JSON json = parseJSON(&jm, str, str+len);
        assert(JSON_type(json) == JSON_Array);
        JSON_free(json);
    }
    JSONMemoryPool_Delete(&jm);
    show_timer("parse");
    fclose(fp);
}

static void test2()
{
    size_t len;
    char buf[SIZE], *str;
    FILE *fp = fopen(JSON_FILE, "r");
    reset_timer();
    JSONMemoryPool jm;
    JSONMemoryPool_Init(&jm);
    while ((str = loadLine(fp, buf)) != NULL) {
        len = strlen(str);
        JSON json = parseJSON(&jm, str, str+len);
        char *p = JSON_toStringWithLength(json, &len);
        assert(JSON_type(json) == JSON_Array);
        assert(p);
        JSON_free(json);
        free(p);
    }
    JSONMemoryPool_Delete(&jm);
    show_timer("pack/unpack");
    fclose(fp);
}

int main(int argc, char const* argv[])
{
    int i, size = 8;
    fprintf(stderr, "benchmark3\n");
    test0();
    for (i = 0; i < size; i++) {
        test1();
    }
    for (i = 0; i < size; i++) {
        test2();
    }

    return 0;
}
{'fraction_non_alphanumeric': 0.16473442976543543, 'fraction_numerical': 0.009975734699379887, 'mean_word_length': 2.7399193548387095, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1828637', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1347, 'n_words': 319}