<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <getopt.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <assert.h>
#include <sys/stat.h>
#include <sys/utsname.h>

#include <hiredis.h>
#include <darray.h>
#include <dutil.h>
#include <dlog.h>

#include <vrt_util.h>
#include <vrt_public.h>
#include <vrabtest.h>
#include <vrt_produce_data.h>

typedef struct produce_scheme {
    long long ckeys_write_idx;
    long long max_ckeys_count;    /* Max keys count that can be cached in the ckeys array. */
    sds *ckeys;    /* Cached keys that may exist in the target redis/vire servers. */

    int hit_ratio;   /* Hit ratio for the read commands. [0%,100%] */
    int hit_ratio_idx;   /* [0,hit_ratio_array_len-1] */
    int hit_ratio_array_len; /* 100 usually */
    int *hit_ratio_array;    /* Stored 0 or 1 for every element, 1 means used key in the cached keys array. */
} produce_scheme;

typedef struct produce_thread {
    int id;
    pthread_t thread_id;

    produce_scheme *ps;

    int pause;
    long long looptimes;
} produce_thread;

data_producer *delete_data_producer = NULL;

static unsigned int key_length_min;
static unsigned int key_length_max;
static unsigned int key_length_range_gap;
static unsigned int field_length_max;
static unsigned int string_length_max;

static int cmd_type;

static darray needed_cmd_type_producer;  /* type:  data_producer*/
static unsigned int needed_cmd_type_producer_count;

int produce_data_threads_count;
static darray produce_threads;

int produce_threads_pause_finished_count;

static sds get_random_cached_key(produce_scheme *ps)
{
    int idx;
    sds key;

    key = ps->ckeys[0];
    if (sdslen(key) == 0) {
        return NULL;
    }

    key = ps->ckeys[ps->max_ckeys_count-1];
    if (sdslen(key) == 0) {
        idx = rand()%ps->ckeys_write_idx;
    } else {
        idx = rand()%ps->max_ckeys_count;
    }
    
    return ps->ckeys[idx];
}

static int get_random_int(void)
{
    if (rand()%2 == 1) {
        return 0 - (int)rand();
    } else {
        return (int)rand();
    }
}

static unsigned int get_random_unsigned_int(void)
{
    return (unsigned int)rand();
}

static char get_random_char(void)
{
    return (char)rand()%250 + 5;
}

static sds get_random_key(void)
{
    unsigned int i, len;
    sds str = sdsempty();
    
    len = key_length_range_gap==0?key_length_min:
        (get_random_unsigned_int()%key_length_range_gap+key_length_min);
    if (len == 0) len ++;
    str = sdsMakeRoomFor(str,(size_t)len);
    sdsIncrLen(str, (int)len);

    for (i = 0; i < len; i ++) {
        str[i] = (char)get_random_char();
    }

    return str;
}

static sds get_random_string(void)
{
    unsigned int i, len;
    sds str = sdsempty();
    
    len = get_random_unsigned_int()%string_length_max;
    str = sdsMakeRoomFor(str,(size_t)len);
    sdsIncrLen(str, (int)len);

    for (i = 0; i < len; i ++) {
        str[i] = get_random_char();
    }

    return str;
}

static sds get_random_score_str(void)
{
    unsigned int decimal_len;
    sds str;

    if (rand()%2 == 1) {
        str = sdsnew("-");
    } else {
        str = sdsempty();
    }

    if (rand()%2 == 1) {
        str = sdscatfmt(str,"%u.%u",
            get_random_unsigned_int(),
            get_random_unsigned_int());
    } else {
        str = sdscatfmt(str,"%u",
            get_random_unsigned_int());
    }

    return str;
}

static unsigned int get_random_field_len(void)
{
    return get_random_unsigned_int()%field_length_max + 1;
}

static sds get_random_key_with_hit_ratio(produce_scheme *ps)
{
    sds key;
    if (ps->hit_ratio_array[ps->hit_ratio_idx++] == 0 || 
        ps->ckeys[ps->ckeys_write_idx] == NULL) {
        key = get_random_key();
    } else {
        key = get_random_cached_key(ps);
        if (key == NULL) key = get_random_key();
        else key = sdsdup(key);
    }
    if (ps->hit_ratio_idx >= ps->hit_ratio_array_len) {
        ps->hit_ratio_idx = 0;
    }
    return key;
}

static data_unit *get_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *set_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = get_random_string();
    
    return du;
}

static data_unit *setnx_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = get_random_string();
    
    return du;
}

static data_unit *setex_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 4;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    du->argv[3] = get_random_string();
    
    return du;
}

static data_unit *psetex_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 4;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    du->argv[3] = get_random_string();
    
    return du;
}

static data_unit *del_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    
    return du;
}

static data_unit *expire_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = sdsfromlonglong(rand()%10000);
    
    return du;
}

static data_unit *expireat_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = sdsfromlonglong(vrt_msec_now()/1000LL+rand()%10000);
    
    return du;
}

static data_unit *exists_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *ttl_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *pttl_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *incr_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    
    return du;
}

static data_unit *decr_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    
    return du;
}

static data_unit *incrby_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    
    return du;
}

static data_unit *decrby_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    
    return du;
}

static data_unit *append_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = get_random_string();
    
    return du;
}

static data_unit *strlen_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *rpush_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    field_length = get_random_field_len();

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2+field_length;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();

    for (j = 0; j < field_length; j ++) {
        du->argv[2+j] = get_random_string();
    }
    
    return du;
}

static data_unit *lpush_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    field_length = get_random_field_len();

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2+field_length;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();

    for (j = 0; j < field_length; j ++) {
        du->argv[2+j] = get_random_string();
    }
    
    return du;
}

static data_unit *zadd_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    field_length = get_random_field_len();

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2+field_length*2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();

    for (j = 2; j < 2+field_length*2; j += 2) {
        du->argv[j] = get_random_score_str();
        du->argv[j+1] = get_random_string();
    }
    
    return du;
}

static data_unit *zincrby_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 4;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = get_random_score_str();;
    du->argv[3] = get_random_string();
    
    return du;
}

static int producers_count;
data_producer redis_data_producer_table[] = {
    /* Key */
    {"del",del_cmd_producer,-2,"w",0,NULL,1,-1,1,TEST_CMD_TYPE_KEY},
    {"exists",exists_cmd_producer,-2,"rF",0,NULL,1,-1,1,TEST_CMD_TYPE_KEY},
    {"ttl",ttl_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"pttl",pttl_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"expire",expire_cmd_producer,3,"wF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"expireat",expireat_cmd_producer,3,"wF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    /* String */
    {"get",get_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"set",set_cmd_producer,-3,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"setnx",setnx_cmd_producer,3,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"setex",setex_cmd_producer,4,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"psetex",psetex_cmd_producer,4,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"incr",incr_cmd_producer,2,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"decr",decr_cmd_producer,2,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"incrby",incrby_cmd_producer,3,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"decrby",decrby_cmd_producer,3,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"append",append_cmd_producer,3,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"strlen",strlen_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    /* List */
    {"rpush",rpush_cmd_producer,-3,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_LIST},
    {"lpush",lpush_cmd_producer,-3,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_LIST},
    /* SortedSet */
    {"zadd",zadd_cmd_producer,-4,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_ZSET},
    {"zincrby",zincrby_cmd_producer,4,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_ZSET}
};

data_unit *data_unit_get(void)
{
    data_unit *du = malloc(sizeof(data_unit));
    du->dp = NULL;
    du->argc = 0;
    du->argv = NULL;
    du->hashvalue = 0;
    return du;
}

void data_unit_put(data_unit *du)
{
    int idx;
    
    for (idx = 0; idx < du->argc; idx ++) {
        if (du->argv[idx])
            sdsfree(du->argv[idx]);
    }<commit_msg>add test command 'zrange' for vireabtest<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <getopt.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <assert.h>
#include <sys/stat.h>
#include <sys/utsname.h>

#include <hiredis.h>
#include <darray.h>
#include <dutil.h>
#include <dlog.h>

#include <vrt_util.h>
#include <vrt_public.h>
#include <vrabtest.h>
#include <vrt_produce_data.h>

typedef struct produce_scheme {
    long long ckeys_write_idx;
    long long max_ckeys_count;    /* Max keys count that can be cached in the ckeys array. */
    sds *ckeys;    /* Cached keys that may exist in the target redis/vire servers. */

    int hit_ratio;   /* Hit ratio for the read commands. [0%,100%] */
    int hit_ratio_idx;   /* [0,hit_ratio_array_len-1] */
    int hit_ratio_array_len; /* 100 usually */
    int *hit_ratio_array;    /* Stored 0 or 1 for every element, 1 means used key in the cached keys array. */
} produce_scheme;

typedef struct produce_thread {
    int id;
    pthread_t thread_id;

    produce_scheme *ps;

    int pause;
    long long looptimes;
} produce_thread;

data_producer *delete_data_producer = NULL;

static unsigned int key_length_min;
static unsigned int key_length_max;
static unsigned int key_length_range_gap;
static unsigned int field_length_max;
static unsigned int string_length_max;

static int cmd_type;

static darray needed_cmd_type_producer;  /* type:  data_producer*/
static unsigned int needed_cmd_type_producer_count;

int produce_data_threads_count;
static darray produce_threads;

int produce_threads_pause_finished_count;

static sds get_random_cached_key(produce_scheme *ps)
{
    int idx;
    sds key;

    key = ps->ckeys[0];
    if (sdslen(key) == 0) {
        return NULL;
    }

    key = ps->ckeys[ps->max_ckeys_count-1];
    if (sdslen(key) == 0) {
        idx = rand()%ps->ckeys_write_idx;
    } else {
        idx = rand()%ps->max_ckeys_count;
    }
    
    return ps->ckeys[idx];
}

static int get_random_int(void)
{
    if (rand()%2 == 1) {
        return 0 - (int)rand();
    } else {
        return (int)rand();
    }
}

static unsigned int get_random_unsigned_int(void)
{
    return (unsigned int)rand();
}

static char get_random_char(void)
{
    return (char)rand()%250 + 5;
}

static sds get_random_key(void)
{
    unsigned int i, len;
    sds str = sdsempty();
    
    len = key_length_range_gap==0?key_length_min:
        (get_random_unsigned_int()%key_length_range_gap+key_length_min);
    if (len == 0) len ++;
    str = sdsMakeRoomFor(str,(size_t)len);
    sdsIncrLen(str, (int)len);

    for (i = 0; i < len; i ++) {
        str[i] = (char)get_random_char();
    }

    return str;
}

static sds get_random_string(void)
{
    unsigned int i, len;
    sds str = sdsempty();
    
    len = get_random_unsigned_int()%string_length_max;
    str = sdsMakeRoomFor(str,(size_t)len);
    sdsIncrLen(str, (int)len);

    for (i = 0; i < len; i ++) {
        str[i] = get_random_char();
    }

    return str;
}

static sds get_random_score_str(void)
{
    unsigned int decimal_len;
    sds str;

    if (rand()%2 == 1) {
        str = sdsnew("-");
    } else {
        str = sdsempty();
    }

    if (rand()%2 == 1) {
        str = sdscatfmt(str,"%u.%u",
            get_random_unsigned_int(),
            get_random_unsigned_int());
    } else {
        str = sdscatfmt(str,"%u",
            get_random_unsigned_int());
    }

    return str;
}

static unsigned int get_random_field_len(void)
{
    return get_random_unsigned_int()%field_length_max + 1;
}

static sds get_random_key_with_hit_ratio(produce_scheme *ps)
{
    sds key;
    if (ps->hit_ratio_array[ps->hit_ratio_idx++] == 0 || 
        ps->ckeys[ps->ckeys_write_idx] == NULL) {
        key = get_random_key();
    } else {
        key = get_random_cached_key(ps);
        if (key == NULL) key = get_random_key();
        else key = sdsdup(key);
    }
    if (ps->hit_ratio_idx >= ps->hit_ratio_array_len) {
        ps->hit_ratio_idx = 0;
    }
    return key;
}

static data_unit *get_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *set_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = get_random_string();
    
    return du;
}

static data_unit *setnx_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = get_random_string();
    
    return du;
}

static data_unit *setex_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 4;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    du->argv[3] = get_random_string();
    
    return du;
}

static data_unit *psetex_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 4;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    du->argv[3] = get_random_string();
    
    return du;
}

static data_unit *del_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    
    return du;
}

static data_unit *expire_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = sdsfromlonglong(rand()%10000);
    
    return du;
}

static data_unit *expireat_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = sdsfromlonglong(vrt_msec_now()/1000LL+rand()%10000);
    
    return du;
}

static data_unit *exists_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *ttl_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *pttl_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *incr_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    
    return du;
}

static data_unit *decr_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    
    return du;
}

static data_unit *incrby_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    
    return du;
}

static data_unit *decrby_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = sdsfromlonglong(rand()%10000);
    
    return du;
}

static data_unit *append_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 3;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();
    du->argv[2] = get_random_string();
    
    return du;
}

static data_unit *strlen_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    
    return du;
}

static data_unit *rpush_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    field_length = get_random_field_len();

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2+field_length;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();

    for (j = 0; j < field_length; j ++) {
        du->argv[2+j] = get_random_string();
    }
    
    return du;
}

static data_unit *lpush_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    field_length = get_random_field_len();

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2+field_length;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();

    for (j = 0; j < field_length; j ++) {
        du->argv[2+j] = get_random_string();
    }
    
    return du;
}

static data_unit *zadd_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    field_length = get_random_field_len();

    du = data_unit_get();
    du->dp = dp;
    du->argc = 2+field_length*2;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key();

    for (j = 2; j < 2+field_length*2; j += 2) {
        du->argv[j] = get_random_score_str();
        du->argv[j+1] = get_random_string();
    }
    
    return du;
}

static data_unit *zincrby_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;

    du = data_unit_get();
    du->dp = dp;
    du->argc = 4;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = get_random_score_str();;
    du->argv[3] = get_random_string();
    
    return du;
}

static data_unit *zrange_cmd_producer(data_producer *dp, produce_scheme *ps)
{
    data_unit *du;
    unsigned int j, field_length;
    int withscores;

    if (rand()%2 == 1) {
        withscores = 1;
    } else {
        withscores = 0;
    }

    du = data_unit_get();
    du->dp = dp;
    du->argc = withscores?5:4;
    du->argv = malloc(du->argc*sizeof(sds));
    du->argv[0] = sdsnew(dp->name);
    du->argv[1] = get_random_key_with_hit_ratio(ps);
    du->argv[2] = sdsfromlonglong(0);;
    du->argv[3] = sdsfromlonglong(get_random_int()%10000);
    if (withscores) du->argv[4] = sdsnew("withscores");
    
    return du;
}

static int producers_count;
data_producer redis_data_producer_table[] = {
    /* Key */
    {"del",del_cmd_producer,-2,"w",0,NULL,1,-1,1,TEST_CMD_TYPE_KEY},
    {"exists",exists_cmd_producer,-2,"rF",0,NULL,1,-1,1,TEST_CMD_TYPE_KEY},
    {"ttl",ttl_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"pttl",pttl_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"expire",expire_cmd_producer,3,"wF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"expireat",expireat_cmd_producer,3,"wF",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    /* String */
    {"get",get_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"set",set_cmd_producer,-3,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"setnx",setnx_cmd_producer,3,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"setex",setex_cmd_producer,4,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"psetex",psetex_cmd_producer,4,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_EXPIRE},
    {"incr",incr_cmd_producer,2,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"decr",decr_cmd_producer,2,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"incrby",incrby_cmd_producer,3,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"decrby",decrby_cmd_producer,3,"wmF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"append",append_cmd_producer,3,"wmA",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    {"strlen",strlen_cmd_producer,2,"rF",0,NULL,1,1,1,TEST_CMD_TYPE_STRING},
    /* List */
    {"rpush",rpush_cmd_producer,-3,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_LIST},
    {"lpush",lpush_cmd_producer,-3,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_LIST},
    /* SortedSet */
    {"zadd",zadd_cmd_producer,-4,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_ZSET},
    {"zincrby",zincrby_cmd_producer,4,"wmFA",0,NULL,1,1,1,TEST_CMD_TYPE_ZSET},
    {"zrange",zrange_cmd_producer,-4,"r",0,NULL,1,1,1,TEST_CMD_TYPE_ZSET}
};

data_unit *data_unit_get(void)
{
    data_unit *du = malloc(sizeof(data_unit));
    du->dp = NULL;
    du->argc = 0;
    du->argv = NULL;
    du->hashvalue = 0;
    return du;
}

void data_unit_put(data_unit *du)
{
    int idx;
    
    for (idx = 0; idx < du->argc; idx ++) {
        if (du->argv[idx])
            sdsfree(du->argv[idx]);
    }
{'fraction_non_alphanumeric': 0.16333392319943374, 'fraction_numerical': 0.01932401344894709, 'mean_word_length': 3.4067373674360573, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 419, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1275328', 'n_tokens_mistral': 11749, 'n_tokens_neox': 11529, 'n_words': 1914}