<commit_before>/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc.  All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"

#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>

DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
    int64_t key;
    int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
                                         {2,5},
                                         {3,6}};

int no_lock_contention=0;

static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {

    src_db = src_db;

    uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
    assert(which == 0);

    // switch the key and val
    dbt_init(dest_key, src_val->data, src_val->size);
    dbt_init(dest_val, src_key->data, src_key->size);

//    printf("dest_key.data = %d\n", *(int*)dest_key->data);
//    printf("dest_val.data = %d\n", *(int*)dest_val->data);

    return 0;
}

static int poll_print(void *extra, float progress) {
    progress = progress;
    extra = extra;
    if ( verbose ) printf("poll_print %f\n", progress);
    return 0;
}

static void run_indexer(DB *src, DB **dbs)
{
    int r;
    DB_TXN    *txn;
    DB_INDEXER *indexer;
    uint32_t db_flags[NUM_DBS];

    if ( verbose ) printf("test_indexer\n");
    for(int i=0;i<NUM_DBS;i++) { 
        db_flags[i] = DB_NOOVERWRITE; 
    }

    // create and initialize loader
    r = env->txn_begin(env, NULL, &txn, 0);                                                               
    CKERR(r);

    if ( verbose ) printf("run_indexer create_indexer\n");
    r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
    CKERR(r);
    r = indexer->set_error_callback(indexer, NULL, NULL);
    CKERR(r);
    r = indexer->set_poll_function(indexer, poll_print, NULL);
    CKERR(r);

    if ( verbose ) printf("run_indexer build\n");
    r = indexer->build(indexer);
    CKERR(r);

    if ( verbose ) printf("run_indexer close\n");
    r = indexer->close(indexer);
    CKERR(r);
    r = txn->commit(txn, DB_TXN_SYNC);
    CKERR(r);
    
    if ( verbose ) printf("run_indexer done\n");
}

char *src_name="src.db";

static void run_test(void) 
{
    int r;
    r = system("rm -rf " ENVDIR);                                                CKERR(r);
    r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);                          CKERR(r);
    r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO);                   CKERR(r);

    r = db_env_create(&env, 0);                                                  CKERR(r);
    r = env->set_lg_dir(env, "log");                                             CKERR(r);
    r = env->set_default_bt_compare(env, int64_dbt_cmp);                         CKERR(r);
    r = env->set_generate_row_callback_for_put(env, put_multiple_generate);      CKERR(r);
    int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
    r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO);               CKERR(r);
    db_env_enable_engine_status(0);  // disable engine status on crash
    env->set_errfile(env, stderr);
    //Disable auto-checkpointing
    r = env->checkpointing_set_period(env, 0);                                   CKERR(r);

    DB *src_db = NULL;
    r = db_create(&src_db, env, 0);                                                             CKERR(r);
    r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);   CKERR(r);
    DBT key, val;
    DB_TXN *txn;
    r = env->txn_begin(env, NULL, &txn, 0);                                      CKERR(r);
    for(int i=0;i<NUM_KV_PAIRS;i++) {
        dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
        dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
        r = src_db->put(src_db, txn, &key, &val, 0);                             CKERR(r);
    }
    if ( no_lock_contention ) {
        r = txn->commit(txn, DB_TXN_SYNC);                                       CKERR(r);
    }

    DB *dbs[NUM_DBS];
    for (int i = 0; i < NUM_DBS; i++) {
        r = db_create(&dbs[i], env, 0); CKERR(r);
        char key_name[32]; 
        sprintf(key_name, "key%d", i);
        r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);   CKERR(r);
        dbs[i]->app_private = (void *) (intptr_t) i;
    }

    run_indexer(src_db, dbs);

    // at this point, no_lock_contention==0 (default), the hot dictionary should have locks on the rows

    // try overwriting a value in hot dictionary[0]
    {
        DB_TXN *owrt_txn;
        r = env->txn_begin(env, NULL, &owrt_txn, 0); 
        CKERR(r);
        
        dbt_init(&key, &kv_pairs[0].key, sizeof(kv_pairs[0].key));
        dbt_init(&key, &kv_pairs[0].val, sizeof(kv_pairs[0].val));
        r = dbs[0]->put(dbs[0], owrt_txn, &key, &val, 0);

        if ( no_lock_contention ) {
            assert(r == 0 );
            if ( verbose ) printf("no lock contention detected, as expected ( put returns 0 )\n");
        } 
        else {
            assert(r == DB_LOCK_NOTGRANTED );
            if ( verbose ) printf("lock contention detected, as expected ( put returns DB_LOCK_NOTGRANTED )\n");
        }
        r = owrt_txn->commit(owrt_txn, DB_TXN_SYNC); 
        CKERR(r);
    }
    if ( verbose ) printf("PASS\n");

    if ( !no_lock_contention) { 
        r = txn->commit(txn, DB_TXN_SYNC);                                       CKERR(r);
    }


    for(int i=0;i<NUM_DBS;i++) {
        r = dbs[i]->close(dbs[i], 0);                                            CKERR(r);
    }

    r = src_db->close(src_db, 0);                                                CKERR(r);
    r = env->close(env, 0);                                                      CKERR(r);
}


// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);

int test_main(int argc, char * const *argv) {
    do_args(argc, argv);
    run_test();
    return 0;
}

static void do_args(int argc, char * const argv[]) {
    int resultcode;
    char *cmd = argv[0];
    argc--; argv++;
    while (argc>0) {
	if (strcmp(argv[0], "-v")==0) {
	    verbose++;
	} else if (strcmp(argv[0],"-q")==0) {
	    verbose--;
	    if (verbose<0) verbose=0;
	} else if (strcmp(argv[0],"-l")==0) {
            no_lock_contention=1;
        } else if (strcmp(argv[0], "-h")==0) {
	    resultcode=0;
	do_usage:
	    fprintf(stderr, "Usage:\n%s\n", cmd);
	    exit(resultcode);
	} else {
	    fprintf(stderr, "Unknown arg: %s\n", argv[0]);
	    resultcode=1;
	    goto do_usage;
	}
	argc--;
	argv++;
    }
}<commit_msg>improve test<commit_after>/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc.  All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"

#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>

DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
    int64_t key;
    int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
                                         {2,5},
                                         {3,6}};

static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {

    src_db = src_db;

    uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
    assert(which == 0);

    // switch the key and val
    dbt_init(dest_key, src_val->data, src_val->size);
    dbt_init(dest_val, src_key->data, src_key->size);

//    printf("dest_key.data = %d\n", *(int*)dest_key->data);
//    printf("dest_val.data = %d\n", *(int*)dest_val->data);

    return 0;
}

static int poll_print(void *extra, float progress) {
    progress = progress;
    extra = extra;
    if ( verbose ) printf("poll_print %f\n", progress);
    return 0;
}

static void run_indexer(DB *src, DB **dbs)
{
    int r;
    DB_TXN    *txn;
    DB_INDEXER *indexer;
    uint32_t db_flags[NUM_DBS];

    if ( verbose ) printf("test_indexer\n");
    for(int i=0;i<NUM_DBS;i++) { 
        db_flags[i] = DB_NOOVERWRITE; 
    }

    // create and initialize loader
    r = env->txn_begin(env, NULL, &txn, 0);                                                               
    CKERR(r);

    if ( verbose ) printf("run_indexer create_indexer\n");
    r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
    CKERR(r);
    r = indexer->set_error_callback(indexer, NULL, NULL);
    CKERR(r);
    r = indexer->set_poll_function(indexer, poll_print, NULL);
    CKERR(r);

    if ( verbose ) printf("run_indexer build\n");
    r = indexer->build(indexer);
    CKERR(r);

    if ( verbose ) printf("run_indexer close\n");
    r = indexer->close(indexer);
    CKERR(r);
    r = txn->commit(txn, DB_TXN_SYNC);
    CKERR(r);
    
    if ( verbose ) printf("run_indexer done\n");
}

char *src_name="src.db";

static void run_test(void) 
{
    int r;
    r = system("rm -rf " ENVDIR);                                                CKERR(r);
    r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);                          CKERR(r);
    r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO);                   CKERR(r);

    r = db_env_create(&env, 0);                                                  CKERR(r);
    r = env->set_lg_dir(env, "log");                                             CKERR(r);
    r = env->set_default_bt_compare(env, int64_dbt_cmp);                         CKERR(r);
    r = env->set_generate_row_callback_for_put(env, put_multiple_generate);      CKERR(r);
    int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
    r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO);               CKERR(r);
    db_env_enable_engine_status(0);  // disable engine status on crash
    env->set_errfile(env, stderr);
    //Disable auto-checkpointing
    r = env->checkpointing_set_period(env, 0);                                   CKERR(r);

    DB *src_db = NULL;
    r = db_create(&src_db, env, 0);                                                             CKERR(r);
    r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);   CKERR(r);
    DBT key, val;
    DB_TXN *txn;
    r = env->txn_begin(env, NULL, &txn, 0);                                      CKERR(r);
    for(int i=0;i<NUM_KV_PAIRS;i++) {
        dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
        dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
        r = src_db->put(src_db, txn, &key, &val, 0);                             CKERR(r);
    }

    DB *dbs[NUM_DBS];
    for (int i = 0; i < NUM_DBS; i++) {
        r = db_create(&dbs[i], env, 0); CKERR(r);
        char key_name[32]; 
        sprintf(key_name, "key%d", i);
        r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666);   CKERR(r);
        dbs[i]->app_private = (void *) (intptr_t) i;
    }

    run_indexer(src_db, dbs);

    // at this point the hot dictionary should have locks on the rows

    // try overwriting a value in hot dictionary[0]
    {
        DB_TXN *owrt_txn;
        r = env->txn_begin(env, NULL, &owrt_txn, 0); 
        CKERR(r);
        
        dbt_init(&key, &kv_pairs[0].key, sizeof(kv_pairs[0].key));
        dbt_init(&key, &kv_pairs[0].val, sizeof(kv_pairs[0].val));
        r = dbs[0]->put(dbs[0], owrt_txn, &key, &val, 0);

        assert(r == DB_LOCK_NOTGRANTED );
        if ( verbose ) printf("lock contention detected, as expected ( put returns DB_LOCK_NOTGRANTED )\n");

        r = owrt_txn->commit(owrt_txn, DB_TXN_SYNC); 
        CKERR(r);
    }

    // close the transaction (releasing locks), and try writing again 
    r = txn->commit(txn, DB_TXN_SYNC);                                       CKERR(r);
    {
        DB_TXN *owrt_txn;
        r = env->txn_begin(env, NULL, &owrt_txn, 0); 
        CKERR(r);
        
        dbt_init(&key, &kv_pairs[0].key, sizeof(kv_pairs[0].key));
        dbt_init(&key, &kv_pairs[0].val, sizeof(kv_pairs[0].val));
        r = dbs[0]->put(dbs[0], owrt_txn, &key, &val, 0);

        assert(r == 0 );
        if ( verbose ) printf("no lock contention detected, as expected ( put returns 0 )\n");

        r = owrt_txn->commit(owrt_txn, DB_TXN_SYNC); 
        CKERR(r);
    }


    if ( verbose ) printf("PASS\n");

    for(int i=0;i<NUM_DBS;i++) {
        r = dbs[i]->close(dbs[i], 0);                                            CKERR(r);
    }

    r = src_db->close(src_db, 0);                                                CKERR(r);
    r = env->close(env, 0);                                                      CKERR(r);
}


// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);

int test_main(int argc, char * const *argv) {
    do_args(argc, argv);
    run_test();
    return 0;
}

static void do_args(int argc, char * const argv[]) {
    int resultcode;
    char *cmd = argv[0];
    argc--; argv++;
    while (argc>0) {
	if (strcmp(argv[0], "-v")==0) {
	    verbose++;
	} else if (strcmp(argv[0],"-q")==0) {
	    verbose--;
	    if (verbose<0) verbose=0;
        } else if (strcmp(argv[0], "-h")==0) {
	    resultcode=0;
	do_usage:
	    fprintf(stderr, "Usage:\n%s\n", cmd);
	    exit(resultcode);
	} else {
	    fprintf(stderr, "Unknown arg: %s\n", argv[0]);
	    resultcode=1;
	    goto do_usage;
	}
	argc--;
	argv++;
    }
}
{'fraction_non_alphanumeric': 0.15446199045170766, 'fraction_numerical': 0.015350716121924348, 'mean_word_length': 2.0372518402855233, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 51, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2865402', 'n_tokens_mistral': 5301, 'n_tokens_neox': 5018, 'n_words': 1025}