<commit_before>namespace tokudb {

    // Set the key_info cardinality counters for the table.
    void set_card_in_key_info(TABLE *table, uint rec_per_keys, uint64_t rec_per_key[]) {
        uint next_key_part = 0;
        for (uint i = 0; i < table->s->keys; i++) {
            bool is_unique_key = (i == table->s->primary_key) || (table->key_info[i].flags & HA_NOSAME);
            uint num_key_parts = get_key_parts(&table->key_info[i]);
            for (uint j = 0; j < num_key_parts; j++) {
                assert(next_key_part < rec_per_keys);
                ulong val = rec_per_key[next_key_part++];
                if (is_unique_key && j == num_key_parts-1)
                    val = 1;
                table->key_info[i].rec_per_key[j] = val;
            }
        }
    }
    
    // Put the cardinality counters into the status dictionary.
    void set_card_in_status(DB *status_db, DB_TXN *txn, uint rec_per_keys, uint64_t rec_per_key[]) {
        // encode cardinality into the buffer
        tokudb::buffer b;
        size_t s;
        s = b.append_ui<uint32_t>(rec_per_keys);
        assert(s > 0);
        for (uint i = 0; i < rec_per_keys; i++) {
            s = b.append_ui<uint64_t>(rec_per_key[i]);
            assert(s > 0);
        }
        // write cardinality to status
        int error = write_to_status(status_db, hatoku_cardinality, b.data(), b.size(), txn);
        assert(error == 0);
    }

    // Get the cardinality counters from the status dictionary.
    int get_card_from_status(DB *status_db, DB_TXN *txn, uint rec_per_keys, uint64_t rec_per_key[]) {
        // read cardinality from status
        void *buf = 0; size_t buf_size = 0;
        int error = get_status_realloc(status_db, txn, hatoku_cardinality, &buf, &buf_size);
        if (error == 0) {
            // decode cardinality from the buffer
            tokudb::buffer b(buf, 0, buf_size);
            size_t s;
            uint32_t num_parts;
            s = b.consume_ui<uint32_t>(&num_parts);
            if (s == 0 || num_parts != rec_per_keys) 
                error = EINVAL;
            if (error == 0) {
                for (uint i = 0; i < rec_per_keys; i++) {
                    s = b.consume_ui<uint64_t>(&rec_per_key[i]);
                    if (s == 0) {
                        error = EINVAL;
                        break;
                    }
                }
            }
        }
        // cleanup
        free(buf);
        return error;
    }

    // Delete the cardinality counters from the status dictionary.
    void delete_card_from_status(DB *status_db, DB_TXN *txn) {
        int error = remove_from_status(status_db, hatoku_cardinality, txn);
        assert(error == 0);
    }

    bool find_index_of_key(const char *key_name, TABLE_SHARE *table_share, uint *index_offset_ptr) {
        for (uint i = 0; i < table_share->keys; i++) {
            if (strcmp(key_name, table_share->key_info[i].name) == 0) {
                *index_offset_ptr = i;
                return true;
            }
        }
        return false;
    }

    // Altered table cardinality = select cardinality data from current table cardinality for keys that exist 
    // in the altered table and the current table.
    void set_card_from_status(DB *status_db, DB_TXN *txn, TABLE_SHARE *table_share, TABLE_SHARE *altered_table_share) {
        int error;
        // read existing cardinality data from status
        uint64_t rec_per_key[table_share->key_parts];
        error = get_card_from_status(status_db, txn, table_share->key_parts, rec_per_key);
        // set altered records per key to unknown
        uint64_t altered_rec_per_key[altered_table_share->key_parts];
        for (uint i = 0; i < altered_table_share->key_parts; i++)
            altered_rec_per_key[i] = 0;
        // compute the beginning of the key offsets in the original table
        uint orig_key_offset[table_share->keys];
        uint orig_key_parts = 0;
        for (uint i = 0; i < table_share->keys; i++) {
            orig_key_offset[i] = orig_key_parts;
            orig_key_parts += table_share->key_info[i].key_parts;
        }
        // if orig card data exists, then use it to compute new card data
        if (error == 0) {
            uint key_parts = 0;
            for (uint i = 0; error == 0 && i < altered_table_share->keys; i++) {
                uint orig_key_index;
                if (find_index_of_key(altered_table_share->key_info[i].name, table_share, &orig_key_index)) {
                    memcpy(&altered_rec_per_key[key_parts], &rec_per_key[orig_key_offset[orig_key_index]], altered_table_share->key_info[i].key_parts);
                }
                key_parts += altered_table_share->key_info[i].key_parts;
            }
        }
        if (error == 0)
            set_card_in_status(status_db, txn, altered_table_share->key_parts, altered_rec_per_key);
        else
            delete_card_from_status(status_db, txn);
    }

    // Compute records per key for all key parts of the ith key of the table.
    // For each key part, put records per key part in *rec_per_key_part[key_part_index].
    // Returns 0 if success, otherwise an error number.
    // TODO statistical dives into the FT
    int analyze_card(DB *db, DB_TXN *txn, bool is_unique __attribute__((unused)), uint64_t num_key_parts, uint64_t *rec_per_key_part,
                     int (*key_compare)(DB *, const DBT *, const DBT *, uint),
                     int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra) {
        int error = 0;
        uint64_t rows = 0;
        uint64_t unique_rows[num_key_parts];
        if (is_unique && num_key_parts == 1) {
            // dont compute for unique keys with a single part.  we already know the answer.
            rows = unique_rows[0] = 1;
        } else {
            DBC *cursor = NULL;
            error = db->cursor(db, txn, &cursor, 0);
            if (error == 0) {
                for (uint64_t i = 0; i < num_key_parts; i++)
                    unique_rows[i] = 1;<commit_msg>compile on mysql 5.6<commit_after>namespace tokudb {

    // Set the key_info cardinality counters for the table.
    void set_card_in_key_info(TABLE *table, uint rec_per_keys, uint64_t rec_per_key[]) {
        uint next_key_part = 0;
        for (uint i = 0; i < table->s->keys; i++) {
            bool is_unique_key = (i == table->s->primary_key) || (table->key_info[i].flags & HA_NOSAME);
            uint num_key_parts = get_key_parts(&table->key_info[i]);
            for (uint j = 0; j < num_key_parts; j++) {
                assert(next_key_part < rec_per_keys);
                ulong val = rec_per_key[next_key_part++];
                if (is_unique_key && j == num_key_parts-1)
                    val = 1;
                table->key_info[i].rec_per_key[j] = val;
            }
        }
    }
    
    // Put the cardinality counters into the status dictionary.
    void set_card_in_status(DB *status_db, DB_TXN *txn, uint rec_per_keys, uint64_t rec_per_key[]) {
        // encode cardinality into the buffer
        tokudb::buffer b;
        size_t s;
        s = b.append_ui<uint32_t>(rec_per_keys);
        assert(s > 0);
        for (uint i = 0; i < rec_per_keys; i++) {
            s = b.append_ui<uint64_t>(rec_per_key[i]);
            assert(s > 0);
        }
        // write cardinality to status
        int error = write_to_status(status_db, hatoku_cardinality, b.data(), b.size(), txn);
        assert(error == 0);
    }

    // Get the cardinality counters from the status dictionary.
    int get_card_from_status(DB *status_db, DB_TXN *txn, uint rec_per_keys, uint64_t rec_per_key[]) {
        // read cardinality from status
        void *buf = 0; size_t buf_size = 0;
        int error = get_status_realloc(status_db, txn, hatoku_cardinality, &buf, &buf_size);
        if (error == 0) {
            // decode cardinality from the buffer
            tokudb::buffer b(buf, 0, buf_size);
            size_t s;
            uint32_t num_parts;
            s = b.consume_ui<uint32_t>(&num_parts);
            if (s == 0 || num_parts != rec_per_keys) 
                error = EINVAL;
            if (error == 0) {
                for (uint i = 0; i < rec_per_keys; i++) {
                    s = b.consume_ui<uint64_t>(&rec_per_key[i]);
                    if (s == 0) {
                        error = EINVAL;
                        break;
                    }
                }
            }
        }
        // cleanup
        free(buf);
        return error;
    }

    // Delete the cardinality counters from the status dictionary.
    void delete_card_from_status(DB *status_db, DB_TXN *txn) {
        int error = remove_from_status(status_db, hatoku_cardinality, txn);
        assert(error == 0);
    }

    bool find_index_of_key(const char *key_name, TABLE_SHARE *table_share, uint *index_offset_ptr) {
        for (uint i = 0; i < table_share->keys; i++) {
            if (strcmp(key_name, table_share->key_info[i].name) == 0) {
                *index_offset_ptr = i;
                return true;
            }
        }
        return false;
    }

    // Altered table cardinality = select cardinality data from current table cardinality for keys that exist 
    // in the altered table and the current table.
    void set_card_from_status(DB *status_db, DB_TXN *txn, TABLE_SHARE *table_share, TABLE_SHARE *altered_table_share) {
        int error;
        // read existing cardinality data from status
        uint64_t rec_per_key[table_share->key_parts];
        error = get_card_from_status(status_db, txn, table_share->key_parts, rec_per_key);
        // set altered records per key to unknown
        uint64_t altered_rec_per_key[altered_table_share->key_parts];
        for (uint i = 0; i < altered_table_share->key_parts; i++)
            altered_rec_per_key[i] = 0;
        // compute the beginning of the key offsets in the original table
        uint orig_key_offset[table_share->keys];
        uint orig_key_parts = 0;
        for (uint i = 0; i < table_share->keys; i++) {
            orig_key_offset[i] = orig_key_parts;
            orig_key_parts += get_key_parts(&table_share->key_info[i]);
        }
        // if orig card data exists, then use it to compute new card data
        if (error == 0) {
            uint next_key_parts = 0;
            for (uint i = 0; error == 0 && i < altered_table_share->keys; i++) {
                uint ith_key_parts = get_key_parts(&altered_table_share->key_info[i]);
                uint orig_key_index;
                if (find_index_of_key(altered_table_share->key_info[i].name, table_share, &orig_key_index)) {
                    memcpy(&altered_rec_per_key[next_key_parts], &rec_per_key[orig_key_offset[orig_key_index]], ith_key_parts);
                }
                next_key_parts += ith_key_parts;
            }
        }
        if (error == 0)
            set_card_in_status(status_db, txn, altered_table_share->key_parts, altered_rec_per_key);
        else
            delete_card_from_status(status_db, txn);
    }

    // Compute records per key for all key parts of the ith key of the table.
    // For each key part, put records per key part in *rec_per_key_part[key_part_index].
    // Returns 0 if success, otherwise an error number.
    // TODO statistical dives into the FT
    int analyze_card(DB *db, DB_TXN *txn, bool is_unique __attribute__((unused)), uint64_t num_key_parts, uint64_t *rec_per_key_part,
                     int (*key_compare)(DB *, const DBT *, const DBT *, uint),
                     int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra) {
        int error = 0;
        uint64_t rows = 0;
        uint64_t unique_rows[num_key_parts];
        if (is_unique && num_key_parts == 1) {
            // dont compute for unique keys with a single part.  we already know the answer.
            rows = unique_rows[0] = 1;
        } else {
            DBC *cursor = NULL;
            error = db->cursor(db, txn, &cursor, 0);
            if (error == 0) {
                for (uint64_t i = 0; i < num_key_parts; i++)
                    unique_rows[i] = 1;
{'fraction_non_alphanumeric': 0.10242898312062577, 'fraction_numerical': 0.011856731165088515, 'mean_word_length': 2.0342243317511866, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6013931', 'n_tokens_mistral': 4027, 'n_tokens_neox': 3797, 'n_words': 989}