<commit_before>#include "TableMetadata.h"

/***
 * Returns the allocation number of bytes required to allocate a type of data
 * @param VT Cassandra Type
 * @return Allocation size needed to store data of type VT
 */
uint16_t TableMetadata::compute_size_of(const ColumnMeta &CM) const {
    switch (CM.type) {
        case CASS_VALUE_TYPE_TEXT:
        case CASS_VALUE_TYPE_VARCHAR:
        case CASS_VALUE_TYPE_ASCII: {
            return sizeof(char *);
        }
        case CASS_VALUE_TYPE_VARINT:
        case CASS_VALUE_TYPE_BIGINT: {
            return sizeof(int64_t);
        }
        case CASS_VALUE_TYPE_BLOB: {
            return sizeof(unsigned char *);
        }
        case CASS_VALUE_TYPE_BOOLEAN: {
            return sizeof(bool);
        }
        case CASS_VALUE_TYPE_COUNTER: {
            return sizeof(uint32_t);
        }
        case CASS_VALUE_TYPE_DECIMAL: {
            //TODO
            std::cerr << "Parse decimals data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_DOUBLE: {
            return sizeof(double);
        }
        case CASS_VALUE_TYPE_FLOAT: {
            return sizeof(float);
        }
        case CASS_VALUE_TYPE_INT: {
            return sizeof(int32_t);
        }
        case CASS_VALUE_TYPE_TIMESTAMP: {
            //TODO
            std::cerr << "Timestamp data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_UUID: {
            return sizeof(uint64_t *);
        }
        case CASS_VALUE_TYPE_TIMEUUID: {
            std::cerr << "TIMEUUID data type supported yet" << std::endl;
            //TODO
            break;
        }
        case CASS_VALUE_TYPE_INET: {
            //TODO
            std::cerr << "INET data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_DATE: {
            std::cerr << "Date data type supported yet" << std::endl;

            //TODO
            break;
        }
        case CASS_VALUE_TYPE_TIME: {
            std::cerr << "Time data type supported yet" << std::endl;
            //TODO

            break;
        }
        case CASS_VALUE_TYPE_SMALL_INT: {
            return sizeof(int16_t);
        }
        case CASS_VALUE_TYPE_TINY_INT: {
            return sizeof(int8_t);
        }
        case CASS_VALUE_TYPE_LIST: {
            std::cerr << "List data type supported yet" << std::endl;
            //TODO
            break;
        }
        case CASS_VALUE_TYPE_MAP: {
            std::cerr << "Map data type supported yet" << std::endl;
            //TODO

            break;
        }
        case CASS_VALUE_TYPE_SET: {
            std::cerr << "Set data type supported yet" << std::endl;
            //TODO

            break;
        }
        case CASS_VALUE_TYPE_TUPLE: {
            //TODO

            std::cerr << "Tuple data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_UDT: {
            throw ModuleException("Can't parse data: User defined type not supported");
        }
        case CASS_VALUE_TYPE_CUSTOM:
        case CASS_VALUE_TYPE_UNKNOWN:
        default:
        {
            throw ModuleException("Can't parse data: Unknown data type or user defined type");
            //TODO
        }
    }
    return 0;
}


TableMetadata::TableMetadata(const char *table_name, const char *keyspace_name,
                             std::vector<std::map<std::string, std::string>> &keys_names,
                             std::vector<std::map<std::string, std::string>> &columns_names,
                             CassSession *session) {


    if (keys_names.empty()) throw ModuleException("TableMetadata: No keys received");

    uint32_t n_keys = (uint32_t) keys_names.size();
    uint32_t n_cols = (uint32_t) columns_names.size(); //TODO check for *
    for (uint32_t i = 0; i < n_keys; ++i) {
        std::transform(keys_names[i]["name"].begin(), keys_names[i]["name"].end(), keys_names[i]["name"].begin(),
                       ::tolower);
    }
    for (uint32_t i = 0; i < n_cols; ++i) {
        std::transform(columns_names[i]["name"].begin(), columns_names[i]["name"].end(),
                       columns_names[i]["name"].begin(), ::tolower);
    }
    this->table = std::string(table_name);
    std::transform(this->table.begin(), this->table.end(), this->table.begin(), ::tolower);
    this->keyspace = std::string(keyspace_name);
    std::transform(this->keyspace.begin(), this->keyspace.end(), this->keyspace.begin(), ::tolower);


    const CassSchemaMeta *schema_meta = cass_session_get_schema_meta(session);
    if (!schema_meta) {
        std::string error_msg = "TableMetadata constructor: Cassandra schema doesn't exist, probably not connected...";
        if (session == NULL) error_msg += "session with cassandra not stablished";
        throw ModuleException(error_msg.c_str());
    }

    const CassKeyspaceMeta *keyspace_meta = cass_schema_meta_keyspace_by_name(schema_meta, this->keyspace.c_str());
    if (!keyspace_meta) {
        throw ModuleException("The keyspace " + std::string(keyspace_name) + " has no metadatas,"
                "check the keyspace name and make sure it exists");
    }


    const CassTableMeta *table_meta = cass_keyspace_meta_table_by_name(keyspace_meta, this->table.c_str());
    if (!table_meta || (cass_table_meta_column_count(table_meta) == 0)) {
        throw ModuleException("The table " + std::string(table_name) + " has no metadatas,"
                " check the table name and make sure it exists");
    }

//TODO Switch to unordered maps for efficiency

    std::map<std::string, ColumnMeta> metadatas;

/*** build metadata ***/

    CassIterator *iterator = cass_iterator_columns_from_table_meta(table_meta);
    while (cass_iterator_next(iterator)) {
        const CassColumnMeta *cmeta = cass_iterator_get_column_meta(iterator);

        const char *value;
        size_t length;
        cass_column_meta_name(cmeta, &value, &length);

        const CassDataType *type = cass_column_meta_data_type(cmeta);

        metadatas[value] = {};
        metadatas[value].type = cass_data_type_type(type);
        metadatas[value].col_type = cass_column_meta_type(cmeta);
    }
    cass_iterator_free(iterator);
    cass_schema_meta_free(schema_meta);


    std::string key = keys_names[0]["name"];
    if (key.empty()) throw ModuleException("Empty key name given on position 0");
    std::string select_where = key + "=? ";
    std::string keys = key;
    std::string col;
    std::string tokens_keys = "";
    if (metadatas[key].col_type == CASS_COLUMN_TYPE_PARTITION_KEY) tokens_keys += key;

    for (uint16_t i = 1; i < n_keys; ++i) {
        key = keys_names[i]["name"];
        if (key.empty()) throw ModuleException("Empty key name given on position: " + std::to_string(i));
        keys += "," + key;
        select_where += "AND " + key + "=? ";
        if (metadatas[key].col_type == CASS_COLUMN_TYPE_PARTITION_KEY) tokens_keys += "," + key;
    }
    if (tokens_keys.empty()) throw ModuleException("No partition key detected among the keys: " + keys);

    std::string cols = "";
    if (!columns_names.empty()) {
        cols += columns_names[0]["name"]; //TODO Check for *
        if (cols.empty()) throw ModuleException("Empty column name given on position 0");
        for (uint16_t i = 1; i < n_cols; ++i) {
            col = columns_names[i]["name"];
            if (col.empty()) throw ModuleException("Empty column name given onposition: " + std::to_string(i));
            cols += "," + col;
        }
    }
    std::string keys_and_cols = keys;<commit_msg>Fix bug on unordered keys in prefetch<commit_after>#include "TableMetadata.h"

/***
 * Returns the allocation number of bytes required to allocate a type of data
 * @param VT Cassandra Type
 * @return Allocation size needed to store data of type VT
 */
uint16_t TableMetadata::compute_size_of(const ColumnMeta &CM) const {
    switch (CM.type) {
        case CASS_VALUE_TYPE_TEXT:
        case CASS_VALUE_TYPE_VARCHAR:
        case CASS_VALUE_TYPE_ASCII: {
            return sizeof(char *);
        }
        case CASS_VALUE_TYPE_VARINT:
        case CASS_VALUE_TYPE_BIGINT: {
            return sizeof(int64_t);
        }
        case CASS_VALUE_TYPE_BLOB: {
            return sizeof(unsigned char *);
        }
        case CASS_VALUE_TYPE_BOOLEAN: {
            return sizeof(bool);
        }
        case CASS_VALUE_TYPE_COUNTER: {
            return sizeof(uint32_t);
        }
        case CASS_VALUE_TYPE_DECIMAL: {
            //TODO
            std::cerr << "Parse decimals data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_DOUBLE: {
            return sizeof(double);
        }
        case CASS_VALUE_TYPE_FLOAT: {
            return sizeof(float);
        }
        case CASS_VALUE_TYPE_INT: {
            return sizeof(int32_t);
        }
        case CASS_VALUE_TYPE_TIMESTAMP: {
            //TODO
            std::cerr << "Timestamp data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_UUID: {
            return sizeof(uint64_t *);
        }
        case CASS_VALUE_TYPE_TIMEUUID: {
            std::cerr << "TIMEUUID data type supported yet" << std::endl;
            //TODO
            break;
        }
        case CASS_VALUE_TYPE_INET: {
            //TODO
            std::cerr << "INET data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_DATE: {
            std::cerr << "Date data type supported yet" << std::endl;

            //TODO
            break;
        }
        case CASS_VALUE_TYPE_TIME: {
            std::cerr << "Time data type supported yet" << std::endl;
            //TODO

            break;
        }
        case CASS_VALUE_TYPE_SMALL_INT: {
            return sizeof(int16_t);
        }
        case CASS_VALUE_TYPE_TINY_INT: {
            return sizeof(int8_t);
        }
        case CASS_VALUE_TYPE_LIST: {
            std::cerr << "List data type supported yet" << std::endl;
            //TODO
            break;
        }
        case CASS_VALUE_TYPE_MAP: {
            std::cerr << "Map data type supported yet" << std::endl;
            //TODO

            break;
        }
        case CASS_VALUE_TYPE_SET: {
            std::cerr << "Set data type supported yet" << std::endl;
            //TODO

            break;
        }
        case CASS_VALUE_TYPE_TUPLE: {
            //TODO

            std::cerr << "Tuple data type supported yet" << std::endl;
            break;
        }
        case CASS_VALUE_TYPE_UDT: {
            throw ModuleException("Can't parse data: User defined type not supported");
        }
        case CASS_VALUE_TYPE_CUSTOM:
        case CASS_VALUE_TYPE_UNKNOWN:
        default:
        {
            throw ModuleException("Can't parse data: Unknown data type or user defined type");
            //TODO
        }
    }
    return 0;
}


TableMetadata::TableMetadata(const char *table_name, const char *keyspace_name,
                             std::vector<std::map<std::string, std::string>> &keys_names,
                             std::vector<std::map<std::string, std::string>> &columns_names,
                             CassSession *session) {


    if (keys_names.empty()) throw ModuleException("TableMetadata: No keys received");

    uint32_t n_keys = (uint32_t) keys_names.size();
    uint32_t n_cols = (uint32_t) columns_names.size(); //TODO check for *
    for (uint32_t i = 0; i < n_keys; ++i) {
        std::transform(keys_names[i]["name"].begin(), keys_names[i]["name"].end(), keys_names[i]["name"].begin(),
                       ::tolower);
    }
    for (uint32_t i = 0; i < n_cols; ++i) {
        std::transform(columns_names[i]["name"].begin(), columns_names[i]["name"].end(),
                       columns_names[i]["name"].begin(), ::tolower);
    }
    this->table = std::string(table_name);
    std::transform(this->table.begin(), this->table.end(), this->table.begin(), ::tolower);
    this->keyspace = std::string(keyspace_name);
    std::transform(this->keyspace.begin(), this->keyspace.end(), this->keyspace.begin(), ::tolower);


    const CassSchemaMeta *schema_meta = cass_session_get_schema_meta(session);
    if (!schema_meta) {
        std::string error_msg = "TableMetadata constructor: Cassandra schema doesn't exist, probably not connected...";
        if (session == NULL) error_msg += "session with cassandra not stablished";
        throw ModuleException(error_msg.c_str());
    }

    const CassKeyspaceMeta *keyspace_meta = cass_schema_meta_keyspace_by_name(schema_meta, this->keyspace.c_str());
    if (!keyspace_meta) {
        throw ModuleException("The keyspace " + std::string(keyspace_name) + " has no metadatas,"
                "check the keyspace name and make sure it exists");
    }


    const CassTableMeta *table_meta = cass_keyspace_meta_table_by_name(keyspace_meta, this->table.c_str());
    if (!table_meta || (cass_table_meta_column_count(table_meta) == 0)) {
        throw ModuleException("The table " + std::string(table_name) + " has no metadatas,"
                " check the table name and make sure it exists");
    }

//TODO Switch to unordered maps for efficiency

    std::map<std::string, ColumnMeta> metadatas;

/*** build metadata ***/

    CassIterator *iterator = cass_iterator_columns_from_table_meta(table_meta);
    while (cass_iterator_next(iterator)) {
        const CassColumnMeta *cmeta = cass_iterator_get_column_meta(iterator);

        const char *value;
        size_t length;
        cass_column_meta_name(cmeta, &value, &length);

        const CassDataType *type = cass_column_meta_data_type(cmeta);

        metadatas[value] = {};
        metadatas[value].type = cass_data_type_type(type);
        metadatas[value].col_type = cass_column_meta_type(cmeta);
    }
    cass_iterator_free(iterator);
    cass_schema_meta_free(schema_meta);


    std::string key = keys_names[0]["name"];
    if (key.empty()) throw ModuleException("Empty key name given on position 0");
    std::string select_where = key + "=? ";
    std::string keys = key;
    std::string col;
    std::string tokens_keys = "";
    if (metadatas[key].col_type == CASS_COLUMN_TYPE_PARTITION_KEY) tokens_keys += key;

    for (uint16_t i = 1; i < n_keys; ++i) {
        key = keys_names[i]["name"];
        if (key.empty()) throw ModuleException("Empty key name given on position: " + std::to_string(i));
        keys += "," + key;
        select_where += "AND " + key + "=? ";
        if (metadatas[key].col_type == CASS_COLUMN_TYPE_PARTITION_KEY) {
            if (!tokens_keys.empty()) tokens_keys += ",";
            tokens_keys += key;
        }
    }
    if (tokens_keys.empty()) throw ModuleException("No partition key detected among the keys: " + keys);

    std::string cols = "";
    if (!columns_names.empty()) {
        cols += columns_names[0]["name"]; //TODO Check for *
        if (cols.empty()) throw ModuleException("Empty column name given on position 0");
        for (uint16_t i = 1; i < n_cols; ++i) {
            col = columns_names[i]["name"];
            if (col.empty()) throw ModuleException("Empty column name given onposition: " + std::to_string(i));
            cols += "," + col;
        }
    }
    std::string keys_and_cols = keys;
{'fraction_non_alphanumeric': 0.12027044597581589, 'fraction_numerical': 0.005070862046547913, 'mean_word_length': 2.298949174351276, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4004468', 'n_tokens_mistral': 4767, 'n_tokens_neox': 4633, 'n_words': 1048}