<commit_before>#include "Prefetch.h"


#define CHECK_CASS(msg) if(rc != CASS_OK){ \
std::cerr<<msg<<std::endl; \
throw std::runtime_error(msg); };

Prefetch::Prefetch(const std::vector<std::pair<int64_t, int64_t>> *token_ranges, uint32_t buff_size,
                   TupleRowFactory *tuple_factory, CassSession *session, std::string query) {
    this->session = session;
    this->t_factory = tuple_factory;
    this->tokens = token_ranges;
    CassFuture *future = cass_session_prepare(session, query.c_str());
    CassError rc = cass_future_error_code(future);
    CHECK_CASS("prefetch cannot prepare");
    this->prepared_query = cass_future_get_prepared(future);
    cass_future_free(future);
    this->data.set_capacity(buff_size);
    this->worker = new std::thread{&Prefetch::consume_tokens, this};

}

PyObject* Prefetch::get_next(){
    if(completed){
        PyErr_SetNone(PyExc_StopIteration);
        return NULL;
    }
    TupleRow *response = NULL;
    data.pop(response);
    if (!response) {
        completed = true;
        PyErr_SetNone(PyExc_StopIteration);
        return NULL;
    }
    PyObject* toberet= t_factory->tuple_as_py(response);
    delete(response);
    return toberet;
}

void Prefetch::consume_tokens() {<commit_msg>No longer returns empty list when tuple row has 0 elements<commit_after>#include "Prefetch.h"


#define CHECK_CASS(msg) if(rc != CASS_OK){ \
std::cerr<<msg<<std::endl; \
throw std::runtime_error(msg); };

Prefetch::Prefetch(const std::vector<std::pair<int64_t, int64_t>> *token_ranges, uint32_t buff_size,
                   TupleRowFactory *tuple_factory, CassSession *session, std::string query) {
    this->session = session;
    this->t_factory = tuple_factory;
    this->tokens = token_ranges;
    CassFuture *future = cass_session_prepare(session, query.c_str());
    CassError rc = cass_future_error_code(future);
    CHECK_CASS("prefetch cannot prepare");
    this->prepared_query = cass_future_get_prepared(future);
    cass_future_free(future);
    this->data.set_capacity(buff_size);
    this->worker = new std::thread{&Prefetch::consume_tokens, this};

}

PyObject* Prefetch::get_next(){
    if(completed){
        PyErr_SetNone(PyExc_StopIteration);
        return NULL;
    }
    TupleRow *response = NULL;
    data.pop(response);
    if (!response||response->n_elem()==0) {
        completed = true;
        PyErr_SetNone(PyExc_StopIteration);
        return NULL;
    }
    PyObject* toberet= t_factory->tuple_as_py(response);
    delete(response);
    return toberet;
}

void Prefetch::consume_tokens() {
{'fraction_non_alphanumeric': 0.12975912975912976, 'fraction_numerical': 0.005439005439005439, 'mean_word_length': 4.170682730923695, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1037883', 'n_tokens_mistral': 877, 'n_tokens_neox': 849, 'n_words': 152}