<commit_before>#include "streamly.h"

static size_t header_handler(char * stream, size_t size, size_t nmemb, VALUE handler) {
    if(TYPE(handler) == T_STRING) {
        rb_str_buf_cat(handler, stream, size * nmemb);
    } else {
        rb_funcall(handler, rb_intern("call"), 1, rb_str_new(stream, size * nmemb));
    }
    return size * nmemb;
}

static size_t data_handler(char * stream, size_t size, size_t nmemb, VALUE handler) {
    if(TYPE(handler) == T_STRING) {
        rb_str_buf_cat(handler, stream, size * nmemb);
    } else {
        rb_funcall(handler, rb_intern("call"), 1, rb_str_new(stream, size * nmemb));
    }
    return size * nmemb;
}

static VALUE select_error(CURLcode code) {
    VALUE error = Qnil;
    
    switch (code) {
        case CURLE_UNSUPPORTED_PROTOCOL:
            error = eUnsupportedProtocol;
            break;
        case CURLE_URL_MALFORMAT:
            error = eURLFormatError;
            break;
        case CURLE_COULDNT_RESOLVE_HOST:
            error = eHostResolutionError;
            break;
        case CURLE_COULDNT_CONNECT:
            error = eConnectionFailed;
            break;
        case CURLE_PARTIAL_FILE:
            error = ePartialFileError;
            break;
        case CURLE_OPERATION_TIMEDOUT:
            error = eTimeoutError;
            break;
        case CURLE_TOO_MANY_REDIRECTS:
            error = eTooManyRedirects;
            break;
        default:
            error = eStreamlyError;
    }

    return error;
}

static VALUE each_http_header(VALUE header, VALUE self) {
    struct curl_instance *instance;
    Data_Get_Struct(self, struct curl_instance, instance);

    VALUE name = rb_obj_as_string(rb_ary_entry(header, 0));
    VALUE value = rb_obj_as_string(rb_ary_entry(header, 1));

    VALUE header_str = Qnil;
    header_str = rb_str_plus(name, rb_str_new2(": "));
    header_str = rb_str_plus(header_str, value);

    instance->headers = curl_slist_append(instance->headers, RSTRING_PTR(header_str));
    return Qnil;
}

void streamly_instance_free(struct curl_instance *curl) {
    curl_easy_cleanup(curl->handle);
    free(curl);
}

VALUE rb_streamly_new(VALUE klass) {
    struct curl_instance* curl;
    VALUE obj = Data_Make_Struct(klass, struct curl_instance, NULL, streamly_instance_free, curl);
    rb_obj_call_init(obj, 0, 0);
    return obj;
}

VALUE rb_streamly_init(VALUE self) {
    struct curl_instance *instance;
    Data_Get_Struct(self, struct curl_instance, instance);
    instance->handle = curl_easy_init();
    return self;
}

// head(url, headers = {})
VALUE rb_streamly_head(int argc, VALUE * argv, VALUE self) {
    CURLcode res;
    struct curl_instance *instance;
    GetInstance(self, instance);
    VALUE handler = Qnil, url = Qnil, headers = Qnil, blk = Qnil;
    
    rb_scan_args(argc, argv, "11&", &url, &headers, &blk);
    
    // Set handler to the passed block, so we can pass chunks to it
    // or just use a string so we can append the response to, and finally return it
    if (blk != Qnil) {
        handler = blk;
    } else {
        handler = rb_str_new2("");
    }
    
    if (instance->handle) {
        curl_easy_setopt(instance->handle, CURLOPT_FOLLOWLOCATION, 1);
        curl_easy_setopt(instance->handle, CURLOPT_MAXREDIRS, 3);
        
        curl_easy_setopt(instance->handle, CURLOPT_NOBODY, 1);
        curl_easy_setopt(instance->handle, CURLOPT_URL, RSTRING_PTR(url));
        
        // Header handling
        curl_easy_setopt(instance->handle, CURLOPT_HEADERFUNCTION, &header_handler);
        curl_easy_setopt(instance->handle, CURLOPT_HEADERDATA, handler);
        
        if (!NIL_P(headers)) {
            if (rb_type(headers) != T_HASH) {
              rb_raise(rb_eArgError, "Headers must be passed in as a hash.");
            }
            
            rb_iterate(rb_each, headers, each_http_header, self);
            
            curl_easy_setopt(instance->handle, CURLOPT_HTTPHEADER, instance->headers);
        }
        
        curl_easy_setopt(instance->handle, CURLOPT_ERRORBUFFER, instance->error_buf);
        
        res = curl_easy_perform(instance->handle);
        if (CURLE_OK != res) {
            rb_raise(select_error(res), instance->error_buf);
        }
        
        // always cleanup
        if (!NIL_P(headers)) {
            curl_slist_free_all(instance->headers);
            instance->headers = NULL;
        }
        curl_easy_reset(instance->handle);
        instance->upload_buf = NULL;
        
        if (TYPE(handler) == T_STRING) {
            return handler;
        }
    }
    return Qnil;
}

// get(url, headers = {})
VALUE rb_streamly_get(int argc, VALUE * argv, VALUE self) {
    CURLcode res;
    struct curl_instance *instance;
    GetInstance(self, instance);
    VALUE handler = Qnil, url = Qnil, headers = Qnil, blk = Qnil;
    
    rb_scan_args(argc, argv, "11&", &url, &headers, &blk);
    
    // Set handler to the passed block, so we can pass chunks to it
    // or just use a string so we can append the response to, and finally return it
    if (blk != Qnil) {
        handler = blk;
    } else {
        handler = rb_str_new2("");
    }
    
    if (instance->handle) {
        curl_easy_setopt(instance->handle, CURLOPT_FOLLOWLOCATION, 1);
        curl_easy_setopt(instance->handle, CURLOPT_MAXREDIRS, 3);
        
        curl_easy_setopt(instance->handle, CURLOPT_ENCODING, "identity, deflate, gzip");
        curl_easy_setopt(instance->handle, CURLOPT_URL, RSTRING_PTR(url));
        
        // TODO: add support to get these to the caller
        // Header handling
        // curl_easy_setopt(instance->handle, CURLOPT_HEADERFUNCTION, &header_handler);
        // curl_easy_setopt(instance->handle, CURLOPT_HEADERDATA, handler);
        
        // Body handling
        curl_easy_setopt(instance->handle, CURLOPT_WRITEFUNCTION, (curl_write_callback)&data_handler);
        curl_easy_setopt(instance->handle, CURLOPT_WRITEDATA, handler);
        
        if (!NIL_P(headers)) {
            if (rb_type(headers) != T_HASH) {<commit_msg>ensure get requests force the allocated curl handle back to GET<commit_after>#include "streamly.h"

static size_t header_handler(char * stream, size_t size, size_t nmemb, VALUE handler) {
    if(TYPE(handler) == T_STRING) {
        rb_str_buf_cat(handler, stream, size * nmemb);
    } else {
        rb_funcall(handler, rb_intern("call"), 1, rb_str_new(stream, size * nmemb));
    }
    return size * nmemb;
}

static size_t data_handler(char * stream, size_t size, size_t nmemb, VALUE handler) {
    if(TYPE(handler) == T_STRING) {
        rb_str_buf_cat(handler, stream, size * nmemb);
    } else {
        rb_funcall(handler, rb_intern("call"), 1, rb_str_new(stream, size * nmemb));
    }
    return size * nmemb;
}

static VALUE select_error(CURLcode code) {
    VALUE error = Qnil;
    
    switch (code) {
        case CURLE_UNSUPPORTED_PROTOCOL:
            error = eUnsupportedProtocol;
            break;
        case CURLE_URL_MALFORMAT:
            error = eURLFormatError;
            break;
        case CURLE_COULDNT_RESOLVE_HOST:
            error = eHostResolutionError;
            break;
        case CURLE_COULDNT_CONNECT:
            error = eConnectionFailed;
            break;
        case CURLE_PARTIAL_FILE:
            error = ePartialFileError;
            break;
        case CURLE_OPERATION_TIMEDOUT:
            error = eTimeoutError;
            break;
        case CURLE_TOO_MANY_REDIRECTS:
            error = eTooManyRedirects;
            break;
        default:
            error = eStreamlyError;
    }

    return error;
}

static VALUE each_http_header(VALUE header, VALUE self) {
    struct curl_instance *instance;
    Data_Get_Struct(self, struct curl_instance, instance);

    VALUE name = rb_obj_as_string(rb_ary_entry(header, 0));
    VALUE value = rb_obj_as_string(rb_ary_entry(header, 1));

    VALUE header_str = Qnil;
    header_str = rb_str_plus(name, rb_str_new2(": "));
    header_str = rb_str_plus(header_str, value);

    instance->headers = curl_slist_append(instance->headers, RSTRING_PTR(header_str));
    return Qnil;
}

void streamly_instance_free(struct curl_instance *curl) {
    curl_easy_cleanup(curl->handle);
    free(curl);
}

VALUE rb_streamly_new(VALUE klass) {
    struct curl_instance* curl;
    VALUE obj = Data_Make_Struct(klass, struct curl_instance, NULL, streamly_instance_free, curl);
    rb_obj_call_init(obj, 0, 0);
    return obj;
}

VALUE rb_streamly_init(VALUE self) {
    struct curl_instance *instance;
    Data_Get_Struct(self, struct curl_instance, instance);
    instance->handle = curl_easy_init();
    return self;
}

// head(url, headers = {})
VALUE rb_streamly_head(int argc, VALUE * argv, VALUE self) {
    CURLcode res;
    struct curl_instance *instance;
    GetInstance(self, instance);
    VALUE handler = Qnil, url = Qnil, headers = Qnil, blk = Qnil;
    
    rb_scan_args(argc, argv, "11&", &url, &headers, &blk);
    
    // Set handler to the passed block, so we can pass chunks to it
    // or just use a string so we can append the response to, and finally return it
    if (blk != Qnil) {
        handler = blk;
    } else {
        handler = rb_str_new2("");
    }
    
    if (instance->handle) {
        curl_easy_setopt(instance->handle, CURLOPT_FOLLOWLOCATION, 1);
        curl_easy_setopt(instance->handle, CURLOPT_MAXREDIRS, 3);
        
        curl_easy_setopt(instance->handle, CURLOPT_NOBODY, 1);
        curl_easy_setopt(instance->handle, CURLOPT_URL, RSTRING_PTR(url));
        
        // Header handling
        curl_easy_setopt(instance->handle, CURLOPT_HEADERFUNCTION, &header_handler);
        curl_easy_setopt(instance->handle, CURLOPT_HEADERDATA, handler);
        
        if (!NIL_P(headers)) {
            if (rb_type(headers) != T_HASH) {
              rb_raise(rb_eArgError, "Headers must be passed in as a hash.");
            }
            
            rb_iterate(rb_each, headers, each_http_header, self);
            
            curl_easy_setopt(instance->handle, CURLOPT_HTTPHEADER, instance->headers);
        }
        
        curl_easy_setopt(instance->handle, CURLOPT_ERRORBUFFER, instance->error_buf);
        
        res = curl_easy_perform(instance->handle);
        if (CURLE_OK != res) {
            rb_raise(select_error(res), instance->error_buf);
        }
        
        // always cleanup
        if (!NIL_P(headers)) {
            curl_slist_free_all(instance->headers);
            instance->headers = NULL;
        }
        curl_easy_reset(instance->handle);
        instance->upload_buf = NULL;
        
        if (TYPE(handler) == T_STRING) {
            return handler;
        }
    }
    return Qnil;
}

// get(url, headers = {})
VALUE rb_streamly_get(int argc, VALUE * argv, VALUE self) {
    CURLcode res;
    struct curl_instance *instance;
    GetInstance(self, instance);
    VALUE handler = Qnil, url = Qnil, headers = Qnil, blk = Qnil;
    
    rb_scan_args(argc, argv, "11&", &url, &headers, &blk);
    
    // Set handler to the passed block, so we can pass chunks to it
    // or just use a string so we can append the response to, and finally return it
    if (blk != Qnil) {
        handler = blk;
    } else {
        handler = rb_str_new2("");
    }
    
    if (instance->handle) {
        curl_easy_setopt(instance->handle, CURLOPT_FOLLOWLOCATION, 1);
        curl_easy_setopt(instance->handle, CURLOPT_MAXREDIRS, 3);
        
        curl_easy_setopt(instance->handle, CURLOPT_HTTPGET, 1);
        curl_easy_setopt(instance->handle, CURLOPT_ENCODING, "identity, deflate, gzip");
        curl_easy_setopt(instance->handle, CURLOPT_URL, RSTRING_PTR(url));
        
        // TODO: add support to get these to the caller
        // Header handling
        // curl_easy_setopt(instance->handle, CURLOPT_HEADERFUNCTION, &header_handler);
        // curl_easy_setopt(instance->handle, CURLOPT_HEADERDATA, handler);
        
        // Body handling
        curl_easy_setopt(instance->handle, CURLOPT_WRITEFUNCTION, (curl_write_callback)&data_handler);
        curl_easy_setopt(instance->handle, CURLOPT_WRITEDATA, handler);
        
        if (!NIL_P(headers)) {
            if (rb_type(headers) != T_HASH) {
{'fraction_non_alphanumeric': 0.1013226649248857, 'fraction_numerical': 0.0030209013716525146, 'mean_word_length': 2.6143405134257893, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1594264', 'n_tokens_mistral': 4052, 'n_tokens_neox': 3873, 'n_words': 871}