<commit_before>#include "http_client_pool.h"
#include <netinet/tcp.h>
#include <sys/types.h>
#include "sstr.h"

#define CLIENT_STACK_SIZE  65536

SSTRL(CRLFCRLF, "\r\n\r\n");
SSTRL(CRLF, "\r\n");
const char CR = '\r';
SSTRL(CONTENT_LENGTH, "\r\nContent-Length: ");
SSTRL(TRANSFER_ENCODING, "\r\nTransfer-Encoding: ");

SSTRL(CONNECTION, "\r\nConnection: ");
SSTRL(CONNECTION_CLOSE, "close");


int http_client_pool_init(struct http_client_pool *http_client_pool, size_t initial, size_t grow) {
    if (0 > ctx_pool_init(&http_client_pool->ctx_pool, initial, grow, CLIENT_STACK_SIZE, sizeof(struct http_client_context)))
        return -1;
    return 0;
}

#define CLIENT_ERROR()                                   \
    {                                                    \
        perror("client error\n"); /* TODO: remove */     \
        close(fd);                                       \
        return;                                          \
    }

#define READ_DATA(cond)                                     \
    while(cond) {                                           \
        if (res == 0)                                       \
            CLIENT_ERROR(); /* partial response */          \
        yield();                                            \
        if ((res = vmbuf_read(&ctx->response, fd)) < 0)     \
            CLIENT_ERROR();                                 \
    }


void http_client_fiber_main(void) {
    struct http_client_context *ctx = http_client_get_context();
    int fd = current_ctx->fd;
    int persistent = 0;
    epoll_worker_set_last_fd(fd); /* needed in the case where epoll_wait never occured */

    /*
     * write request
     */
    int res;
    for (; (res = vmbuf_write(&ctx->request, fd)) == 0; yield());
    if (0 > res)
        perror("write");
    /*
     * HTTP header
     */
    uint32_t eoh_ofs;
    char *data;
    char *eoh;
    res = 1; /* READ_DATA checks res */
    READ_DATA(NULL == (eoh = strstr(data = vmbuf_data(&ctx->response), CRLFCRLF)));
    eoh_ofs = eoh - data + SSTRLEN(CRLFCRLF);
    *eoh = 0;
    char *p = strstr(data, CONNECTION);
    if (p != NULL) {
        p += SSTRLEN(CONNECTION);
        persistent = (0 == SSTRNCMPI(CONNECTION_CLOSE, p) ? 0 : 1);
    }
    SSTRL(HTTP, "HTTP/");
    if (0 != SSTRNCMP(HTTP, data))
        CLIENT_ERROR();

    p = strchrnul(data, ' ');
    int code = (*p ? atoi(p + 1) : 0);
    if (0 == code)
        CLIENT_ERROR();
    if (code == 204 || code == 304) /* No Content,  Not Modified */
        return;

    char *content_len = strstr(data, CONTENT_LENGTH);
    if (NULL != content_len) {
        content_len += SSTRLEN(CONTENT_LENGTH);
        size_t content_end = eoh_ofs + atoi(content_len);
        READ_DATA(vmbuf_wlocpos(&ctx->response) < content_end);
    } else {
        char *transfer_encoding_str = strstr(data, TRANSFER_ENCODING);
        if (NULL != transfer_encoding_str &&
            0 == SSTRNCMP(transfer_encoding_str + SSTRLEN(TRANSFER_ENCODING), "chunked")) {
#if 0

            size_t chunk_start = eoh_ofs;
            data = vmbuf_data(&ctx->response);
            *vmbuf_wloc(&ctx->response) = 0;
            char *chunk_size = data + chunk_start;
            /*
              <chunk size in hex>\r\n
              <..... chunk data .....>
              chunk size == 0 is end of chunks
            */
            char *p;
            READ_DATA((p = strchrnul(chunk_size, '\r')) == 0);

#endif

        } else {
            for (;;yield()) {
                if ((res = vmbuf_read(&ctx->response, fd)) < 0)
                    CLIENT_ERROR();<commit_msg>http_client fixes and improvements<commit_after>#include "http_client_pool.h"
#include <netinet/tcp.h>
#include <sys/types.h>
#include "sstr.h"

#define CLIENT_STACK_SIZE  65536

SSTRL(CRLFCRLF, "\r\n\r\n");
SSTRL(CRLF, "\r\n");
const char CR = '\r';
SSTRL(CONTENT_LENGTH, "\r\nContent-Length: ");
SSTRL(TRANSFER_ENCODING, "\r\nTransfer-Encoding: ");

SSTRL(CONNECTION, "\r\nConnection: ");
SSTRL(CONNECTION_CLOSE, "close");


int http_client_pool_init(struct http_client_pool *http_client_pool, size_t initial, size_t grow) {
    if (0 > ctx_pool_init(&http_client_pool->ctx_pool, initial, grow, CLIENT_STACK_SIZE, sizeof(struct http_client_context)))
        return -1;
    return 0;
}

#define CLIENT_ERROR()                                   \
    {                                                    \
        perror("client error\n"); /* TODO: remove */     \
        close(fd);                                       \
        return;                                          \
    }

#define _READ_MORE_DATA(cond, extra)                        \
    while(cond) {                                           \
        if (res == 0)                                       \
            CLIENT_ERROR(); /* partial response */          \
        yield();                                            \
        if ((res = vmbuf_read(&ctx->response, fd)) < 0)     \
            CLIENT_ERROR();                                 \
        extra;                                              \
    }

#define READ_MORE_DATA_STR(cond)                                \
    _READ_MORE_DATA(cond, *vmbuf_wloc(&ctx->response) = 0)

#define READ_MORE_DATA(cond)                    \
    _READ_MORE_DATA(cond, )


void http_client_fiber_main(void) {
    struct http_client_context *ctx = http_client_get_context();
    int fd = current_ctx->fd;
    int persistent = 0;
    epoll_worker_set_last_fd(fd); /* needed in the case where epoll_wait never occured */

    /*
     * write request
     */
    int res;
    for (; (res = vmbuf_write(&ctx->request, fd)) == 0; yield());
    if (0 > res)
        perror("write");
    /*
     * HTTP header
     */
    uint32_t eoh_ofs;
    char *data;
    char *eoh;
    res = vmbuf_read(&ctx->response, fd);
    *vmbuf_wloc(&ctx->response) = 0;
    READ_MORE_DATA_STR(NULL == (eoh = strstr(data = vmbuf_data(&ctx->response), CRLFCRLF)));
    eoh_ofs = eoh - data + SSTRLEN(CRLFCRLF);
    *eoh = 0;
    char *p = strstr(data, CONNECTION);
    if (p != NULL) {
        p += SSTRLEN(CONNECTION);
        persistent = (0 == SSTRNCMPI(CONNECTION_CLOSE, p) ? 0 : 1);
    }
    SSTRL(HTTP, "HTTP/");
    if (0 != SSTRNCMP(HTTP, data))
        CLIENT_ERROR();

    p = strchrnul(data, ' ');
    int code = (*p ? atoi(p + 1) : 0);
    if (0 == code)
        CLIENT_ERROR();
    if (code == 204 || code == 304) /* No Content,  Not Modified */
        return;

    char *content_len = strstr(data, CONTENT_LENGTH);
    if (NULL != content_len) {
        content_len += SSTRLEN(CONTENT_LENGTH);
        size_t content_end = eoh_ofs + atoi(content_len);
        READ_MORE_DATA(vmbuf_wlocpos(&ctx->response) < content_end);
    } else {
        char *transfer_encoding_str = strstr(data, TRANSFER_ENCODING);
        if (NULL != transfer_encoding_str &&
            0 == SSTRNCMP(transfer_encoding_str + SSTRLEN(TRANSFER_ENCODING), "chunked")) {
            size_t chunk_start = eoh_ofs;
            /*
              <chunk size in hex>\r\n
              <..... chunk data .....>
              chunk size == 0 is end of chunks
            */
            char *p;
            READ_MORE_DATA_STR((p = strchrnul((data = vmbuf_data(&ctx->response)) + chunk_start, '\r')) == 0);
            uint32_t s = strtoul(data + chunk_start, NULL, 16);
            chunk_start = p - data + SSTRLEN(CRLF);
            (void)s;
        } else {
            for (;;yield()) {
                if ((res = vmbuf_read(&ctx->response, fd)) < 0)
                    CLIENT_ERROR();
{'fraction_non_alphanumeric': 0.13456708526107072, 'fraction_numerical': 0.00991407799074686, 'mean_word_length': 1.6810772501771793, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4589158', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2420, 'n_words': 500}