<commit_before>/*
 * This istream filter removes HTTP chunking.
 *
 * author: <NAME> <<EMAIL>>
 */

#include "istream.h"
#include "fifo-buffer.h"

#include <daemon/log.h>

#include <assert.h>
#include <string.h>

struct istream_dechunk {
    struct istream output;
    istream_t input;
    enum {
        NONE,
        CLOSED,
        SIZE,
        AFTER_SIZE,
        DATA,
        AFTER_DATA,
        TRAILER,
        TRAILER_DATA
    } state;
    size_t size;
    unsigned had_input:1, had_output:1;

    void (*eof_callback)(void *ctx);
    void *ctx;
};


static void
dechunk_close(struct istream_dechunk *dechunk)
{
    dechunk->state = CLOSED;

    if (dechunk->input != NULL)
        istream_clear_unref_handler(&dechunk->input);
    
    istream_invoke_abort(&dechunk->output);
}


static void
dechunk_eof_detected(struct istream_dechunk *dechunk)
{
    assert(dechunk->input != NULL);
    assert(dechunk->state == TRAILER);
    assert(dechunk->size == 0);

    istream_clear_unref_handler(&dechunk->input);

    pool_ref(dechunk->output.pool);

    if (dechunk->eof_callback != NULL)
        dechunk->eof_callback(dechunk->ctx);

    istream_invoke_eof(&dechunk->output);

    pool_unref(dechunk->output.pool);
}


/*
 * istream handler
 *
 */

static size_t
dechunk_input_data(const void *data0, size_t length, void *ctx)
{
    struct istream_dechunk *dechunk = ctx;
    const char *data = data0;
    size_t position = 0, digit, size, nbytes;

    assert(dechunk->input != NULL);

    dechunk->had_input = 1;

    while (position < length) {
        switch (dechunk->state) {
        case NONE:
        case SIZE:
            if (data[position] >= '0' && data[position] <= '9') {
                digit = data[position] - '0';
            } else if (data[position] >= 'a' && data[position] <= 'f') {
                digit = data[position] - 'a' + 0xa;
            } else if (data[position] >= 'A' && data[position] <= 'F') {
                digit = data[position] - 'A' + 0xa;
            } else if (dechunk->state == SIZE) {
                dechunk->state = AFTER_SIZE;
                ++position;
                continue;
            } else {
                daemon_log(2, "chunk length expected\n");
                dechunk_close(dechunk);
                return 0;
            }

            if (dechunk->state == NONE) {
                dechunk->state = SIZE;
                dechunk->size = 0;
            }

            ++position;
            dechunk->size = dechunk->size * 0x10 + digit;
            break;

        case CLOSED:
            assert(0);
            break;

        case AFTER_SIZE:
            if (data[position++] == '\n') {
                if (dechunk->size == 0)
                    dechunk->state = TRAILER;
                else
                    dechunk->state = DATA;
            }
            break;

        case DATA:
            assert(dechunk->size > 0);

            size = length - position;
            if (size > dechunk->size)
                size = dechunk->size;
            dechunk->had_output = 1;
            nbytes = istream_invoke_data(&dechunk->output, data + position, size);
            assert(nbytes <= size);

            if (nbytes == 0)
                return dechunk->state == CLOSED ? 0 : position;

            dechunk->size -= nbytes;
            if (dechunk->size == 0)
                dechunk->state = AFTER_DATA;

            position += nbytes;
            break;

        case AFTER_DATA:
            if (data[position] == '\n') {
                dechunk->state = NONE;
            } else if (data[position] != '\r') {
                daemon_log(2, "newline expected\n");
                dechunk_close(dechunk);
                return 0;
            }
            ++position;
            break;

        case TRAILER:
            if (data[position] == '\n') {
                dechunk_eof_detected(dechunk);
                return dechunk->state == CLOSED ? 0 : position + 1;
            } else if (data[position] == '\r') {
                ++position;
            } else {
                ++position;
                dechunk->state = TRAILER_DATA;
            }
            break;

        case TRAILER_DATA:
            if (data[position++] == '\n')
                dechunk->state = TRAILER;
            break;
        }
    }

    return position;
}

static void
dechunk_input_eof(void *ctx)
{
    struct istream_dechunk *dechunk = ctx;

    istream_clear_unref(&dechunk->input);

    if (dechunk->state != NONE) {
        daemon_log(2, "premature EOF in dechunker");<commit_msg>added state EOF_DETECTED<commit_after>/*
 * This istream filter removes HTTP chunking.
 *
 * author: <NAME> <<EMAIL>>
 */

#include "istream.h"
#include "fifo-buffer.h"

#include <daemon/log.h>

#include <assert.h>
#include <string.h>

struct istream_dechunk {
    struct istream output;
    istream_t input;
    enum {
        NONE,
        CLOSED,
        SIZE,
        AFTER_SIZE,
        DATA,
        AFTER_DATA,
        TRAILER,
        TRAILER_DATA,
        EOF_DETECTED
    } state;
    size_t size;
    unsigned had_input:1, had_output:1;

    void (*eof_callback)(void *ctx);
    void *ctx;
};


static void
dechunk_close(struct istream_dechunk *dechunk)
{
    dechunk->state = CLOSED;

    if (dechunk->input != NULL)
        istream_clear_unref_handler(&dechunk->input);
    
    istream_invoke_abort(&dechunk->output);
}


static void
dechunk_eof_detected(struct istream_dechunk *dechunk)
{
    assert(dechunk->input != NULL);
    assert(dechunk->state == TRAILER);
    assert(dechunk->size == 0);

    dechunk->state = EOF_DETECTED;

    istream_clear_unref_handler(&dechunk->input);

    pool_ref(dechunk->output.pool);

    if (dechunk->eof_callback != NULL)
        dechunk->eof_callback(dechunk->ctx);

    istream_invoke_eof(&dechunk->output);

    pool_unref(dechunk->output.pool);
}


/*
 * istream handler
 *
 */

static size_t
dechunk_input_data(const void *data0, size_t length, void *ctx)
{
    struct istream_dechunk *dechunk = ctx;
    const char *data = data0;
    size_t position = 0, digit, size, nbytes;

    assert(dechunk->input != NULL);

    dechunk->had_input = 1;

    while (position < length) {
        switch (dechunk->state) {
        case NONE:
        case SIZE:
            if (data[position] >= '0' && data[position] <= '9') {
                digit = data[position] - '0';
            } else if (data[position] >= 'a' && data[position] <= 'f') {
                digit = data[position] - 'a' + 0xa;
            } else if (data[position] >= 'A' && data[position] <= 'F') {
                digit = data[position] - 'A' + 0xa;
            } else if (dechunk->state == SIZE) {
                dechunk->state = AFTER_SIZE;
                ++position;
                continue;
            } else {
                daemon_log(2, "chunk length expected\n");
                dechunk_close(dechunk);
                return 0;
            }

            if (dechunk->state == NONE) {
                dechunk->state = SIZE;
                dechunk->size = 0;
            }

            ++position;
            dechunk->size = dechunk->size * 0x10 + digit;
            break;

        case CLOSED:
            assert(0);
            break;

        case AFTER_SIZE:
            if (data[position++] == '\n') {
                if (dechunk->size == 0)
                    dechunk->state = TRAILER;
                else
                    dechunk->state = DATA;
            }
            break;

        case DATA:
            assert(dechunk->size > 0);

            size = length - position;
            if (size > dechunk->size)
                size = dechunk->size;
            dechunk->had_output = 1;
            nbytes = istream_invoke_data(&dechunk->output, data + position, size);
            assert(nbytes <= size);

            if (nbytes == 0)
                return dechunk->state == CLOSED ? 0 : position;

            dechunk->size -= nbytes;
            if (dechunk->size == 0)
                dechunk->state = AFTER_DATA;

            position += nbytes;
            break;

        case AFTER_DATA:
            if (data[position] == '\n') {
                dechunk->state = NONE;
            } else if (data[position] != '\r') {
                daemon_log(2, "newline expected\n");
                dechunk_close(dechunk);
                return 0;
            }
            ++position;
            break;

        case TRAILER:
            if (data[position] == '\n') {
                dechunk_eof_detected(dechunk);
                return dechunk->state == CLOSED ? 0 : position + 1;
            } else if (data[position] == '\r') {
                ++position;
            } else {
                ++position;
                dechunk->state = TRAILER_DATA;
            }
            break;

        case TRAILER_DATA:
            if (data[position++] == '\n')
                dechunk->state = TRAILER;
            break;

        case EOF_DETECTED:
            assert(0);
            return 0;
        }
    }

    return position;
}

static void
dechunk_input_eof(void *ctx)
{
    struct istream_dechunk *dechunk = ctx;

    istream_clear_unref(&dechunk->input);

    if (dechunk->state != NONE) {
        daemon_log(2, "premature EOF in dechunker");
{'fraction_non_alphanumeric': 0.1257562662057044, 'fraction_numerical': 0.006698357821953328, 'mean_word_length': 1.605403884041655, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 110, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1267575', 'n_tokens_mistral': 2987, 'n_tokens_neox': 2794, 'n_words': 605}