<commit_before>#include <detect/detect_re2c.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

int
detect_re2c_add_data(
    struct detect_re2c *ctx, const void *data, size_t siz, bool fin)
{
    assert(ctx->data == NULL ||
           ctx->pos == ctx->data + ctx->siz);

    if (!ctx->tmp_data_in_use) {
        ctx->pos = data;
        ctx->start = data;
        ctx->marker = NULL;
        ctx->ctxmarker = NULL;
    }
    ctx->data = data;
    ctx->siz = siz;
    ctx->data_copied = 0;
    ctx->fin = fin;

    return (0);
}

static bool
detect_re2c_chk_switch_to_data(
    struct detect_re2c *ctx, const unsigned char **end)
{
    size_t data_since_start;
    const unsigned char *tmp_data_end = *end;
    const unsigned char *data_copied_ptr;

    data_since_start = tmp_data_end - ctx->start;
    if (ctx->data_copied < data_since_start)
        return (false);

    data_copied_ptr = ctx->data + ctx->data_copied;

    ctx->pos = data_copied_ptr - (tmp_data_end - ctx->pos);
    ctx->start = data_copied_ptr - data_since_start;
    ctx->tmp_data_in_use = false;
    ctx->data_copied = 0;
    ctx->tmp_data_siz = 0;
    *end = ctx->data + ctx->siz;

    return (true);
}

int
detect_re2c_prepare_input(
    struct detect_re2c *ctx, const unsigned char **end, unsigned maxfill)
{
    if (!ctx->tmp_data_in_use) {
        *end = ctx->data + ctx->siz;
        goto fill;
    }

    *end = ctx->tmp_data + ctx->tmp_data_siz;
    detect_re2c_chk_switch_to_data(ctx, end);

  fill:<commit_msg>finalize correctly on no data<commit_after>#include <detect/detect_re2c.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>

int
detect_re2c_add_data(
    struct detect_re2c *ctx, const void *data, size_t siz, bool fin)
{
    assert(ctx->data == NULL ||
           ctx->pos == ctx->data + ctx->siz);

    if (!ctx->tmp_data_in_use) {
        ctx->pos = data;
        ctx->start = data;
        ctx->marker = NULL;
        ctx->ctxmarker = NULL;
    }
    ctx->data = data;
    ctx->siz = siz;
    ctx->data_copied = 0;
    ctx->fin = fin;

    return (0);
}

static bool
detect_re2c_chk_switch_to_data(
    struct detect_re2c *ctx, const unsigned char **end)
{
    size_t data_since_start;
    const unsigned char *tmp_data_end = *end;
    const unsigned char *data_copied_ptr;

    data_since_start = tmp_data_end - ctx->start;
    if (ctx->data_copied < data_since_start)
        return (false);

    data_copied_ptr = ctx->data + ctx->data_copied;

    ctx->pos = data_copied_ptr - (tmp_data_end - ctx->pos);
    ctx->start = data_copied_ptr - data_since_start;
    ctx->tmp_data_in_use = false;
    ctx->data_copied = 0;
    ctx->tmp_data_siz = 0;
    *end = ctx->data + ctx->siz;

    return (true);
}

int
detect_re2c_prepare_input(
    struct detect_re2c *ctx, const unsigned char **end, unsigned maxfill)
{
    if (!ctx->tmp_data_in_use) {
        *end = ctx->data + ctx->siz;
        goto fill;
    }

    *end = ctx->tmp_data + ctx->tmp_data_siz;
    detect_re2c_chk_switch_to_data(ctx, end);

  fill:
    if (!ctx->yyfill_need && *end == ctx->pos && ctx->fin) {
        /*
         * A special case: we have to finalize the data
         * but have no more characters in the buffer.
         * Require at least one character.
         */
        ctx->yyfill_need = 1;
    }
{'fraction_non_alphanumeric': 0.13117699910152741, 'fraction_numerical': 0.007487271638215035, 'mean_word_length': 2.71523915461624, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63951', 'n_tokens_mistral': 1258, 'n_tokens_neox': 1235, 'n_words': 274}