<commit_before>/*
 * Process CM4all commands in a HTML stream, e.g. embeddings.
 *
 * author: <NAME> <<EMAIL>>
 */

#include "istream.h"
#include "growing-buffer.h"
#include "valgrind.h"

#include <daemon/log.h>

#include <assert.h>

struct substitution {
    struct substitution *next;
    struct replace *replace;
    off_t start, end;
    istream_t istream;
};

struct replace {
    struct istream output;
    istream_t input;

    unsigned had_input:1, had_output:1;

    int finished;
    struct growing_buffer *buffer;
    off_t source_length, position;

    struct substitution *first_substitution, **append_substitution_p;

    int read_locked;

#ifndef NDEBUG
    off_t last_substitution_end;
#endif
};

/** is this substitution object the last chunk in this stream, i.e. is
    there no source data following it? */
static inline int
substitution_is_tail(const struct substitution *s)
{
    assert(s != NULL);
    assert(s->replace != NULL);
    assert(s->replace->buffer == NULL || s->end <= s->replace->source_length);

    return s->next == NULL && (s->replace->buffer == NULL ||
                               s->end == s->replace->source_length);
}

static void
replace_read(struct replace *replace);

/**
 * Activate the next substitution object after s.
 */
static void
replace_to_next_substitution(struct replace *replace, struct substitution *s)
{
    assert(replace->first_substitution == s);
    assert(replace->buffer == NULL || replace->position == s->start);
    assert(s->istream == NULL);
    assert(replace->buffer == NULL || s->start <= s->end);

    if (replace->buffer != NULL) {
        growing_buffer_consume(replace->buffer, s->end - s->start);
        replace->position = s->end;
    }
<commit_msg>added local var<commit_after>/*
 * Process CM4all commands in a HTML stream, e.g. embeddings.
 *
 * author: <NAME> <<EMAIL>>
 */

#include "istream.h"
#include "growing-buffer.h"
#include "valgrind.h"

#include <daemon/log.h>

#include <assert.h>

struct substitution {
    struct substitution *next;
    struct replace *replace;
    off_t start, end;
    istream_t istream;
};

struct replace {
    struct istream output;
    istream_t input;

    unsigned had_input:1, had_output:1;

    int finished;
    struct growing_buffer *buffer;
    off_t source_length, position;

    struct substitution *first_substitution, **append_substitution_p;

    int read_locked;

#ifndef NDEBUG
    off_t last_substitution_end;
#endif
};

/** is this substitution object the last chunk in this stream, i.e. is
    there no source data following it? */
static inline int
substitution_is_tail(const struct substitution *s)
{
    struct replace *replace = s->replace;

    assert(s->replace != NULL);
    assert(s->replace->buffer == NULL || s->end <= s->replace->source_length);

    return s->next == NULL && (replace->buffer == NULL ||
                               s->end == replace->source_length);
}

static void
replace_read(struct replace *replace);

/**
 * Activate the next substitution object after s.
 */
static void
replace_to_next_substitution(struct replace *replace, struct substitution *s)
{
    assert(replace->first_substitution == s);
    assert(replace->buffer == NULL || replace->position == s->start);
    assert(s->istream == NULL);
    assert(replace->buffer == NULL || s->start <= s->end);

    if (replace->buffer != NULL) {
        growing_buffer_consume(replace->buffer, s->end - s->start);
        replace->position = s->end;
    }

{'fraction_non_alphanumeric': 0.13218390804597702, 'fraction_numerical': 0.0017241379310344827, 'mean_word_length': 3.389659520807062, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4700753', 'n_tokens_mistral': 1099, 'n_tokens_neox': 1055, 'n_words': 294}