<commit_before>/*
 * Helper functions for buffered istream implementations.
 *
 * author: <NAME> <<EMAIL>>
 */

#include "istream.h"
#include "fifo-buffer.h"

/**
 * @return the number of bytes still in the buffer
 */
static inline size_t
istream_buffer_consume(struct istream *istream, fifo_buffer_t buffer)
{
    const void *data;
    size_t length, consumed;
    
    data = fifo_buffer_read(buffer, &length);
    if (data == NULL)
        return 0;

    consumed = istream_invoke_data(istream, data, length);
    fifo_buffer_consume(buffer, consumed);
    return length - consumed;
}
<commit_msg>header ifdef<commit_after>/*
 * Helper functions for buffered istream implementations.
 *
 * author: <NAME> <<EMAIL>>
 */

#ifndef __ISTREAM_BUFFER_H
#define __ISTREAM_BUFFER_H

#include "istream.h"
#include "fifo-buffer.h"

/**
 * @return the number of bytes still in the buffer
 */
static inline size_t
istream_buffer_consume(struct istream *istream, fifo_buffer_t buffer)
{
    const void *data;
    size_t length, consumed;
    
    data = fifo_buffer_read(buffer, &length);
    if (data == NULL)
        return 0;

    consumed = istream_invoke_data(istream, data, length);
    fifo_buffer_consume(buffer, consumed);
    return length - consumed;
}

#endif

{'fraction_non_alphanumeric': 0.10855784469096671, 'fraction_numerical': 0.001584786053882726, 'mean_word_length': 3.839080459770115, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3353407', 'n_tokens_mistral': 424, 'n_tokens_neox': 417, 'n_words': 114}