<commit_before>#ifndef _SNARKFRONT_DATA_BUFFER_HPP_
#define _SNARKFRONT_DATA_BUFFER_HPP_

#include <array>
#include <climits>
#include <cstdint>
#include <istream>
#include <memory>
#include <ostream>
#include <string>
#include <vector>

namespace snarkfront {

////////////////////////////////////////////////////////////////////////////////
// data buffer
//

template <typename T>
class DataBuffer
{
public:
    DataBuffer() = default;

    template <typename X>
    DataBuffer(X& a) // not const reference for std::ostream
        : m_buf(a)
    {}

    template <typename X, typename U>
    DataBuffer(X& a, const U b) // PrintHex constructor takes two arguments
        : m_buf(a, b)
    {}

    T& operator* () {
        return m_buf;
    }

    const T* operator-> () const {
        return std::addressof(m_buf);
    }

    void push8(const std::uint8_t a) {
        m_buf.pushOctet(a);
    }

    void push32(const std::uint32_t a) {
        push8(a >> 3 * CHAR_BIT);
        push8((a >> 2 * CHAR_BIT) & 0xff);
        push8((a >> CHAR_BIT) & 0xff);
        push8(a & 0xff);
    }

    void push64(const std::uint64_t a) {
        push32(a >> 4 * CHAR_BIT);
        push32(a & 0xffffffff);
    }

    void pushText(const std::string& a) {
        for (const auto& c : a)
            push8(c);
    }

    void push(const std::uint8_t a) { push8(a); }
    void push(const std::uint32_t a) { push32(a); }
    void push(const std::uint64_t a) { push64(a); }
    void push(const std::string& a) { pushText(a); }
    void push(const char* a) { push(std::string(a)); }

    template <typename U, std::size_t N>
    void push(const std::array<U, N>& a) {
        for (const auto& b : a)
            push(b);
    }

    template <typename U>
    void push(const std::vector<U>& a) {
        for (const auto& b : a)
            push(b);
    }

    void push(const DataBuffer& other) {
        for (const auto& a : other->data())
            push8(a);
    }

private:
    T m_buf;
};

// (extract) read from stream into data buffer object
template <typename T>
std::istream& operator>> (std::istream& is, DataBuffer<T>& a) {
    std::uint8_t c;
    while (! is.eof() && (is >> c)) {
        a.push8(c);
    }

    return is;
}

////////////////////////////////////////////////////////////////////////////////
// clear text buffer
//

class ClearText
{
public:
    ClearText() = default;
    ClearText(DataBuffer<ClearText>&);

    void clear();<commit_msg>bugfix extract whitespace from stream<commit_after>#ifndef _SNARKFRONT_DATA_BUFFER_HPP_
#define _SNARKFRONT_DATA_BUFFER_HPP_

#include <array>
#include <climits>
#include <cstdint>
#include <istream>
#include <memory>
#include <ostream>
#include <string>
#include <vector>

namespace snarkfront {

////////////////////////////////////////////////////////////////////////////////
// data buffer
//

template <typename T>
class DataBuffer
{
public:
    DataBuffer() = default;

    template <typename X>
    DataBuffer(X& a) // not const reference for std::ostream
        : m_buf(a)
    {}

    template <typename X, typename U>
    DataBuffer(X& a, const U b) // PrintHex constructor takes two arguments
        : m_buf(a, b)
    {}

    T& operator* () {
        return m_buf;
    }

    const T* operator-> () const {
        return std::addressof(m_buf);
    }

    void push8(const std::uint8_t a) {
        m_buf.pushOctet(a);
    }

    void push32(const std::uint32_t a) {
        push8(a >> 3 * CHAR_BIT);
        push8((a >> 2 * CHAR_BIT) & 0xff);
        push8((a >> CHAR_BIT) & 0xff);
        push8(a & 0xff);
    }

    void push64(const std::uint64_t a) {
        push32(a >> 4 * CHAR_BIT);
        push32(a & 0xffffffff);
    }

    void pushText(const std::string& a) {
        for (const auto& c : a)
            push8(c);
    }

    void push(const std::uint8_t a) { push8(a); }
    void push(const std::uint32_t a) { push32(a); }
    void push(const std::uint64_t a) { push64(a); }
    void push(const std::string& a) { pushText(a); }
    void push(const char* a) { push(std::string(a)); }

    template <typename U, std::size_t N>
    void push(const std::array<U, N>& a) {
        for (const auto& b : a)
            push(b);
    }

    template <typename U>
    void push(const std::vector<U>& a) {
        for (const auto& b : a)
            push(b);
    }

    void push(const DataBuffer& other) {
        for (const auto& a : other->data())
            push8(a);
    }

private:
    T m_buf;
};

// (extract) read from stream into data buffer object
template <typename T>
std::istream& operator>> (std::istream& is, DataBuffer<T>& a) {
    char c;
    while (!is.eof() && !!is.get(c))
        a.push8(c);

    return is;
}

////////////////////////////////////////////////////////////////////////////////
// clear text buffer
//

class ClearText
{
public:
    ClearText() = default;
    ClearText(DataBuffer<ClearText>&);

    void clear();
{'fraction_non_alphanumeric': 0.20065053872738361, 'fraction_numerical': 0.015653588127668226, 'mean_word_length': 2.00183038438072, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1248450', 'n_tokens_mistral': 1704, 'n_tokens_neox': 1621, 'n_words': 430}