<commit_before>#ifndef FEK_STREAMS_HPP
#define FEK_STREAMS_HPP

#include <cassert>
#include <cstring>

#include <streambuf>
#include <istream>
#include <iterator>
#include <limits>
#include <array>

// for now-owning istream (istringstream makes a copy)
class streambufview : public std::streambuf {
public:
    // if c++>=17 add string_view constructor
    streambufview(const char* data, std::size_t len) : m_begin(data), m_end(m_begin + len), m_current(m_begin) { }
    explicit streambufview(const char* data) : streambufview(data, std::strlen(data)) { }

    // FIXME: verify that iterator is randomaccess and of type char
    template <class it>
    streambufview(it begin_, it end_) : m_begin(&(*begin_), &(*end_)) {
        static_assert(std::is_same<std::random_access_iterator_tag, typename std::iterator_traits<it>::iterator_category>::value,
            "streambufview only accepts random access iterators.");
    }

private:
    virtual int_type underflow() override {
        if (m_current == m_end) {
            return traits_type::eof();<commit_msg>updated FIXME<commit_after>#ifndef FEK_STREAMS_HPP
#define FEK_STREAMS_HPP

#include <cassert>
#include <cstring>

#include <streambuf>
#include <istream>
#include <iterator>
#include <limits>
#include <array>

// for now-owning istream (istringstream makes a copy)
class streambufview : public std::streambuf {
public:
    // if c++>=17 add string_view constructor
    streambufview(const char* data, std::size_t len) : m_begin(data), m_end(m_begin + len), m_current(m_begin) { }
    explicit streambufview(const char* data) : streambufview(data, std::strlen(data)) { }

    // FIXME: verify that iterator is of type char
    template <class it>
    streambufview(it begin_, it end_) : m_begin(&(*begin_), &(*end_)) {
        static_assert(std::is_same<std::random_access_iterator_tag, typename std::iterator_traits<it>::iterator_category>::value,
            "streambufview only accepts random access iterators.");
    }

private:
    virtual int_type underflow() override {
        if (m_current == m_end) {
            return traits_type::eof();
{'fraction_non_alphanumeric': 0.12387706855791962, 'fraction_numerical': 0.0018912529550827422, 'mean_word_length': 4.568421052631579, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3056712', 'n_tokens_mistral': 677, 'n_tokens_neox': 691, 'n_words': 174}