<commit_before>#ifndef TYPE_READER_H
#define TYPE_READER_H

#include <istream>

namespace cpputils
{

template<typename T, int S>
class array_reader {
public:
    void read_from_stream(std::istream &stream) {
        stream.read((char *)&buffer[0], sizeof(T) * S);
    }

    int size() const {
        return S * sizeof(T);
    }

    T value(int index) const {
        return buffer[index];
    }

private:
    T buffer[S];
};

template<typename T>
class value_reader : public array_reader<T, 1> {
public:
    T value() const {
        return array_reader<T, 1>::value(0);
    }
};

template<typename T, typename Allocator = std::allocator<T>>
class vector_reader {
public:
    vector_reader(int size) : buffer(size) {}

    void read_from_stream(std::istream &stream) {
        stream.read((char *)buffer.data(), sizeof(T) * buffer.size());
    }

    T value(int index) const {
        return buffer[index];
    }

    int size() const {
        return sizeof(T) * buffer.size();
    }

private:
    std::vector<T, Allocator> buffer;
};

template<std::size_t I = 0, typename... Tp>
typename std::enable_if<I == sizeof...(Tp), int>::type
block_size(std::tuple<Tp...>&) { return 0; }

template<std::size_t I = 0, typename... Tp>
typename std::enable_if<I < sizeof...(Tp), int>::type
block_size(std::tuple<Tp...>& t) {
    return std::get<I>(t).size() + block_size<I + 1, Tp...>(t);
}

}

#endif // TYPE_READER_H<commit_msg>properly calculate size (block_size) for simple types<commit_after>#ifndef TYPE_READER_H
#define TYPE_READER_H

#include <istream>
#include <tuple>

namespace cpputils
{

template<typename T, int S>
class array_reader {
public:
    void read_from_stream(std::istream &stream) {
        stream.read((char *)&buffer[0], sizeof(T) * S);
    }

    int size() const {
        return S * sizeof(T);
    }

    T value(int index) const {
        return buffer[index];
    }

private:
    T buffer[S];
};

template<typename T>
class value_reader : public array_reader<T, 1> {
public:
    T value() const {
        return array_reader<T, 1>::value(0);
    }
};

template<typename T, typename Allocator = std::allocator<T>>
class vector_reader {
public:
    vector_reader(int size) : buffer(size) {}

    void read_from_stream(std::istream &stream) {
        stream.read((char *)buffer.data(), sizeof(T) * buffer.size());
    }

    T value(int index) const {
        return buffer[index];
    }

    int size() const {
        return sizeof(T) * buffer.size();
    }

private:
    std::vector<T, Allocator> buffer;
};

/* sizeof POD type */
template<std::size_t I = 0, typename... Tp>
typename std::enable_if<std::is_pod<typename std::tuple_element<I, std::tuple<Tp...>>::type>::value, int>::type
calculate_size(std::tuple<Tp...>&) {
    return sizeof(typename std::tuple_element<I, std::tuple<Tp...>>::type);
}

/* sizeof non-pod type, require to implement size method */
template<std::size_t I = 0, typename... Tp>
typename std::enable_if<!std::is_pod<typename std::tuple_element<I, std::tuple<Tp...>>::type>::value, int>::type
calculate_size(std::tuple<Tp...>& t) {
    return std::get<I>(t).size();
}

template<std::size_t I = 0, typename... Tp>
typename std::enable_if<I == sizeof...(Tp), int>::type
block_size(std::tuple<Tp...>&) { return 0; }

template<std::size_t I = 0, typename... Tp>
typename std::enable_if<I < sizeof...(Tp), int>::type
block_size(std::tuple<Tp...>& t) {
    return calculate_size<I>(t) + block_size<I + 1, Tp...>(t);
}

}

#endif // TYPE_READER_H
{'fraction_non_alphanumeric': 0.17578013169195533, 'fraction_numerical': 0.005153163469796736, 'mean_word_length': 3.6524633821571237, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7281052', 'n_tokens_mistral': 1266, 'n_tokens_neox': 1220, 'n_words': 277}