<commit_before>#ifndef COMPONENTS_HPP
#define COMPONENTS_HPP

#include <unordered_map>
#include <vector>

/** For logging **/
#include "logger.hpp"
#include <ostream>
#include <typeinfo>

typedef unsigned int ID;

template<class T, class Container = std::unordered_map<ID, T>>
class Components {
private:
    Container container;

public:
    void insert(const std::pair<const ID, const T&> pair) {
        std::ostringstream oss;
        oss << "Adding " << typeid(T).name() << " to id " << pair.first;
        Logger::log(oss, Log::INFO);
        container.insert(pair);
    }

    void sync(Components<T>& components) {
        for(auto it : components) {
            container[it.first] = components[it.first];
        }
    }

    Components<T, Container>& operator=(Components<T, Container> components) {
        container.swap(components.container);
        return *this;
    }

    T& operator[](const ID id) {
        if(container.find(id) == container.end()) {
            std::ostringstream oss;
            oss << "Creating " << typeid(T).name() << " on " << id;
            Logger::log(oss, Log::INFO);
        }
        return container[id];
    }

    const T& operator[](const ID id) const {
        if(container.find(id) == container.end()) {
            std::ostringstream oss;
            oss << "Creating " << typeid(T).name() << " on " << id;
            Logger::log(oss, Log::INFO);
        }
        return container[id];
    }

    T& at(const ID id) {
        //Log before the error before throwing it
        try {
            return container.at(id);
        } catch (std::out_of_range oor) {
            std::ostringstream oss;
            oss << "Tried accesing absent " << typeid(T).name() << " on id " << id;
            Logger::log(oss, Log::ERROR);
            throw oor;
        }
    }

    const T& at(const ID id) const {
        //Log before the error before throwing it
        try {
            return container.at(id);
        } catch (std::out_of_range oor) {
            std::ostringstream oss;
            oss << "Tried accesing absent " << typeid(T).name() << " on id " << id;
            Logger::log(oss, Log::ERROR);
            throw oor;
        }
    }

    constexpr auto find(const ID id) {
        return container.find(id);
    }

    constexpr bool contains(const ID id) {
        return container.find(id) != container.end();
    }

    constexpr auto begin() const {
        return container.begin();
    }

    constexpr auto end() const {
        return container.end();
    }

    auto erase(const ID id) {
        std::ostringstream oss;
        oss << "Erasing " << typeid(T).name() << " from id " << id;
        Logger::log(oss, Log::INFO);
        return container.erase(id);
    }

    constexpr void clear() {
        container.clear();
    }

    constexpr auto size() const {
        return container.size();
    }

    constexpr auto empty() const {
        return container.empty();
    }

    /** Split the component into several containers no more components than componentsPerContainer
        but there may be less components in each container. There can never be more components in
        a container than there are IDs, hence the type of parameter is ID **/
    std::vector<Components<T>> split(ID componentsPerContainer) {
        const int numContainers = (size() / componentsPerContainer) + ((size() % componentsPerContainer) > 0 ? 1 : 0);
        std::vector<Components<T>> containers(numContainers, Components<T>());
        int componentIndex = 0;
        for(auto& smallerContainer : containers) {
            for(int i = 0; i < componentsPerContainer && componentIndex < size(); i++) {
                smallerContainer.insert({componentIndex, (*this)[componentIndex]});
                componentIndex++;
            }
        }
        return containers;
    }

    template<class Archive>
    constexpr void serialize(Archive& ar) {
        ar(container);
    }
};

#endif //COMPONENTS_HPP
<commit_msg>fixed bug where splitting components container into smaller containers would mess up the keys, ie the ids<commit_after>#ifndef COMPONENTS_HPP
#define COMPONENTS_HPP

#include <unordered_map>
#include <vector>

/** For logging **/
#include "logger.hpp"
#include <ostream>
#include <typeinfo>

typedef unsigned int ID;

template<class T, class Container = std::unordered_map<ID, T>>
class Components {
private:
    Container container;

public:
    void insert(const std::pair<const ID, const T&> pair) {
        std::ostringstream oss;
        oss << "Adding " << typeid(T).name() << " to id " << pair.first;
        Logger::log(oss, Log::INFO);
        container.insert(pair);
    }

    void sync(Components<T>& components) {
        for(auto it : components) {
            container[it.first] = components[it.first];
        }
    }

    Components<T, Container>& operator=(Components<T, Container> components) {
        container.swap(components.container);
        return *this;
    }

    T& operator[](const ID id) {
        if(container.find(id) == container.end()) {
            std::ostringstream oss;
            oss << "Creating " << typeid(T).name() << " on " << id;
            Logger::log(oss, Log::INFO);
        }
        return container[id];
    }

    const T& operator[](const ID id) const {
        if(container.find(id) == container.end()) {
            std::ostringstream oss;
            oss << "Creating " << typeid(T).name() << " on " << id;
            Logger::log(oss, Log::INFO);
        }
        return container[id];
    }

    T& at(const ID id) {
        //Log before the error before throwing it
        try {
            return container.at(id);
        } catch (std::out_of_range oor) {
            std::ostringstream oss;
            oss << "Tried accesing absent " << typeid(T).name() << " on id " << id;
            Logger::log(oss, Log::ERROR);
            throw oor;
        }
    }

    const T& at(const ID id) const {
        //Log before the error before throwing it
        try {
            return container.at(id);
        } catch (std::out_of_range oor) {
            std::ostringstream oss;
            oss << "Tried accesing absent " << typeid(T).name() << " on id " << id;
            Logger::log(oss, Log::ERROR);
            throw oor;
        }
    }

    constexpr auto find(const ID id) {
        return container.find(id);
    }

    constexpr bool contains(const ID id) {
        return container.find(id) != container.end();
    }

    constexpr auto begin() const {
        return container.begin();
    }

    constexpr auto end() const {
        return container.end();
    }

    auto erase(const ID id) {
        std::ostringstream oss;
        oss << "Erasing " << typeid(T).name() << " from id " << id;
        Logger::log(oss, Log::INFO);
        return container.erase(id);
    }

    constexpr void clear() {
        container.clear();
    }

    constexpr auto size() const {
        return container.size();
    }

    constexpr auto empty() const {
        return container.empty();
    }

    /** Split the component into several containers no more components than componentsPerContainer
        but there may be less components in each container. There can never be more components in
        a container than there are IDs, hence the type of parameter is ID **/
    std::vector<Components<T>> split(ID componentsPerContainer) {
        const int numContainers = (size() / componentsPerContainer) + ((size() % componentsPerContainer) > 0 ? 1 : 0);
        std::vector<Components<T>> containers(numContainers, Components<T>());
        auto it(begin());
        for(auto& smallerContainer : containers) {
            for(int i = 0; i < componentsPerContainer && it != end(); i++) {
                smallerContainer.insert(*it++);
            }
        }
        return containers;
    }

    template<class Archive>
    constexpr void serialize(Archive& ar) {
        ar(container);
    }
};

#endif //COMPONENTS_HPP

{'fraction_non_alphanumeric': 0.13145363408521304, 'fraction_numerical': 0.0011278195488721805, 'mean_word_length': 2.235103364410215, 'pattern_counts': {'":': 0, '<': 134, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4771570', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2253, 'n_words': 631}