<commit_before>#ifndef HCKT_TREE_H
#define HCKT_TREE_H

#include <cassert>
#include <vector>
#include <bitset>

template <typename ValueType>
class hckt_tree
{
protected:
    std::bitset<64> bitset;
    std::vector<ValueType> values;
    std::vector<hckt_tree<ValueType>*> children;

    size_t get_children_position(const size_t position) const
    {
        size_t c { 0 };

        for(size_t i=0; i<position; ++i) {
            if(bitset[i]) {
                ++c;
            }
        }
        
        return c;
    }


public:
    hckt_tree() : bitset { 0 }, values { }, children { }
    {
    }

    ~hckt_tree()
    {
        collapse();
    }
    
    size_t calculate_memory_size() const
    {
        size_t size = sizeof(bitset)
                    + sizeof(values) + (values.size() * sizeof(ValueType))
                    + sizeof(children) + (children.size() * sizeof(hckt_tree<ValueType>*));

        for(auto child : children) {
            size += child->calculate_memory_size();
        }
        
        return size;
    }


    /*
     * convert three part positions to memory location
     * as input: (left = 0 | right = 1) + (top = 0 | bottom = 2)
     *
     * result will be:
     * y1:x1:y2:x2:y3:x3
     *
     */
    static size_t get_position(size_t d1, size_t d2, size_t d3)
    {
        return ((1 << 0) * ((d1 & 2) >> 1))
             + ((1 << 1) *  (d1 & 1))
             + ((1 << 2) * ((d2 & 2) >> 1))
             + ((1 << 3) *  (d2 & 1))
             + ((1 << 4) * ((d3 & 2) >> 1))
             + ((1 << 5) *  (d3 & 1));
    }

    /*
     * check if we have any children
     */
    bool empty() const
    {
        return bitset.none();
    }
<commit_msg>add support for 3d<commit_after>#ifndef HCKT_TREE_H
#define HCKT_TREE_H

#include <cassert>
#include <vector>
#include <bitset>

template <typename ValueType>
class hckt_tree
{
protected:
    std::bitset<64> bitset;
    std::vector<ValueType> values;
    std::vector<hckt_tree<ValueType>*> children;

    size_t get_children_position(const size_t position) const
    {
        size_t c { 0 };

        for(size_t i=0; i<position; ++i) {
            if(bitset[i]) {
                ++c;
            }
        }
        
        return c;
    }


public:
    hckt_tree() : bitset { 0 }, values { }, children { }
    {
    }

    ~hckt_tree()
    {
        collapse();
    }
    
    size_t calculate_memory_size() const
    {
        size_t size { sizeof(bitset)
                    + sizeof(values) + (values.size() * sizeof(ValueType))
                    + sizeof(children) + (children.size() * sizeof(hckt_tree<ValueType>*))
        };

        for(auto child : children) {
            size += child->calculate_memory_size();
        }
        
        return size;
    }


    /*
     * 2d get position
     * convert three part positions to memory location
     * as input: (left = 0 | right = 1) + (top = 0 | bottom = 2)
     *
     * result will be:
     * y1:x1:y2:x2:y3:x3
     *
     */
    static size_t get_position(const size_t d1, const size_t d2, const size_t d3)
    {
        assert(d1 >= 0 && d1 < 4);
        assert(d2 >= 0 && d2 < 4);
        assert(d3 >= 0 && d3 < 4);

        return ((1 << 0) * ((d1 & 2) >> 1))
             + ((1 << 1) *  (d1 & 1))
             + ((1 << 2) * ((d2 & 2) >> 1))
             + ((1 << 3) *  (d2 & 1))
             + ((1 << 4) * ((d3 & 2) >> 1))
             + ((1 << 5) *  (d3 & 1));
    }

    /*
     * 3d get position
     * convert two part positions to memory location
     * as input: (left = 0 | right = 1) + (top = 0 | bottom = 2) + (in = 0 | out = 4)
     *
     * result will be:
     * z1:y1:x1:z2:y2:x2
     *
     */
    static size_t get_position(const size_t d1, const size_t d2)
    {
        assert(d1 >= 0 && d1 < 8);
        assert(d2 >= 0 && d2 < 8);

        return ((1 << 0) * ((d1 & 4) >> 2))
             + ((1 << 1) * ((d1 & 2) >> 1))
             + ((1 << 2) *  (d1 & 1))
             + ((1 << 3) * ((d2 & 4) >> 2))
             + ((1 << 4) * ((d2 & 2) >> 1))
             + ((1 << 5) *  (d2 & 1));
    }

    /*
     * check if we have any children
     */
    bool empty() const
    {
        return bitset.none();
    }

{'fraction_non_alphanumeric': 0.16186622232801715, 'fraction_numerical': 0.03689597714829802, 'mean_word_length': 1.5747549019607843, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3772649', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1506, 'n_words': 360}