<commit_before>#include "patricia.hpp"

extern "C" {
#include "indexed_ewah.h"
}

#define LARGE_LIST_THRESOLD 100000

typedef uint32_t Ipv4Ip;
typedef uint64_t Ipv4ListId;

typedef PatriciaKey<Ipv4Ip> Ipv4Mask;
typedef Patricia<Ipv4Ip, Ipv4ListId> Ipv4Tree;
typedef PatriciaElem<Ipv4Ip, Ipv4ListId> Ipv4TreeElem;

struct Ipv4List {
    Ipv4List(uint64_t i, unsigned l, const unsigned char *d) {
        id = i;
        length = l;
        data = d;
    }

    uint64_t id;
    unsigned length;
    const unsigned char *data;
};

class Ipv4Map {
public:
    Ipv4ListId id;

    Ipv4Map(Ipv4ListId i) {
        id = i;
        bitmap.map = ewah_new();
    }

    void add_ip(Ipv4Ip ip) {
        if (!finalized) {
            ewah_set(bitmap.map, ip);
        }
    }

    void finalize(void) {
        finalized = true;
        ewah_build_index(&bitmap);
    }

    bool lookup(Ipv4Ip ip) {
        return indexed_ewah_get(&bitmap, ip);
    }

private:
    indexed_ewah_map bitmap;
    bool finalized;
};

class Ipv4Index {
public:
    Ipv4Index(std::vector<Ipv4List>& lists) {<commit_msg>Fix finalized not being initialized<commit_after>#include "patricia.hpp"

extern "C" {
#include "indexed_ewah.h"
}

#define LARGE_LIST_THRESOLD 100000

typedef uint32_t Ipv4Ip;
typedef uint64_t Ipv4ListId;

typedef PatriciaKey<Ipv4Ip> Ipv4Mask;
typedef Patricia<Ipv4Ip, Ipv4ListId> Ipv4Tree;
typedef PatriciaElem<Ipv4Ip, Ipv4ListId> Ipv4TreeElem;

struct Ipv4List {
    Ipv4List(uint64_t i, unsigned l, const unsigned char *d) {
        id = i;
        length = l;
        data = d;
    }

    uint64_t id;
    unsigned length;
    const unsigned char *data;
};

class Ipv4Map {
public:
    Ipv4ListId id;

    Ipv4Map(Ipv4ListId i) {
        id = i;
        finalized = false;
        bitmap.map = ewah_new();
    }

    void add_ip(Ipv4Ip ip) {
        if (!finalized) {
            ewah_set(bitmap.map, ip);
        }
    }

    void finalize(void) {
        finalized = true;
        ewah_build_index(&bitmap);
    }

    bool lookup(Ipv4Ip ip) {
        return indexed_ewah_get(&bitmap, ip);
    }

private:
    indexed_ewah_map bitmap;
    bool finalized;
};

class Ipv4Index {
public:
    Ipv4Index(std::vector<Ipv4List>& lists) {
{'fraction_non_alphanumeric': 0.10067415730337079, 'fraction_numerical': 0.03146067415730337, 'mean_word_length': 2.722408026755853, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4467367', 'n_tokens_mistral': 919, 'n_tokens_neox': 854, 'n_words': 184}