<commit_before># cython: infer_types=True
# cython: cdivision=True
#
cimport cython


DEF EMPTY_KEY = 0
DEF DELETED_KEY = 1


cdef class PreshMap:
    """Hash map that assumes keys come pre-hashed. Maps uint64_t --> uint64_t.
    Uses open addressing with linear probing.

    Usage
        map = PreshMap() # Create a table
        map = PreshMap(initial_size=1024) # Create with initial size (efficiency)
        map[key] = value # Set a value to a key
        value = map[key] # Get a value given a key
        for key, value in map.items(): # Iterate over items
        len(map) # Get number of inserted keys
    """
    def __init__(self, size_t initial_size=8):
        # Size must be power of two
        if initial_size == 0:
            initial_size = 8
        if initial_size & (initial_size - 1) != 0:
            power = 1
            while power < initial_size:
                power *= 2
            initial_size = power
        self.mem = Pool()
        self.c_map = <MapStruct*>self.mem.alloc(1, sizeof(MapStruct))
        map_init(self.mem, self.c_map, initial_size)

    property length:
        def __get__(self):
            return self.c_map.length

    def items(self):
        cdef key_t key
        cdef void* value
        cdef int i = 0
        while map_iter(self.c_map, &i, &key, &value):
            yield key, <size_t>value

    def keys(self):
        for key, _ in self.items():
            yield key

    def values(self):
        for _, value in self.items():
            yield value

    def pop(self, key_t key, default=None):
        cdef void* value = map_get(self.c_map, key)
        map_clear(self.c_map, key)
        return <size_t>value if value != NULL else default

    def __getitem__(self, key_t key):
        cdef void* value = map_get(self.c_map, key)
        return <size_t>value if value != NULL else None

    def __setitem__(self, key_t key, size_t value):
        map_set(self.mem, self.c_map, key, <void*>value)

    def __delitem__(self, key_t key):
        map_clear(self.c_map, key)

    def __len__(self):
        return self.length

    def __contains__(self, key_t key):
        cdef void* value = map_get(self.c_map, key)
        return True if value != NULL else False

    def __iter__(self):
        for key in self.keys():
            yield key

    cdef inline void* get(self, key_t key) nogil:
        return map_get(self.c_map, key)<commit_msg>Fix length attribute on PreshMap<commit_after># cython: infer_types=True
# cython: cdivision=True
#
cimport cython


DEF EMPTY_KEY = 0
DEF DELETED_KEY = 1


cdef class PreshMap:
    """Hash map that assumes keys come pre-hashed. Maps uint64_t --> uint64_t.
    Uses open addressing with linear probing.

    Usage
        map = PreshMap() # Create a table
        map = PreshMap(initial_size=1024) # Create with initial size (efficiency)
        map[key] = value # Set a value to a key
        value = map[key] # Get a value given a key
        for key, value in map.items(): # Iterate over items
        len(map) # Get number of inserted keys
    """
    def __init__(self, size_t initial_size=8):
        # Size must be power of two
        if initial_size == 0:
            initial_size = 8
        if initial_size & (initial_size - 1) != 0:
            power = 1
            while power < initial_size:
                power *= 2
            initial_size = power
        self.mem = Pool()
        self.c_map = <MapStruct*>self.mem.alloc(1, sizeof(MapStruct))
        map_init(self.mem, self.c_map, initial_size)

    property capacity:
        def __get__(self):
            return self.c_map.length

    def items(self):
        cdef key_t key
        cdef void* value
        cdef int i = 0
        while map_iter(self.c_map, &i, &key, &value):
            yield key, <size_t>value

    def keys(self):
        for key, _ in self.items():
            yield key

    def values(self):
        for _, value in self.items():
            yield value

    def pop(self, key_t key, default=None):
        cdef void* value = map_get(self.c_map, key)
        map_clear(self.c_map, key)
        return <size_t>value if value != NULL else default

    def __getitem__(self, key_t key):
        cdef void* value = map_get(self.c_map, key)
        return <size_t>value if value != NULL else None

    def __setitem__(self, key_t key, size_t value):
        map_set(self.mem, self.c_map, key, <void*>value)

    def __delitem__(self, key_t key):
        map_clear(self.c_map, key)

    def __len__(self):
        return self.c_map.filled

    def __contains__(self, key_t key):
        cdef void* value = map_get(self.c_map, key)
        return True if value != NULL else False

    def __iter__(self):
        for key in self.keys():
            yield key

    cdef inline void* get(self, key_t key) nogil:
        return map_get(self.c_map, key)
{'fraction_non_alphanumeric': 0.09416855553266021, 'fraction_numerical': 0.00783020811868947, 'mean_word_length': 2.16015625, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '182141', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1523, 'n_words': 489}