<commit_before>import codecs

from libc.string cimport memcpy
from murmurhash.mrmr cimport hash64


from .typedefs cimport hash_t


SEPARATOR = '\n|-SEP-|\n'


cpdef hash_t hash_string(unicode string) except 0:
    chars = <Py_UNICODE*>string
    return hash64(chars, len(string) * sizeof(Py_UNICODE), 0)


"""
cdef class SymbolMap:
    def __init__(self):
        self._string_to_id = {'': 0}
        self._id_to_string = ['']

    def __iter__(self):
        for id_, string in enumerate(self._id_to_string[1:]):
            yield string, id_

    def __len__(self):
        return len(self._id_to_string)

    def __getitem__(self, object string_or_id):
        cdef bytes byte_string
        if isinstance(string_or_id, int) or isinstance(string_or_id, long):
            if string_or_id < 1 or string_or_id >= self.size:
                raise IndexError(string_or_id)
            return self._int_to_string[string_or_id]
        else:
            string = string_or_id
            if isinstance(string, unicode):
                string = string.encode('utf8')
            if string in self._string_to_id:
                id_ = self._string_to_id[string]
            else:
                id_ = len(self._string_to_id)
                self._string_to_id[string] = id_
                self._id_to_string.append(string)
            return id_
"""


cdef class StringStore:
    '''Map strings to and from integer IDs.'''
    def __init__(self):
        self.mem = Pool()
        self._map = PreshMap()
        self._resize_at = 10000
        self.strings = <Utf8Str*>self.mem.alloc(self._resize_at, sizeof(Utf8Str))
        self.size = 1

    property size:
        def __get__(self):
            return self.size-1

    def __getitem__(self, object string_or_id):
        cdef bytes byte_string
        cdef const Utf8Str* utf8str
        if isinstance(string_or_id, int) or isinstance(string_or_id, long):
            if string_or_id == 0:
                return u''
            elif string_or_id < 1 or string_or_id >= self.size:<commit_msg>* Add __len__ function to StringStore<commit_after>import codecs

from libc.string cimport memcpy
from murmurhash.mrmr cimport hash64


from .typedefs cimport hash_t


SEPARATOR = '\n|-SEP-|\n'


cpdef hash_t hash_string(unicode string) except 0:
    chars = <Py_UNICODE*>string
    return hash64(chars, len(string) * sizeof(Py_UNICODE), 0)


"""
cdef class SymbolMap:
    def __init__(self):
        self._string_to_id = {'': 0}
        self._id_to_string = ['']

    def __iter__(self):
        for id_, string in enumerate(self._id_to_string[1:]):
            yield string, id_

    def __len__(self):
        return len(self._id_to_string)

    def __getitem__(self, object string_or_id):
        cdef bytes byte_string
        if isinstance(string_or_id, int) or isinstance(string_or_id, long):
            if string_or_id < 1 or string_or_id >= self.size:
                raise IndexError(string_or_id)
            return self._int_to_string[string_or_id]
        else:
            string = string_or_id
            if isinstance(string, unicode):
                string = string.encode('utf8')
            if string in self._string_to_id:
                id_ = self._string_to_id[string]
            else:
                id_ = len(self._string_to_id)
                self._string_to_id[string] = id_
                self._id_to_string.append(string)
            return id_
"""


cdef class StringStore:
    '''Map strings to and from integer IDs.'''
    def __init__(self):
        self.mem = Pool()
        self._map = PreshMap()
        self._resize_at = 10000
        self.strings = <Utf8Str*>self.mem.alloc(self._resize_at, sizeof(Utf8Str))
        self.size = 1

    property size:
        def __get__(self):
            return self.size-1

    def __len__(self):
        return self.size

    def __getitem__(self, object string_or_id):
        cdef bytes byte_string
        cdef const Utf8Str* utf8str
        if isinstance(string_or_id, int) or isinstance(string_or_id, long):
            if string_or_id == 0:
                return u''
            elif string_or_id < 1 or string_or_id >= self.size:
{'fraction_non_alphanumeric': 0.09152053807350469, 'fraction_numerical': 0.011049723756906077, 'mean_word_length': 2.212962962962963, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7415635', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1298, 'n_words': 314}