<commit_before>from libc.stdint cimport int64_t

from cymem.cymem cimport Pool
from preshed.maps cimport PreshMap
from murmurhash.mrmr cimport hash64

from .typedefs cimport attr_t, hash_t


cpdef hash_t hash_string(unicode string) except 0


ctypedef union Utf8Str:
    unsigned char[8] s
    unsigned char* p


cdef class StringStore:
    cdef Pool mem
    cdef Utf8Str* c
    cdef int64_t size

    cdef public PreshMap _map
    cdef int64_t _resize_at

    cdef const Utf8Str* intern(self, unicode py_string) except NULL
    cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length) except NULL<commit_msg>Update strings.pxd<commit_after>from libc.stdint cimport int64_t

from cymem.cymem cimport Pool
from preshed.maps cimport PreshMap
from murmurhash.mrmr cimport hash64

from .typedefs cimport attr_t, hash_t


cpdef hash_t hash_string(unicode string) except 0


ctypedef union Utf8Str:
    unsigned char[8] s
    unsigned char* p


cdef class StringStore:
    cdef Pool mem
    cdef Utf8Str* c
    cdef int64_t size
    cdef bint is_frozen

    cdef public PreshMap _map
    cdef public PreshMap _oov
    cdef int64_t _resize_at

    cdef const Utf8Str* intern_unicode(self, unicode py_string)
    cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length)
{'fraction_non_alphanumeric': 0.042735042735042736, 'fraction_numerical': 0.024864024864024864, 'mean_word_length': 3.953846153846154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '303704', 'n_tokens_mistral': 494, 'n_tokens_neox': 457, 'n_words': 149}