<commit_before>from .typedefs cimport hash_t, flags_t, id_t, len_t, tag_t, attr_t
from .typedefs cimport ID, ORTH, LOWER, NORM, SHAPE, PREFIX, SUFFIX, LENGTH, CLUSTER

from .attrs cimport attr_id_t

from .structs cimport LexemeC
from .strings cimport StringStore

from numpy cimport ndarray



cdef LexemeC EMPTY_LEXEME


cdef int set_lex_struct_props(LexemeC* lex, dict props, StringStore strings,
                              const float* empty_vec) except -1

cdef class Lexeme:
    cdef readonly ndarray repvec

    cdef readonly flags_t flags<commit_msg>* Import Lexeme attrs from spacy.attrs, not spacy.typedefs<commit_after>from .typedefs cimport attr_t, hash_t, flags_t, id_t, len_t, tag_t
from .attrs cimport attr_id_t
from .attrs cimport ID, ORTH, LOWER, NORM, SHAPE, PREFIX, SUFFIX, LENGTH, CLUSTER

from .structs cimport LexemeC
from .strings cimport StringStore

from numpy cimport ndarray



cdef LexemeC EMPTY_LEXEME


cdef int set_lex_struct_props(LexemeC* lex, dict props, StringStore strings,
                              const float* empty_vec) except -1

cdef class Lexeme:
    cdef readonly ndarray repvec

    cdef readonly flags_t flags
{'fraction_non_alphanumeric': 0.05512489233419466, 'fraction_numerical': 0.0017226528854435831, 'mean_word_length': 3.7818930041152266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '182798', 'n_tokens_mistral': 419, 'n_tokens_neox': 395, 'n_words': 134}