<commit_before>from .typedefs cimport hash_t, flags_t, id_t, len_t, tag_t, attr_t, attr_id_t
from .typedefs cimport ID, ORTH, LOWER, NORM, SHAPE, PREFIX, SUFFIX, LENGTH, CLUSTER
from .structs cimport LexemeC
from .strings cimport StringStore

from numpy cimport ndarray



cdef LexemeC EMPTY_LEXEME


cdef int set_lex_struct_props(LexemeC* lex, dict props, StringStore strings,
                              const float* empty_vec) except -1
 
cdef class Lexeme:
    cdef readonly ndarray repvec

    cdef readonly flags_t flags
    cdef readonly attr_t id
    cdef readonly attr_t length

    cdef readonly attr_t orth
    cdef readonly attr_t lower
    cdef readonly attr_t norm
    cdef readonly attr_t shape
    cdef readonly attr_t prefix
    cdef readonly attr_t suffix

    cdef readonly unicode orth_
    cdef readonly unicode lower_
    cdef readonly unicode norm_
    cdef readonly unicode shape_
    cdef readonly unicode prefix_
    cdef readonly unicode suffix_

    cdef readonly attr_t cluster
    cdef readonly float prob
    cdef readonly float sentiment

    # Workaround for an apparent bug in the way the decorator is handled ---
    # TODO: post bug report / patch to Cython.
    @staticmethod
    cdef inline Lexeme from_ptr(const LexemeC* ptr, StringStore strings):
        cdef Lexeme py = Lexeme.__new__(Lexeme, 300)
        for i in range(300):
            py.repvec[i] = ptr.repvec[i]
        py.flags = ptr.flags
        py.id = ptr.id
        py.length = ptr.length

        py.orth = ptr.orth
        py.lower = ptr.lower
        py.norm = ptr.norm
        py.shape = ptr.shape
        py.prefix = ptr.prefix
        py.suffix = ptr.suffix

        py.orth_ = strings[ptr.orth]
        py.lower_ = strings[ptr.lower]
        py.norm_ = strings[ptr.norm]
        py.shape_ = strings[ptr.shape]
        py.prefix_ = strings[ptr.prefix]
        py.suffix_ = strings[ptr.suffix]

        py.cluster = ptr.cluster
        py.prob = ptr.prob
        py.sentiment = ptr.sentiment
        return py


cdef inline bint check_flag(const LexemeC* lexeme, attr_id_t flag_id) nogil:
    return lexeme.flags & (1 << flag_id)<commit_msg>* Store the l2 norm of the word's vector<commit_after>from .typedefs cimport hash_t, flags_t, id_t, len_t, tag_t, attr_t, attr_id_t
from .typedefs cimport ID, ORTH, LOWER, NORM, SHAPE, PREFIX, SUFFIX, LENGTH, CLUSTER
from .structs cimport LexemeC
from .strings cimport StringStore

from numpy cimport ndarray



cdef LexemeC EMPTY_LEXEME


cdef int set_lex_struct_props(LexemeC* lex, dict props, StringStore strings,
                              const float* empty_vec) except -1
 
cdef class Lexeme:
    cdef readonly ndarray repvec

    cdef readonly flags_t flags
    cdef readonly attr_t id
    cdef readonly attr_t length

    cdef readonly attr_t orth
    cdef readonly attr_t lower
    cdef readonly attr_t norm
    cdef readonly attr_t shape
    cdef readonly attr_t prefix
    cdef readonly attr_t suffix

    cdef readonly unicode orth_
    cdef readonly unicode lower_
    cdef readonly unicode norm_
    cdef readonly unicode shape_
    cdef readonly unicode prefix_
    cdef readonly unicode suffix_

    cdef readonly attr_t cluster
    cdef readonly float prob
    cdef readonly float sentiment
    cdef readonly float l2_norm

    # Workaround for an apparent bug in the way the decorator is handled ---
    # TODO: post bug report / patch to Cython.
    @staticmethod
    cdef inline Lexeme from_ptr(const LexemeC* ptr, StringStore strings):
        cdef Lexeme py = Lexeme.__new__(Lexeme, 300)
        for i in range(300):
            py.repvec[i] = ptr.repvec[i]
        py.l2_norm = ptr.l2_norm
        py.flags = ptr.flags
        py.id = ptr.id
        py.length = ptr.length

        py.orth = ptr.orth
        py.lower = ptr.lower
        py.norm = ptr.norm
        py.shape = ptr.shape
        py.prefix = ptr.prefix
        py.suffix = ptr.suffix

        py.orth_ = strings[ptr.orth]
        py.lower_ = strings[ptr.lower]
        py.norm_ = strings[ptr.norm]
        py.shape_ = strings[ptr.shape]
        py.prefix_ = strings[ptr.prefix]
        py.suffix_ = strings[ptr.suffix]

        py.cluster = ptr.cluster
        py.prob = ptr.prob
        py.sentiment = ptr.sentiment
        return py

    cpdef bint check(self, attr_id_t flag_id) except -1


cdef inline bint check_flag(const LexemeC* lexeme, attr_id_t flag_id) nogil:
    return lexeme.flags & (1 << flag_id)
{'fraction_non_alphanumeric': 0.06310352571300247, 'fraction_numerical': 0.004715921850437907, 'mean_word_length': 2.4824081313526194, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1628701', 'n_tokens_mistral': 1472, 'n_tokens_neox': 1390, 'n_words': 458}