<commit_before>from numpy cimport ndarray
from ..vocab cimport Vocab
from ..structs cimport TokenC
from ..attrs cimport attr_id_t
from .doc cimport Doc


cdef class Token:
    cdef readonly Vocab vocab
    cdef TokenC* c
    cdef readonly int i
    cdef readonly Doc doc
    cdef public ndarray tensor

    @staticmethod
    cdef inline Token cinit(Vocab vocab, const TokenC* token, int offset, Doc doc):
        if offset < 0 or offset >= doc.length:
            msg = "Attempt to access token at %d, max length %d"
            raise IndexError(msg % (offset, doc.length))
        if doc._py_tokens[offset] != None:
            return doc._py_tokens[offset]
        cdef Token self = Token.__new__(Token, vocab, doc, offset)
        doc._py_tokens[offset] = self
        return self

    cpdef bint check_flag(self, attr_id_t flag_id) except -1<commit_msg>Remove tensor property from token.<commit_after>from numpy cimport ndarray
from ..vocab cimport Vocab
from ..structs cimport TokenC
from ..attrs cimport attr_id_t
from .doc cimport Doc


cdef class Token:
    cdef readonly Vocab vocab
    cdef TokenC* c
    cdef readonly int i
    cdef readonly Doc doc

    @staticmethod
    cdef inline Token cinit(Vocab vocab, const TokenC* token, int offset, Doc doc):
        if offset < 0 or offset >= doc.length:
            msg = "Attempt to access token at %d, max length %d"
            raise IndexError(msg % (offset, doc.length))
        if doc._py_tokens[offset] != None:
            return doc._py_tokens[offset]
        cdef Token self = Token.__new__(Token, vocab, doc, offset)
        doc._py_tokens[offset] = self
        return self

    cpdef bint check_flag(self, attr_id_t flag_id) except -1
{'fraction_non_alphanumeric': 0.0733568075117371, 'fraction_numerical': 0.002347417840375587, 'mean_word_length': 2.722707423580786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5683613', 'n_tokens_mistral': 531, 'n_tokens_neox': 520, 'n_words': 195}