<commit_before>from libc.stdio cimport printf, puts
from libc.stdlib cimport malloc
from libc.string cimport memcpy
from libc.stdint cimport (
    int8_t,
    int16_t,
    int32_t,
    int64_t,
)

ctypedef fused num:
    int8_t
    int16_t
    int32_t
    int64_t
    double


cdef public enum TType:
    STOP = 0
    VOID = 1
    BOOL = 2
    BYTE = 3
    I08 = 3
    DOUBLE = 4
    I16 = 6
    I32 = 8
    I64 = 10
    STRING = 11
    UTF7 = 11
    STRUCT = 12
    MAP = 13
    SET = 14
    LIST = 15
    UTF8 = 16
    UTF16 = 17


cdef:<commit_msg>add BINARY TType to cybinary protocol<commit_after>from libc.stdio cimport printf, puts
from libc.stdlib cimport malloc
from libc.string cimport memcpy
from libc.stdint cimport (
    int8_t,
    int16_t,
    int32_t,
    int64_t,
)

ctypedef fused num:
    int8_t
    int16_t
    int32_t
    int64_t
    double


cdef public enum TType:
    STOP = 0
    VOID = 1
    BOOL = 2
    BYTE = 3
    I08 = 3
    DOUBLE = 4
    I16 = 6
    I32 = 8
    I64 = 10
    STRING = 11
    UTF7 = 11
    BINARY = 11  # This here just for parsing. For all purposes, it's a string
    STRUCT = 12
    MAP = 13
    SET = 14
    LIST = 15
    UTF8 = 16
    UTF16 = 17


cdef:
{'fraction_non_alphanumeric': 0.06058091286307054, 'fraction_numerical': 0.08796680497925312, 'mean_word_length': 1.851063829787234, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6177220', 'n_tokens_mistral': 587, 'n_tokens_neox': 465, 'n_words': 151}