<commit_before>from libc.stdio cimport printf
from libc.stdlib cimport malloc
from libc.string cimport memcpy
from libc.stdint cimport (
    int8_t,
    int16_t,
    int32_t,
    int64_t,
)

ctypedef fused num:
    int8_t
    int16_t
    int32_t
    int64_t
    double


cdef public enum TType:
    STOP = 0
    VOID = 1
    BOOL = 2
    BYTE = 3
    I08 = 3
    DOUBLE = 4
    I16 = 6
    I32 = 8
    I64 = 10
    STRING = 11
    UTF7 = 11
    STRUCT = 12
    MAP = 13
    SET = 14
    LIST = 15
    UTF8 = 16
    UTF16 = 17


cdef:
    int VERSION_MASK = 0xffff0000
    int VERSION_1 = 0x80010000
    int TYPE_MASK = 0x000000ff

cdef:
    int8_t int8_sz = sizeof(int8_t)
    int8_t int16_sz = sizeof(int16_t)
    int8_t int32_sz = sizeof(int32_t)
    int8_t int64_sz = sizeof(int64_t)
    int8_t double_sz = sizeof(double)

# memprint is a helper to print memory buf in hex code
cdef void memprint(char* buf, int sz):
    cdef int i
    for i in range(sz):
        printf("%x ", <unsigned char> buf[i])
    printf("\n")


##########
# fast pack

cdef void _revert_pack(char* buf, num* x, int8_t sz):
    cdef:
        int i
        char* s = <char*> x
    for i in range(sz):
        buf[i] = s[sz-1-i]

cdef void _write_num(char* buf, num val):
    _revert_pack(buf, &val, sizeof(val))

cdef void _write_string(char* buf, bytes val):
    cdef:
        int32_t val_len = len(val)
    _revert_pack(buf, &val_len, int32_sz)
    memcpy(buf + int32_sz, <char*>val, val_len)

cdef bytes _pack_num(num val):
    cdef:
        int sz = sizeof(val)
        char* buf = <char*>malloc(sz)
    _revert_pack(buf, &val, sz)
    return buf[:sz]

cpdef bytes pack_i8(int8_t val):
    return _pack_num(val)

cpdef bytes pack_i16(int16_t val):
    return _pack_num(val)

cpdef bytes pack_i32(int32_t val):
    return _pack_num(val)

cpdef bytes pack_i64(int64_t val):
    return _pack_num(val)

cpdef bytes pack_bool(int8_t bool_):
    if bool_:
        return pack_i8(1)
    else:
        return pack_i8(0)

cpdef bytes pack_double(double val):
    return _pack_num(val)

cpdef bytes pack_string(bytes val):
    cdef:
        int val_len = len(val)
        int sz = sizeof(val_len)
    cdef char* buf = <char*>malloc(sz + val_len)
    _write_string(buf, val)
    return buf[:sz + val_len]


##########
# fast unpack


cdef void _revert_unpack(num* x, char* buf, int8_t sz):
    cdef:
        int i
        char tmp

    for i in range(sz / 2):
        tmp = buf[i]
        buf[i] = buf[sz-1-i]
        buf[sz-1-i] = tmp

    memcpy(x, buf, sz)


cpdef int8_t unpack_i8(bytes buf):
    cdef:
        char* read = buf
        int8_t x
    memcpy(&x, read, int8_sz)
    return x

cpdef int16_t unpack_i16(bytes buf):
    cdef:
        char* read = buf
        int16_t x
    _revert_unpack(&x, read, int16_sz)
    return x

cpdef int32_t unpack_i32(bytes buf):
    cdef:
        char* read = buf
        int32_t x<commit_msg>size_t for size<commit_after>from libc.stdio cimport printf
from libc.stdlib cimport malloc
from libc.string cimport memcpy
from libc.stdint cimport (
    int8_t,
    int16_t,
    int32_t,
    int64_t,
)

ctypedef fused num:
    int8_t
    int16_t
    int32_t
    int64_t
    double


cdef public enum TType:
    STOP = 0
    VOID = 1
    BOOL = 2
    BYTE = 3
    I08 = 3
    DOUBLE = 4
    I16 = 6
    I32 = 8
    I64 = 10
    STRING = 11
    UTF7 = 11
    STRUCT = 12
    MAP = 13
    SET = 14
    LIST = 15
    UTF8 = 16
    UTF16 = 17


cdef:
    int VERSION_MASK = 0xffff0000
    int VERSION_1 = 0x80010000
    int TYPE_MASK = 0x000000ff

cdef:
    size_t int8_sz = sizeof(int8_t)
    size_t int16_sz = sizeof(int16_t)
    size_t int32_sz = sizeof(int32_t)
    size_t int64_sz = sizeof(int64_t)
    size_t double_sz = sizeof(double)

# memprint is a helper to print memory buf in hex code
cdef void memprint(char* buf, size_t sz):
    cdef int i
    for i in range(sz):
        printf("%x ", <unsigned char> buf[i])
    printf("\n")


##########
# fast pack

cdef void _revert_pack(char* buf, num* x, size_t sz):
    cdef:
        int i
        char* s = <char*> x
    for i in range(sz):
        buf[i] = s[sz-1-i]

cdef void _write_num(char* buf, num val):
    _revert_pack(buf, &val, sizeof(val))

cdef void _write_string(char* buf, bytes val):
    cdef:
        int32_t val_len = len(val)
    _revert_pack(buf, &val_len, int32_sz)
    memcpy(buf + int32_sz, <char*>val, val_len)

cdef bytes _pack_num(num val):
    cdef:
        size_t sz = sizeof(val)
        char* buf = <char*>malloc(sz)
    _revert_pack(buf, &val, sz)
    return buf[:sz]

cpdef bytes pack_i8(int8_t val):
    return _pack_num(val)

cpdef bytes pack_i16(int16_t val):
    return _pack_num(val)

cpdef bytes pack_i32(int32_t val):
    return _pack_num(val)

cpdef bytes pack_i64(int64_t val):
    return _pack_num(val)

cpdef bytes pack_bool(int8_t bool_):
    if bool_:
        return pack_i8(1)
    else:
        return pack_i8(0)

cpdef bytes pack_double(double val):
    return _pack_num(val)

cpdef bytes pack_string(bytes val):
    cdef:
        int val_len = len(val)
        size_t sz = sizeof(val_len)
    cdef char* buf = <char*>malloc(sz + val_len)
    _write_string(buf, val)
    return buf[:sz + val_len]


##########
# fast unpack


cdef void _revert_unpack(num* x, char* buf, size_t sz):
    cdef:
        int i
        char tmp

    for i in range(sz / 2):
        tmp = buf[i]
        buf[i] = buf[sz-1-i]
        buf[sz-1-i] = tmp

    memcpy(x, buf, sz)


cpdef int8_t unpack_i8(bytes buf):
    cdef:
        char* read = buf
        int8_t x
    memcpy(&x, read, int8_sz)
    return x

cpdef int16_t unpack_i16(bytes buf):
    cdef:
        char* read = buf
        int16_t x
    _revert_unpack(&x, read, int16_sz)
    return x

cpdef int32_t unpack_i32(bytes buf):
    cdef:
        char* read = buf
        int32_t x
{'fraction_non_alphanumeric': 0.10128755364806867, 'fraction_numerical': 0.047553648068669525, 'mean_word_length': 2.413005272407733, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5873499', 'n_tokens_mistral': 2522, 'n_tokens_neox': 2309, 'n_words': 633}