<commit_before>package msgpack

import (
    "io"
    "os"
    "unsafe"
    "reflect"
)

func readByte(reader io.Reader) (v uint8, err os.Error) {
    data := [1]byte{}
    _, e := reader.Read(data[0:])
    if e != nil { return 0, e }
    return data[0], nil
}

func readUint16(reader io.Reader) (v uint16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint16(data[0]) >> 8) | uint16(data[1]), n, nil
}

func readUint32(reader io.Reader) (v uint32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint32(data[0]) << 24) | (uint32(data[1]) << 16) | (uint32(data[2]) << 8) | uint32(data[1]), n, nil
}

func readUint64(reader io.Reader) (v uint64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint64(data[0]) << 56) | (uint64(data[1]) << 48) | (uint64(data[2]) << 40) | (uint64(data[3]) << 32) | (uint64(data[4]) << 24) | (uint64(data[5]) << 16) | (uint64(data[6]) << 8) | uint64(data[7]), n, nil
}

func readInt16(reader io.Reader) (v int16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int16(data[0]) << 8) | int16(data[1]), n, nil
}

func readInt32(reader io.Reader) (v int32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int32(data[0]) << 24) | (int32(data[1]) << 16) | (int32(data[2]) << 8) | int32(data[1]), n, nil
}

func readInt64(reader io.Reader) (v int64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int64(data[0]) << 56) | (int64(data[1]) << 48) | (int64(data[2]) << 40) | (int64(data[3]) << 32) | (int64(data[4]) << 24) | (int64(data[5]) << 16) | (int64(data[6]) << 8) | int64(data[7]), n, nil
}<commit_msg>Fix bugs in unpacker.<commit_after>package msgpack

import (
    "io"
    "os"
    "unsafe"
    "reflect"
)

func readByte(reader io.Reader) (v uint8, err os.Error) {
    data := [1]byte{}
    _, e := reader.Read(data[0:])
    if e != nil { return 0, e }
    return data[0], nil
}

func readUint16(reader io.Reader) (v uint16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint16(data[0]) << 8) | uint16(data[1]), n, nil
}

func readUint32(reader io.Reader) (v uint32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint32(data[0]) << 24) | (uint32(data[1]) << 16) | (uint32(data[2]) << 8) | uint32(data[3]), n, nil
}

func readUint64(reader io.Reader) (v uint64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint64(data[0]) << 56) | (uint64(data[1]) << 48) | (uint64(data[2]) << 40) | (uint64(data[3]) << 32) | (uint64(data[4]) << 24) | (uint64(data[5]) << 16) | (uint64(data[6]) << 8) | uint64(data[7]), n, nil
}

func readInt16(reader io.Reader) (v int16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int16(data[0]) << 8) | int16(data[1]), n, nil
}

func readInt32(reader io.Reader) (v int32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int32(data[0]) << 24) | (int32(data[1]) << 16) | (int32(data[2]) << 8) | int32(data[3]), n, nil
}

func readInt64(reader io.Reader) (v int64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int64(data[0]) << 56) | (int64(data[1]) << 48) | (int64(data[2]) << 40) | (int64(data[3]) << 32) | (int64(data[4]) << 24) | (int64(data[5]) << 16) | (int64(data[6]) << 8) | int64(data[7]), n, nil
}
{'fraction_non_alphanumeric': 0.23228542914171657, 'fraction_numerical': 0.0843313373253493, 'mean_word_length': 3.2155625657202944, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3975619', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1687, 'n_words': 445}