<commit_before>package recordio

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"errors"
	"github.com/golang/snappy"
	"hash/crc32"
	"io"
)

var (
	ErrReadBytes  = errors.New("Read bytes error")
	ErrWriteBytes = errors.New("Write bytes error")
	ErrChecksum   = errors.New("Checksum Error")
)

const (
	GzipCompress   = 1 << iota
	SnappyCompress = 1 << iota
	BodyChecksum   = 1 << iota
)

const DefaultFlags = BodyChecksum | SnappyCompress

const (
	recordHeaderSize = 16
)

type Flags uint32

type RecordHeader struct {
	BodyLength   uint32
	Flags        Flags
	BodyChecksum uint32
}

func (header *RecordHeader) MarshalBinary() (data []byte, err error) {
	output := &bytes.Buffer{}
	output.Grow(recordHeaderSize)
	if err := binary.Write(output, binary.LittleEndian, header); err != nil {
		return nil, err
	}
	if err := binary.Write(output, binary.LittleEndian, crc32.ChecksumIEEE(output.Bytes())); err != nil {
		return nil, err
	}
	return output.Bytes(), nil
}

func (header *RecordHeader) UnmarshalBinary(data []byte) error {
	withChecksum := struct {
		RecordHeader
		Checksum uint32
	}{}
	r := bytes.NewReader(data)
	if err := binary.Read(r, binary.LittleEndian, &withChecksum); err != nil {
		return err
	}
	if withChecksum.Checksum != crc32.ChecksumIEEE(data[:12]) {
		return ErrChecksum
	}
	*header = withChecksum.RecordHeader
	return nil
}

type Reader struct {
	bytesReader      io.Reader
	Options          Flags
	BytesReaderError error
	LastError        error
}

func NewReader(reader io.Reader, options Flags) *Reader {
	return &Reader{
		bytesReader: reader,<commit_msg>panic when we should panic<commit_after>package recordio

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"errors"
	"github.com/golang/snappy"
	"hash/crc32"
	"io"
)

var (
	ErrReadBytes  = errors.New("Read bytes error")
	ErrWriteBytes = errors.New("Write bytes error")
	ErrChecksum   = errors.New("Checksum Error")
)

const (
	GzipCompress   = 1 << iota
	SnappyCompress = 1 << iota
	BodyChecksum   = 1 << iota
)

const DefaultFlags = BodyChecksum | SnappyCompress

const (
	recordHeaderSize = 16
)

type Flags uint32

type RecordHeader struct {
	BodyLength   uint32
	Flags        Flags
	BodyChecksum uint32
}

func (header *RecordHeader) MarshalBinary() (data []byte, err error) {
	output := &bytes.Buffer{}
	output.Grow(recordHeaderSize)
	if err := binary.Write(output, binary.LittleEndian, header); err != nil {
		panic(err)
	}
	if err := binary.Write(output, binary.LittleEndian, crc32.ChecksumIEEE(output.Bytes())); err != nil {
		panic(err)
	}
	return output.Bytes(), nil
}

func (header *RecordHeader) UnmarshalBinary(data []byte) error {
	withChecksum := struct {
		RecordHeader
		Checksum uint32
	}{}
	r := bytes.NewReader(data)
	if err := binary.Read(r, binary.LittleEndian, &withChecksum); err != nil {
		panic(err)
	}
	if withChecksum.Checksum != crc32.ChecksumIEEE(data[:12]) {
		return ErrChecksum
	}
	*header = withChecksum.RecordHeader
	return nil
}

type Reader struct {
	bytesReader      io.Reader
	Options          Flags
	BytesReaderError error
	LastError        error
}

func NewReader(reader io.Reader, options Flags) *Reader {
	return &Reader{
		bytesReader: reader,
{'fraction_non_alphanumeric': 0.11616954474097331, 'fraction_numerical': 0.013186813186813187, 'mean_word_length': 4.105769230769231, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '445080', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1120, 'n_words': 272}