<commit_before>package binny

import (
	"bufio"
	"encoding"
	"encoding/gob"
	"io"
	"math"
	"reflect"
	"unsafe"
)

const DefaultEncoderBufferSize = 4096

// Marshaler is the interface implemented by objects that can marshal themselves into a binary representation.
// Implementing this bypasses reflection and is generally faster but not nessecery optimized.
type Marshaler interface {
	MarshalBinny(enc *Encoder) error
}

type Encoder struct {
	w *bufio.Writer
}

// NewEncoder returns a new encoder with the DefaultEncoderBufferSize
func NewEncoder(w io.Writer) *Encoder {
	return NewEncoderSize(w, DefaultEncoderBufferSize)
}

// NewEncoder returns a new encoder with the specific buffer size, minimum is 24 bytes.
func NewEncoderSize(w io.Writer, sz int) *Encoder {
	if sz < 24 {
		sz = 24
	}
	return &Encoder{
		w: bufio.NewWriterSize(w, sz),
	}
}

// Reset discards any unflushed buffered data, clears any error, and
// resets b to write its output to w.
func (enc *Encoder) Reset(w io.Writer) {
	enc.w.Reset(w)
}

func (enc *Encoder) writeType(t Type) error {
	return enc.w.WriteByte(byte(t))
}

func (enc *Encoder) WriteVarUint(x uint64) (err error) {
	enc.writeType(VarUint)
	return enc.writeVarUint(x)
}

func (enc *Encoder) writeVarUint(x uint64) (err error) {
	for x >= 0x80 {
		enc.w.WriteByte(byte(x) | 0x80)
		x >>= 7
	}
	err = enc.w.WriteByte(byte(x))
	return
}

func (enc *Encoder) WriteVarInt(x int64) (err error) {
	enc.writeType(VarInt)
	return enc.writeVarInt(x)
}

func (enc *Encoder) writeVarInt(x int64) error {
	ux := uint64(x) << 1
	if x < 0 {
		ux = ^ux
	}
	return enc.writeVarUint(ux)
}

func (enc *Encoder) WriteString(v string) error {
	enc.writeType(String)
	enc.writeLen(len(v))
	_, err := enc.w.WriteString(v)
	return err
}

func (enc *Encoder) WriteBytes(v []byte) error {
	enc.writeType(ByteSlice)
	enc.writeLen(len(v))
	_, err := enc.Write(v)
	return err
}

func (enc *Encoder) Encode(v interface{}) error {
	switch v := v.(type) {
	case Marshaler:
		return v.MarshalBinny(enc)
	case encoding.BinaryMarshaler:
		return enc.WriteBinary(v)
	case gob.GobEncoder:
		return enc.WriteGob(v)
	case string:
		return enc.WriteString(v)
	case []byte:
		return enc.WriteBytes(v)
	case int64:
		return enc.WriteInt(int64(v))
	case int32:
		return enc.WriteInt(int64(v))
	case int16:
		return enc.WriteInt(int64(v))
	case int8:
		return enc.WriteInt8(v)
	case int:<commit_msg>auto flush on encode<commit_after>package binny

import (
	"bufio"
	"encoding"
	"encoding/gob"
	"io"
	"math"
	"reflect"
	"unsafe"
)

const DefaultEncoderBufferSize = 4096

// Marshaler is the interface implemented by objects that can marshal themselves into a binary representation.
// Implementing this bypasses reflection and is generally faster but not nessecery optimized.
type Marshaler interface {
	MarshalBinny(enc *Encoder) error
}

type Encoder struct {
	w *bufio.Writer
}

// NewEncoder returns a new encoder with the DefaultEncoderBufferSize
func NewEncoder(w io.Writer) *Encoder {
	return NewEncoderSize(w, DefaultEncoderBufferSize)
}

// NewEncoder returns a new encoder with the specific buffer size, minimum is 24 bytes.
func NewEncoderSize(w io.Writer, sz int) *Encoder {
	if sz < 24 {
		sz = 24
	}
	return &Encoder{
		w: bufio.NewWriterSize(w, sz),
	}
}

// Reset discards any unflushed buffered data, clears any error, and
// resets b to write its output to w.
func (enc *Encoder) Reset(w io.Writer) {
	enc.w.Reset(w)
}

func (enc *Encoder) writeType(t Type) error {
	return enc.w.WriteByte(byte(t))
}

func (enc *Encoder) WriteVarUint(x uint64) (err error) {
	enc.writeType(VarUint)
	return enc.writeVarUint(x)
}

func (enc *Encoder) writeVarUint(x uint64) (err error) {
	for x >= 0x80 {
		enc.w.WriteByte(byte(x) | 0x80)
		x >>= 7
	}
	err = enc.w.WriteByte(byte(x))
	return
}

func (enc *Encoder) WriteVarInt(x int64) (err error) {
	enc.writeType(VarInt)
	return enc.writeVarInt(x)
}

func (enc *Encoder) writeVarInt(x int64) error {
	ux := uint64(x) << 1
	if x < 0 {
		ux = ^ux
	}
	return enc.writeVarUint(ux)
}

func (enc *Encoder) WriteString(v string) error {
	enc.writeType(String)
	enc.writeLen(len(v))
	_, err := enc.w.WriteString(v)
	return err
}

func (enc *Encoder) WriteBytes(v []byte) error {
	enc.writeType(ByteSlice)
	enc.writeLen(len(v))
	_, err := enc.Write(v)
	return err
}

func (enc *Encoder) Encode(v interface{}) error {
	defer enc.Flush()
	switch v := v.(type) {
	case Marshaler:
		return v.MarshalBinny(enc)
	case encoding.BinaryMarshaler:
		return enc.WriteBinary(v)
	case gob.GobEncoder:
		return enc.WriteGob(v)
	case string:
		return enc.WriteString(v)
	case []byte:
		return enc.WriteBytes(v)
	case int64:
		return enc.WriteInt(int64(v))
	case int32:
		return enc.WriteInt(int64(v))
	case int16:
		return enc.WriteInt(int64(v))
	case int8:
		return enc.WriteInt8(v)
	case int:
{'fraction_non_alphanumeric': 0.12090418913313977, 'fraction_numerical': 0.01783492326835338, 'mean_word_length': 4.288377192982456, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4244215', 'n_tokens_mistral': 1868, 'n_tokens_neox': 1773, 'n_words': 508}