<commit_before>package msgpack

import (
	"bytes"
	"fmt"
	"reflect"
	"sync"

	"github.com/vmihailenco/msgpack/codes"
)

var extTypes = make(map[int8]reflect.Type)

var bufferPool = &sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}

// RegisterExt records a type, identified by a value for that type,
// under the provided id. That id will identify the concrete type of a value
// sent or received as an interface variable. Only types that will be
// transferred as implementations of interface values need to be registered.
// Expecting to be used only during initialization, it panics if the mapping
// between types and ids is not a bijection.
func RegisterExt(id int8, value interface{}) {
	typ := reflect.TypeOf(value)
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}
	ptr := reflect.PtrTo(typ)

	if _, ok := extTypes[id]; ok {
		panic(fmt.Errorf("msgpack: ext with id=%d is already registered", id))
	}

	registerExt(id, ptr, getEncoder(ptr), nil)
	registerExt(id, typ, getEncoder(typ), getDecoder(typ))
}

func registerExt(id int8, typ reflect.Type, enc encoderFunc, dec decoderFunc) {
	if dec != nil {
		extTypes[id] = typ
	}
	if enc != nil {
		typEncMap[typ] = makeExtEncoder(id, enc)
	}
	if dec != nil {
		typDecMap[typ] = dec
	}
}

func makeExtEncoder(typeId int8, enc encoderFunc) encoderFunc {
	return func(e *Encoder, v reflect.Value) error {
		buf := bufferPool.Get().(*bytes.Buffer)
		defer bufferPool.Put(buf)
		buf.Reset()

		oldw := e.w
		e.w = buf
		err := enc(e, v)
		e.w = oldw

		if err != nil {
			return err
		}

		if err := e.encodeExtLen(buf.Len()); err != nil {
			return err
		}
		if err := e.w.WriteByte(byte(typeId)); err != nil {
			return err
		}
		return e.write(buf.Bytes())
	}
}

func (e *Encoder) encodeExtLen(l int) error {
	switch l {
	case 1:
		return e.writeCode(codes.FixExt1)
	case 2:
		return e.writeCode(codes.FixExt2)
	case 4:
		return e.writeCode(codes.FixExt4)
	case 8:
		return e.writeCode(codes.FixExt8)
	case 16:
		return e.writeCode(codes.FixExt16)
	}
	if l < 256 {
		return e.write1(codes.Ext8, uint64(l))<commit_msg>Export EncodeExtHeader in Encoder<commit_after>package msgpack

import (
	"bytes"
	"fmt"
	"reflect"
	"sync"

	"github.com/vmihailenco/msgpack/codes"
)

var extTypes = make(map[int8]reflect.Type)

var bufferPool = &sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}

// RegisterExt records a type, identified by a value for that type,
// under the provided id. That id will identify the concrete type of a value
// sent or received as an interface variable. Only types that will be
// transferred as implementations of interface values need to be registered.
// Expecting to be used only during initialization, it panics if the mapping
// between types and ids is not a bijection.
func RegisterExt(id int8, value interface{}) {
	typ := reflect.TypeOf(value)
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}
	ptr := reflect.PtrTo(typ)

	if _, ok := extTypes[id]; ok {
		panic(fmt.Errorf("msgpack: ext with id=%d is already registered", id))
	}

	registerExt(id, ptr, getEncoder(ptr), nil)
	registerExt(id, typ, getEncoder(typ), getDecoder(typ))
}

func registerExt(id int8, typ reflect.Type, enc encoderFunc, dec decoderFunc) {
	if dec != nil {
		extTypes[id] = typ
	}
	if enc != nil {
		typEncMap[typ] = makeExtEncoder(id, enc)
	}
	if dec != nil {
		typDecMap[typ] = dec
	}
}

func (e *Encoder) EncodeExtHeader(typeId byte, length int) error {
	if err := e.encodeExtLen(length); err != nil {
		return err
	}
	if err := e.w.WriteByte(typeId); err != nil {
		return err
	}
	return nil
}

func makeExtEncoder(typeId int8, enc encoderFunc) encoderFunc {
	return func(e *Encoder, v reflect.Value) error {
		buf := bufferPool.Get().(*bytes.Buffer)
		defer bufferPool.Put(buf)
		buf.Reset()

		oldw := e.w
		e.w = buf
		err := enc(e, v)
		e.w = oldw

		if err != nil {
			return err
		}

		if err := e.EncodeExtHeader(byte(typeId), buf.Len()); err != nil {
			return err
		}
		return e.write(buf.Bytes())
	}
}

func (e *Encoder) encodeExtLen(l int) error {
	switch l {
	case 1:
		return e.writeCode(codes.FixExt1)
	case 2:
		return e.writeCode(codes.FixExt2)
	case 4:
		return e.writeCode(codes.FixExt4)
	case 8:
		return e.writeCode(codes.FixExt8)
	case 16:
		return e.writeCode(codes.FixExt16)
	}
	if l < 256 {
		return e.write1(codes.Ext8, uint64(l))
{'fraction_non_alphanumeric': 0.1298671552908841, 'fraction_numerical': 0.010535959688502062, 'mean_word_length': 3.6755888650963597, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1926041', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1558, 'n_words': 472}