<commit_before>package enc

import (
	"bufio"
	"encoding/base64"
	"errors"
	"io"
	"strconv"
	"unicode/utf8"
)

type kind byte

const (
	invalid kind = iota
	kmap
	karray
	kint
	kuint
	kfloat32
	kfloat64
	kextension
	kbytes
	kstring
	knull
)

var null = []byte("null")
var hex = []byte("0123456789abcdef")

// this is the interface
// used to write json
type jsWriter interface {
	io.Writer
	io.ByteWriter
	WriteString(string) (int, error)
}

// AsJSON reads MessagePack from src and
// translates it to JSON
func AsJSON(src io.Reader) io.Reader {
	rd, wr := io.Pipe()
	go func() {
		w := bufio.NewWriterSize(wr, 512)
		_, err := CopyToJSON(w, src)
		if err != nil {
			rd.CloseWithError(err)
		} else {
			err = w.Flush()
			if err != nil {
				rd.CloseWithError(err)
			} else {
				rd.Close()
			}
		}
	}()
	return rd
}

// CopyToJson reads a single MsgPack-encoded message from 'src' and
// writes it as JSON to 'dst'. It returns the number of bytes written,
// and any errors encountered in the process.
func CopyToJSON(dst io.Writer, src io.Reader) (n int, err error) {
	var w jsWriter
	var cast bool
	if jsw, ok := dst.(jsWriter); ok {
		w = jsw
		cast = true
	} else {
		w = bufio.NewWriterSize(w, 256)
	}
	r := NewDecoder(src)
	var k kind
	k, err = r.nextKind()
	if err != nil {
		return
	}
	switch k {
	case kmap:
		n, err = rwMap(w, r)
	case karray:
		n, err = rwArray(w, r)
	default:
		return 0, errors.New("enc: 'src' must represent a map or array")
	}
	if err != nil {
		return
	}
	if !cast {
		err = w.(*bufio.Writer).Flush()
	}
	return
}

func (m *MsgReader) nextKind() (kind, error) {
	var lead []byte
	var err error
	lead, err = m.r.Peek(1)
	if err != nil {
		return invalid, err
	}
	switch lead[0] {
	case mmap16, mmap32:
		return kmap, nil
	case marray16, marray32:
		return karray, nil
	case mfloat32:
		return kfloat32, nil
	case mfloat64:
		return kfloat64, nil
	case mint8, mint16, mint32, mint64:
		return kint, nil
	case muint8, muint16, muint32, muint64:
		return kuint, nil
	case mfixext1, mfixext2, mfixext4, mfixext8, mfixext16, mext8, mext16, mext32:
		return kextension, nil
	case mstr8, mstr16, mstr32:
		return kstring, nil
	case mbin8, mbin16, mbin32:
		return kbytes, nil
	case mnil:
		return knull, nil
	default:
		if isfixmap(lead[0]) {
			return kmap, nil
		}
		if isfixint(lead[0]) || isnfixint(lead[0]) {
			return kint, nil
		}
		if isfixstr(lead[0]) {
			return kstring, nil
		}
		if isfixarray(lead[0]) {
			return karray, nil
		}
		return invalid, nil
	}
}<commit_msg>check fixed encoding before huge switch case - saves ~100ns in bench<commit_after>package enc

import (
	"bufio"
	"encoding/base64"
	"errors"
	"io"
	"strconv"
	"unicode/utf8"
)

type kind byte

const (
	invalid kind = iota
	kmap
	karray
	kint
	kuint
	kfloat32
	kfloat64
	kextension
	kbytes
	kstring
	knull
)

var null = []byte("null")
var hex = []byte("0123456789abcdef")

// this is the interface
// used to write json
type jsWriter interface {
	io.Writer
	io.ByteWriter
	WriteString(string) (int, error)
}

// AsJSON reads MessagePack from src and
// translates it to JSON
func AsJSON(src io.Reader) io.Reader {
	rd, wr := io.Pipe()
	go func() {
		_, err := CopyToJSON(wr, src)
		if err != nil {
			rd.CloseWithError(err)
		} else {
			rd.Close()
		}
	}()
	return rd
}

// CopyToJson reads a single MsgPack-encoded message from 'src' and
// writes it as JSON to 'dst'. It returns the number of bytes written,
// and any errors encountered in the process.
func CopyToJSON(dst io.Writer, src io.Reader) (n int, err error) {
	var w jsWriter
	var cast bool
	if jsw, ok := dst.(jsWriter); ok {
		w = jsw
		cast = true
	} else {
		w = bufio.NewWriterSize(w, 256)
	}
	r := NewDecoder(src)
	var k kind
	k, err = r.nextKind()
	if err != nil {
		return
	}
	switch k {
	case kmap:
		n, err = rwMap(w, r)
	case karray:
		n, err = rwArray(w, r)
	default:
		return 0, errors.New("enc: 'src' must represent a map or array")
	}
	if err != nil {
		return
	}
	if !cast {
		err = w.(*bufio.Writer).Flush()
	}
	return
}

func (m *MsgReader) nextKind() (kind, error) {
	var lead []byte
	var err error
	lead, err = m.r.Peek(1)
	if err != nil {
		return invalid, err
	}
	if isfixmap(lead[0]) {
		return kmap, nil
	}
	if isfixint(lead[0]) || isnfixint(lead[0]) {
		return kint, nil
	}
	if isfixstr(lead[0]) {
		return kstring, nil
	}
	if isfixarray(lead[0]) {
		return karray, nil
	}
	switch lead[0] {
	case mmap16, mmap32:
		return kmap, nil
	case marray16, marray32:
		return karray, nil
	case mfloat32:
		return kfloat32, nil
	case mfloat64:
		return kfloat64, nil
	case mint8, mint16, mint32, mint64:
		return kint, nil
	case muint8, muint16, muint32, muint64:
		return kuint, nil
	case mfixext1, mfixext2, mfixext4, mfixext8, mfixext16, mext8, mext16, mext32:
		return kextension, nil
	case mstr8, mstr16, mstr32:
		return kstring, nil
	case mbin8, mbin16, mbin32:
		return kbytes, nil
	case mnil:
		return knull, nil
	default:
		return invalid, nil
	}
}
{'fraction_non_alphanumeric': 0.11982218630026267, 'fraction_numerical': 0.03313800767831885, 'mean_word_length': 3.282006920415225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5670997', 'n_tokens_mistral': 2183, 'n_tokens_neox': 1997, 'n_words': 616}