<commit_before>package xlate

import (
	"io"

	"github.com/polydawn/go-xlate/json"
	"github.com/polydawn/go-xlate/obj"
)

func NewJsonEncoder(wr io.Writer) *JsonEncoder {
	return &JsonEncoder{wr}
}

type JsonEncoder struct {
	wr io.Writer
}

func (d *JsonEncoder) Marshal(v interface{}) {
	TokenPump{
		nil, // todo get the rest of obj.NewMarshaller(v) in place
		json.NewSerializer(d.wr),
	}.Run()
}

func NewJsonDecoder(r io.Reader) *JsonDecoder {
	return &JsonDecoder{r}
}

type JsonDecoder struct {
	r io.Reader
}

func (d *JsonDecoder) Unmarshal(v interface{}) {
	TokenPump{
		nil, // todo get the whole json package in place
		obj.NewUnmarshaler(v),
	}.Run()
}<commit_msg>Rig obj marshaller into the top level encoder setup functions, and add helpers for basic buffered use.<commit_after>package xlate

import (
	"bytes"
	"io"

	"github.com/polydawn/go-xlate/json"
	"github.com/polydawn/go-xlate/obj"
)

func NewJsonEncoder(wr io.Writer) *JsonEncoder {
	return &JsonEncoder{wr}
}

type JsonEncoder struct {
	wr io.Writer
}

func (d *JsonEncoder) Marshal(v interface{}) {
	TokenPump{
		obj.NewMarshaler(&obj.Suite{}, v),
		json.NewSerializer(d.wr),
	}.Run()
}

func JsonEncode(v interface{}) []byte {
	var buf bytes.Buffer
	NewJsonEncoder(&buf).Marshal(v)
	return buf.Bytes()
}

func NewJsonDecoder(r io.Reader) *JsonDecoder {
	return &JsonDecoder{r}
}

type JsonDecoder struct {
	r io.Reader
}

func (d *JsonDecoder) Unmarshal(v interface{}) {
	TokenPump{
		nil, // todo get the whole json package in place
		obj.NewUnmarshaler(v),
	}.Run()
}
{'fraction_non_alphanumeric': 0.13372093023255813, 'fraction_numerical': 0.0, 'mean_word_length': 4.416083916083916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4244239', 'n_tokens_mistral': 582, 'n_tokens_neox': 566, 'n_words': 144}