<commit_before>// Package serialize provides conveniency functions
// for serializing values to either JSON or XML
package serialize

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"io"
	"net/http"
	"runtime"
	"strconv"
)

type SerializationFormat int

const (
	Json SerializationFormat = iota
	Xml
)

// JSONWriter is the interface implemented by types which
// can write themselves as JSON into an io.Writer. You can
// use the gondola command for generating the code to implement
// this interface in your own types.
type JSONWriter interface {
	WriteJSON(w io.Writer) (int, error)
}

const bufSize = 4096

var (
	buffers = make(chan *bytes.Buffer, runtime.GOMAXPROCS(0))
)<commit_msg>Update documentation<commit_after>// Package serialize provides conveniency functions
// for serializing values to either JSON or XML
package serialize

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"io"
	"net/http"
	"runtime"
	"strconv"
)

// SerializationFormat indicates the format used
// to serialize an object.
type SerializationFormat int

const (
	// Serialize to JSON
	Json SerializationFormat = iota
	// Serialize to XML
	Xml
)

// JSONWriter is the interface implemented by types which
// can write themselves as JSON into an io.Writer. You can
// use the gondola command for generating the code to implement
// this interface in your own types.
type JSONWriter interface {
	// WriteJSON writes the object encoded as JSON into
	// the given io.Writer, returning the number of bytes
	// written and any potential errors.
	WriteJSON(w io.Writer) (int, error)
}

const bufSize = 4096

var (
	buffers = make(chan *bytes.Buffer, runtime.GOMAXPROCS(0))
)
{'fraction_non_alphanumeric': 0.08323281061519903, 'fraction_numerical': 0.006031363088057901, 'mean_word_length': 3.952238805970149, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330346', 'n_tokens_mistral': 502, 'n_tokens_neox': 492, 'n_words': 199}