<commit_before>package yaml

import (
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"github.com/goccy/go-yaml/ast"
	"github.com/goccy/go-yaml/internal/errors"
	"github.com/goccy/go-yaml/lexer"
	"github.com/goccy/go-yaml/parser"
	"github.com/goccy/go-yaml/printer"
	"github.com/goccy/go-yaml/token"
	"golang.org/x/xerrors"
)

const (
	// DefaultIndentSpaces default number of space for indent
	DefaultIndentSpaces = 2
)

// Encoder writes YAML values to an output stream.
type Encoder struct {
	writer             io.Writer
	opts               []EncodeOption
	indent             int
	isFlowStyle        bool
	anchorPtrToNameMap map[uintptr]string

	line        int
	column      int
	offset      int
	indentNum   int
	indentLevel int
}

// NewEncoder returns a new encoder that writes to w.
// The Encoder should be closed after use to flush all data to w.
func NewEncoder(w io.Writer, opts ...EncodeOption) *Encoder {
	return &Encoder{
		writer:             w,
		opts:               opts,
		indent:             DefaultIndentSpaces,
		anchorPtrToNameMap: map[uintptr]string{},
		line:               1,
		column:             1,
		offset:             0,
	}
}

// Close closes the encoder by writing any remaining data.
// It does not write a stream terminating string "...".
func (e *Encoder) Close() error {
	return nil
}

// Encode writes the YAML encoding of v to the stream.
// If multiple items are encoded to the stream,
// the second and subsequent document will be preceded with a "---" document separator,
// but the first will not.
//
// See the documentation for Marshal for details about the conversion of Go values to YAML.
func (e *Encoder) Encode(v interface{}) error {
	for _, opt := range e.opts {
		if err := opt(e); err != nil {
			return errors.Wrapf(err, "failed to run option for encoder")
		}
	}
	node, err := e.encodeValue(reflect.ValueOf(v), 1)
	if err != nil {
		return errors.Wrapf(err, "failed to encode value")
	}
	var p printer.Printer
	e.writer.Write(p.PrintNode(node))
	return nil
}

func (e *Encoder) encodeDocument(doc []byte) (ast.Node, error) {
	var (
		parser parser.Parser
	)
	tokens := lexer.Tokenize(string(doc))
	docNode, err := parser.Parse(tokens)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse yaml")
	}
	for _, node := range docNode.Nodes {
		if node != nil {
			return node, nil
		}
	}
	return nil, nil
}

func (e *Encoder) isInvalidValue(v reflect.Value) bool {
	if !v.IsValid() {
		return true
	}
	kind := v.Type().Kind()
	if kind == reflect.Ptr && v.IsNil() {
		return true
	}
	if kind == reflect.Interface && v.IsNil() {
		return true
	}
	return false
}

func (e *Encoder) encodeValue(v reflect.Value, column int) (ast.Node, error) {
	if e.isInvalidValue(v) {
		return e.encodeNil(), nil
	}
	if v.CanInterface() {
		if marshaler, ok := v.Interface().(BytesMarshaler); ok {
			doc, err := marshaler.MarshalYAML()
			if err != nil {
				return nil, errors.Wrapf(err, "failed to MarshalYAML")
			}
			node, err := e.encodeDocument(doc)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to encode document")
			}
			return node, nil
		} else if marshaler, ok := v.Interface().(InterfaceMarshaler); ok {
			marshalV, err := marshaler.MarshalYAML()
			if err != nil {
				return nil, errors.Wrapf(err, "failed to MarshalYAML")
			}
			return e.encodeValue(reflect.ValueOf(marshalV), column)
		}
	}
	switch v.Type().Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return e.encodeInt(v.Int()), nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return e.encodeUint(v.Uint()), nil
	case reflect.Float32, reflect.Float64:
		return e.encodeFloat(v.Float()), nil
	case reflect.Ptr, reflect.Interface:
		return e.encodeValue(v.Elem(), column)
	case reflect.String:
		return e.encodeString(v.String(), column), nil
	case reflect.Bool:
		return e.encodeBool(v.Bool()), nil
	case reflect.Slice:
		if mapSlice, ok := v.Interface().(MapSlice); ok {
			return e.encodeMapSlice(mapSlice, column)
		}
		return e.encodeSlice(v)
	case reflect.Struct:
		if v.CanInterface() {
			if mapItem, ok := v.Interface().(MapItem); ok {
				return e.encodeMapItem(mapItem, column)
			}
		}
		return e.encodeStruct(v, column)
	case reflect.Map:
		return e.encodeMap(v, column), nil
	default:
		fmt.Printf("valueType = [%s]\n", v.Type().String())
	}
	return nil, nil
}
<commit_msg>Add error handling<commit_after>package yaml

import (
	"fmt"
	"io"
	"math"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"github.com/goccy/go-yaml/ast"
	"github.com/goccy/go-yaml/internal/errors"
	"github.com/goccy/go-yaml/lexer"
	"github.com/goccy/go-yaml/parser"
	"github.com/goccy/go-yaml/printer"
	"github.com/goccy/go-yaml/token"
	"golang.org/x/xerrors"
)

const (
	// DefaultIndentSpaces default number of space for indent
	DefaultIndentSpaces = 2
)

// Encoder writes YAML values to an output stream.
type Encoder struct {
	writer             io.Writer
	opts               []EncodeOption
	indent             int
	isFlowStyle        bool
	anchorPtrToNameMap map[uintptr]string

	line        int
	column      int
	offset      int
	indentNum   int
	indentLevel int
}

// NewEncoder returns a new encoder that writes to w.
// The Encoder should be closed after use to flush all data to w.
func NewEncoder(w io.Writer, opts ...EncodeOption) *Encoder {
	return &Encoder{
		writer:             w,
		opts:               opts,
		indent:             DefaultIndentSpaces,
		anchorPtrToNameMap: map[uintptr]string{},
		line:               1,
		column:             1,
		offset:             0,
	}
}

// Close closes the encoder by writing any remaining data.
// It does not write a stream terminating string "...".
func (e *Encoder) Close() error {
	return nil
}

// Encode writes the YAML encoding of v to the stream.
// If multiple items are encoded to the stream,
// the second and subsequent document will be preceded with a "---" document separator,
// but the first will not.
//
// See the documentation for Marshal for details about the conversion of Go values to YAML.
func (e *Encoder) Encode(v interface{}) error {
	for _, opt := range e.opts {
		if err := opt(e); err != nil {
			return errors.Wrapf(err, "failed to run option for encoder")
		}
	}
	node, err := e.encodeValue(reflect.ValueOf(v), 1)
	if err != nil {
		return errors.Wrapf(err, "failed to encode value")
	}
	var p printer.Printer
	e.writer.Write(p.PrintNode(node))
	return nil
}

func (e *Encoder) encodeDocument(doc []byte) (ast.Node, error) {
	var (
		parser parser.Parser
	)
	tokens := lexer.Tokenize(string(doc))
	docNode, err := parser.Parse(tokens)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse yaml")
	}
	for _, node := range docNode.Nodes {
		if node != nil {
			return node, nil
		}
	}
	return nil, nil
}

func (e *Encoder) isInvalidValue(v reflect.Value) bool {
	if !v.IsValid() {
		return true
	}
	kind := v.Type().Kind()
	if kind == reflect.Ptr && v.IsNil() {
		return true
	}
	if kind == reflect.Interface && v.IsNil() {
		return true
	}
	return false
}

func (e *Encoder) encodeValue(v reflect.Value, column int) (ast.Node, error) {
	if e.isInvalidValue(v) {
		return e.encodeNil(), nil
	}
	if v.CanInterface() {
		if marshaler, ok := v.Interface().(BytesMarshaler); ok {
			doc, err := marshaler.MarshalYAML()
			if err != nil {
				return nil, errors.Wrapf(err, "failed to MarshalYAML")
			}
			node, err := e.encodeDocument(doc)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to encode document")
			}
			return node, nil
		} else if marshaler, ok := v.Interface().(InterfaceMarshaler); ok {
			marshalV, err := marshaler.MarshalYAML()
			if err != nil {
				return nil, errors.Wrapf(err, "failed to MarshalYAML")
			}
			return e.encodeValue(reflect.ValueOf(marshalV), column)
		}
	}
	switch v.Type().Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return e.encodeInt(v.Int()), nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return e.encodeUint(v.Uint()), nil
	case reflect.Float32, reflect.Float64:
		return e.encodeFloat(v.Float()), nil
	case reflect.Ptr, reflect.Interface:
		return e.encodeValue(v.Elem(), column)
	case reflect.String:
		return e.encodeString(v.String(), column), nil
	case reflect.Bool:
		return e.encodeBool(v.Bool()), nil
	case reflect.Slice:
		if mapSlice, ok := v.Interface().(MapSlice); ok {
			return e.encodeMapSlice(mapSlice, column)
		}
		return e.encodeSlice(v)
	case reflect.Struct:
		if v.CanInterface() {
			if mapItem, ok := v.Interface().(MapItem); ok {
				return e.encodeMapItem(mapItem, column)
			}
		}
		return e.encodeStruct(v, column)
	case reflect.Map:
		return e.encodeMap(v, column), nil
	default:
		return nil, xerrors.Errorf("unknown value type %s", v.Type().String())
	}
	return nil, nil
}

{'fraction_non_alphanumeric': 0.1301354401805869, 'fraction_numerical': 0.005191873589164785, 'mean_word_length': 3.173810645313236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2668650', 'n_tokens_mistral': 3242, 'n_tokens_neox': 3016, 'n_words': 876}