<commit_before>package codecs

import (
	"fmt"
	"io"

	"golang.org/x/net/html/charset"

	"github.com/vjeantet/bitfan/codecs/csvcodec"
	"github.com/vjeantet/bitfan/codecs/jsoncodec"
	"github.com/vjeantet/bitfan/codecs/jsonlinescodec"
	"github.com/vjeantet/bitfan/codecs/linecodec"
	"github.com/vjeantet/bitfan/codecs/multilinecodec"
	"github.com/vjeantet/bitfan/codecs/plaincodec"
	"github.com/vjeantet/bitfan/codecs/rubydebugcodec"
	"github.com/vjeantet/bitfan/core/config"
)

type Codec struct {
	Name    string
	Charset string
	Options map[string]interface{}
}

func New(name string) Codec {
	return Codec{
		Name:    name,
		Charset: "utf-8",
		Options: map[string]interface{}{},
	}
}

func NewFromConfig(conf *config.Codec) Codec {
	name := conf.Name
	c := New(name)
	for i, k := range conf.Options {
		if i == "charset" {
			c.Charset = k.(string)
			continue
		}
		c.Options[i] = k
	}
	return c
}

func (c *Codec) String() string {
	return c.Name
}

func (c *Codec) Encoder(w io.Writer) (Encoder, error) {
	var enc Encoder

	if w == nil {
		return enc, fmt.Errorf("codecs.Codec.Encoder error : no writer !")
	}

	// charset ?
	switch c.Name {
	case "pp":
		enc = rubydebugcodec.New(c.Options).Encoder(w)
	case "rubydebug":
		enc = rubydebugcodec.New(c.Options).Encoder(w)
	case "line":
		enc = linecodec.New(c.Options).Encoder(w)
	case "json":
		enc = jsoncodec.New(c.Options).Encoder(w)
		//TODO default
	}

	return enc, nil
}

func (c *Codec) Decoder(r io.Reader) (Decoder, error) {
	var dec Decoder

	if r == nil {
		return dec, fmt.Errorf("codecs.Codec.Decoder error : no reader !")
	}

	var cr io.Reader
	var err error
	cr, err = charset.NewReaderLabel(c.Charset, r)
	if err != nil {
		return dec, err
	}

	switch c.Name {
	case "line":
		dec = linecodec.New(c.Options).Decoder(cr)
	case "multiline":
		dec = multilinecodec.New(c.Options).Decoder(cr)
	case "csv":
		dec = csvcodec.New(c.Options).Decoder(cr)
	case "json":
		dec = jsoncodec.New(c.Options).Decoder(cr)
	case "json_lines":
		dec = jsonlinescodec.New(c.Options).Decoder(cr)
	default:
		dec = plaincodec.New(c.Options).Decoder(cr)
	}

	return dec, nil
}
<commit_msg>codecs - encode raise error when no encoder defined<commit_after>package codecs

import (
	"fmt"
	"io"

	"golang.org/x/net/html/charset"

	"github.com/vjeantet/bitfan/codecs/csvcodec"
	"github.com/vjeantet/bitfan/codecs/jsoncodec"
	"github.com/vjeantet/bitfan/codecs/jsonlinescodec"
	"github.com/vjeantet/bitfan/codecs/linecodec"
	"github.com/vjeantet/bitfan/codecs/multilinecodec"
	"github.com/vjeantet/bitfan/codecs/plaincodec"
	"github.com/vjeantet/bitfan/codecs/rubydebugcodec"
	"github.com/vjeantet/bitfan/core/config"
)

type Codec struct {
	Name    string
	Charset string
	Options map[string]interface{}
}

func New(name string) Codec {
	return Codec{
		Name:    name,
		Charset: "utf-8",
		Options: map[string]interface{}{},
	}
}

func NewFromConfig(conf *config.Codec) Codec {
	name := conf.Name
	c := New(name)
	for i, k := range conf.Options {
		if i == "charset" {
			c.Charset = k.(string)
			continue
		}
		c.Options[i] = k
	}
	return c
}

func (c *Codec) String() string {
	return c.Name
}

func (c *Codec) Encoder(w io.Writer) (Encoder, error) {
	var enc Encoder

	if w == nil {
		return enc, fmt.Errorf("codecs.Codec.Encoder error : no writer !")
	}

	// charset ?
	switch c.Name {
	case "pp":
		enc = rubydebugcodec.New(c.Options).Encoder(w)
	case "rubydebug":
		enc = rubydebugcodec.New(c.Options).Encoder(w)
	case "line":
		enc = linecodec.New(c.Options).Encoder(w)
	case "json":
		enc = jsoncodec.New(c.Options).Encoder(w)
	default:
		return enc, fmt.Errorf("no encoder defined")
	}

	return enc, nil
}

func (c *Codec) Decoder(r io.Reader) (Decoder, error) {
	var dec Decoder

	if r == nil {
		return dec, fmt.Errorf("codecs.Codec.Decoder error : no reader !")
	}

	var cr io.Reader
	var err error
	cr, err = charset.NewReaderLabel(c.Charset, r)
	if err != nil {
		return dec, err
	}

	switch c.Name {
	case "line":
		dec = linecodec.New(c.Options).Decoder(cr)
	case "multiline":
		dec = multilinecodec.New(c.Options).Decoder(cr)
	case "csv":
		dec = csvcodec.New(c.Options).Decoder(cr)
	case "json":
		dec = jsoncodec.New(c.Options).Decoder(cr)
	case "json_lines":
		dec = jsonlinescodec.New(c.Options).Decoder(cr)
	default:
		dec = plaincodec.New(c.Options).Decoder(cr)
	}

	return dec, nil
}

{'fraction_non_alphanumeric': 0.15059687786960516, 'fraction_numerical': 0.0004591368227731864, 'mean_word_length': 3.830376940133038, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5782404', 'n_tokens_mistral': 1809, 'n_tokens_neox': 1724, 'n_words': 353}