<commit_before>package message

import (
	"bytes"
	"strings"
	"io"
	"io/ioutil"
	"testing"
)

var testEncodings = []struct{
	enc string
	encoded string
	decoded string
}{
	{
		enc: "binary",
		encoded: "café",
		decoded: "café",
	},
	{
		enc: "8bit",
		encoded: "café",
		decoded: "café",
	},
	{
		enc: "7bit",
		encoded: "hi there",
		decoded: "hi there",
	},
	{
		enc: "quoted-printable",
		encoded: "caf=E9",
		decoded: "café",
	},
	{
		enc: "base64",
		encoded: "Y2Fmw6k=",
		decoded: "café",
	},
}

func Testdecode(t *testing.T) {
	for _, test := range testEncodings {
		r := decode(test.enc, strings.NewReader(test.encoded))
		if b, err := ioutil.ReadAll(r); err != nil {
			t.Errorf("Expected no error when reading encoding %q, but got: %v", test.enc, err)
		} else if s := string(b); s != test.decoded {
			t.Errorf("Expected decoded text to be %q but got %q", test.decoded, s)
		}
	}
}

func Testencode(t *testing.T) {
	for _, test := range testEncodings {
		var b bytes.Buffer
		wc := encode(test.enc, &b)
		io.WriteString(wc, test.decoded)
		wc.Close()
		if s := b.String(); s != test.encoded {
			t.Errorf("Expected encoded text to be %q but got %q", test.encoded, s)
		}
	}
}<commit_msg>message: encoding tests were not executed<commit_after>package message

import (
	"bytes"
	"strings"
	"io"
	"io/ioutil"
	"testing"
)

var testEncodings = []struct{
	enc string
	encoded string
	decoded string
}{
	{
		enc: "binary",
		encoded: "café",
		decoded: "café",
	},
	{
		enc: "8bit",
		encoded: "café",
		decoded: "café",
	},
	{
		enc: "7bit",
		encoded: "hi there",
		decoded: "hi there",
	},
	{
		enc: "quoted-printable",
		encoded: "caf=C3=A9",
		decoded: "café",
	},
	{
		enc: "base64",
		encoded: "Y2Fmw6k=",
		decoded: "café",
	},
}

func TestDecode(t *testing.T) {
	for _, test := range testEncodings {
		r := decode(test.enc, strings.NewReader(test.encoded))
		if b, err := ioutil.ReadAll(r); err != nil {
			t.Errorf("Expected no error when reading encoding %q, but got: %v", test.enc, err)
		} else if s := string(b); s != test.decoded {
			t.Errorf("Expected decoded text to be %q but got %q", test.decoded, s)
		}
	}
}

func TestEncode(t *testing.T) {
	for _, test := range testEncodings {
		var b bytes.Buffer
		wc := encode(test.enc, &b)
		io.WriteString(wc, test.decoded)
		wc.Close()
		if s := b.String(); s != test.encoded {
			t.Errorf("Expected encoded text to be %q but got %q", test.encoded, s)
		}
	}
}
{'fraction_non_alphanumeric': 0.1760592348827643, 'fraction_numerical': 0.006170300287947347, 'mean_word_length': 3.4057971014492754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1406832', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1004, 'n_words': 257}