<commit_before>package tester

// These tests are in the tester package rather than the xmlwriter
// package to try to avoid issues with tools like dep vendoring more
// stuff than it should (https://github.com/golang/dep/issues/120).

import (
	"bytes"
	"fmt"
	"strings"
	"testing"
	"unicode/utf8"

	xw "github.com/shabbyrobe/xmlwriter"

	tt "github.com/shabbyrobe/xmlwriter/testtool"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
)

func TestEncodingWindows1252(t *testing.T) {
	b := &bytes.Buffer{}
	enc := charmap.Windows1252.NewEncoder()
	w := xw.OpenEncoding(b, "windows-1252", enc)
	xw.Must(w.Start(xw.Doc{}))
	xw.Must(w.Start(xw.Elem{Name: "hello"}))
	xw.Must(w.Write(xw.Text("Résumé")))
	xw.Must(w.Write(xw.Text("😀")))
	xw.Must(w.EndAllFlush())
	out := b.Bytes()

	// byte representation of expected windows-1252 encoded text -
	// attempting to decode as string yields panic
	check := []byte{'R', 0xE9, 's', 'u', 'm', 0xE9, '&', '#', '1', '2', '8', '5', '1', '2', ';'}
	tt.Assert(t, bytes.Contains(out, check))
}

func TestEncodeRunesInISO88591(t *testing.T) {
	b := &bytes.Buffer{}
	enc := charmap.ISO8859_1.NewEncoder()
	w := xw.OpenEncoding(b, "ISO-8859-1", enc)
	xw.Must(w.Start(xw.Doc{}))
	xw.Must(w.Start(xw.Elem{Name: "hello"}))
	xw.Must(w.Write(xw.Text("😀")))
	xw.Must(w.EndAllFlush())
	out := b.String()

	check := "<hello>&#128512;</hello>"
	tt.Assert(t, strings.Contains(out, check))
}

func TestAssumptionsAboutHTMLEscaper(t *testing.T) {
	encoder := charmap.ISO8859_1.NewEncoder()

	for i := 0; i < 16384; i++ {
		b := &bytes.Buffer{}
		writer := encoding.HTMLEscapeUnsupported(encoder).Writer(b)
		dst := make([]byte, 32)
		r := rune(i)
		l := utf8.EncodeRune(dst, r)
		writer.Write(dst[:l])
		if i < 256 {
			tt.Equals(t, string([]byte{byte(i)}), b.String())
		} else {<commit_msg>Add UTF-16LE test<commit_after>package tester

// These tests are in the tester package rather than the xmlwriter
// package to try to avoid issues with tools like dep vendoring more
// stuff than it should (https://github.com/golang/dep/issues/120).

import (
	"bytes"
	"fmt"
	"strings"
	"testing"
	"unicode/utf8"

	xw "github.com/shabbyrobe/xmlwriter"

	tt "github.com/shabbyrobe/xmlwriter/testtool"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/encoding/unicode"
)

func TestEncodingWindows1252(t *testing.T) {
	b := &bytes.Buffer{}
	enc := charmap.Windows1252.NewEncoder()
	w := xw.OpenEncoding(b, "windows-1252", enc)
	xw.Must(w.Start(xw.Doc{}))
	xw.Must(w.Start(xw.Elem{Name: "hello"}))
	xw.Must(w.Write(xw.Text("Résumé")))
	xw.Must(w.Write(xw.Text("😀")))
	xw.Must(w.EndAllFlush())
	out := b.Bytes()

	// byte representation of expected windows-1252 encoded text -
	// attempting to decode as string yields panic
	check := []byte{'R', 0xE9, 's', 'u', 'm', 0xE9, '&', '#', '1', '2', '8', '5', '1', '2', ';'}
	tt.Assert(t, bytes.Contains(out, check))
}

func TestEncodingUTF16BE(t *testing.T) {
	b := &bytes.Buffer{}
	enc := unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM).NewEncoder()
	w := xw.OpenEncoding(b, "utf-16be", enc)
	xw.Must(w.Start(xw.Doc{}))
	xw.Must(w.Start(xw.Elem{Name: "hello"}))
	xw.Must(w.Write(xw.Text("Résumé")))
	xw.Must(w.Write(xw.Text("😀")))
	xw.Must(w.EndAllFlush())
	out := b.Bytes()

	tt.Assert(t, bytes.HasPrefix(out, []byte{0xFE, 0xFF}))
	tt.Assert(t, bytes.Contains(out, []byte{0xD8, 0x3D, 0xDE, 0x00}))
	tt.Assert(t, bytes.Contains(out, []byte{0x00, 0x3C, 0x00, 0x68, 0x00, 0x65, 0x00, 0x6C, 0x00, 0x6C, 0x00, 0x6F}))
}

func TestEncodeRunesInISO88591(t *testing.T) {
	b := &bytes.Buffer{}
	enc := charmap.ISO8859_1.NewEncoder()
	w := xw.OpenEncoding(b, "ISO-8859-1", enc)
	xw.Must(w.Start(xw.Doc{}))
	xw.Must(w.Start(xw.Elem{Name: "hello"}))
	xw.Must(w.Write(xw.Text("😀")))
	xw.Must(w.EndAllFlush())
	out := b.String()

	check := "<hello>&#128512;</hello>"
	tt.Assert(t, strings.Contains(out, check))
}

func TestAssumptionsAboutHTMLEscaper(t *testing.T) {
	encoder := charmap.ISO8859_1.NewEncoder()

	for i := 0; i < 16384; i++ {
		b := &bytes.Buffer{}
		writer := encoding.HTMLEscapeUnsupported(encoder).Writer(b)
		dst := make([]byte, 32)
		r := rune(i)
		l := utf8.EncodeRune(dst, r)
		writer.Write(dst[:l])
		if i < 256 {
			tt.Equals(t, string([]byte{byte(i)}), b.String())
		} else {
{'fraction_non_alphanumeric': 0.2071611253196931, 'fraction_numerical': 0.043245756800744016, 'mean_word_length': 4.515384615384615, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6504557', 'n_tokens_mistral': 1971, 'n_tokens_neox': 1829, 'n_words': 337}