<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package flate

import (
	"bytes";
	"io";
	"os";
	"testing";
)

type deflateTest struct {
	in []byte;
	level int;
	out []byte;
}

type deflateInflateTest struct {
	in [] byte;
}

type reverseBitsTest struct {
	in uint16;
	bitCount uint8;
	out uint16;
}

var deflateTests = []*deflateTest {
	&deflateTest { []byte{ }, 0, []byte{ 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, -1, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, DefaultCompression, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, 4, []byte{ 18, 4, 4, 0, 0, 255, 255 } },

	&deflateTest { []byte{ 0x11 }, 0, []byte { 0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x12 }, 0, []byte{ 0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, 0,
		[]byte{ 0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{}, 1, []byte{ 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, 1, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x12 }, 1, []byte{ 18, 20, 2, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, 1, []byte{ 18, 132, 2, 64, 0, 0, 0, 255, 255 } },
	&deflateTest { []byte{}, 9, []byte{ 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, 9, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x12 }, 9, []byte{ 18, 20, 2, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, 9, []byte{ 18, 132, 2, 64, 0, 0, 0, 255, 255 } },
}

var deflateInflateTests = []*deflateInflateTest {
	&deflateInflateTest { []byte{ } },
	&deflateInflateTest { []byte{ 0x11 } },
	&deflateInflateTest { []byte{ 0x11, 0x12 } },
	&deflateInflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 } },
	&deflateInflateTest { []byte{ 0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13 } },
	&deflateInflateTest { getLargeDataChunk() },
}

var reverseBitsTests = []*reverseBitsTest {
	&reverseBitsTest { 1, 1, 1 },
	&reverseBitsTest { 1, 2, 2 },
	&reverseBitsTest { 1, 3, 4 },
	&reverseBitsTest { 1, 4, 8 },
	&reverseBitsTest { 1, 5, 16 },
	&reverseBitsTest { 17, 5, 17 },
	&reverseBitsTest { 257, 9, 257 },
	&reverseBitsTest { 29, 5, 23 },
}

func getLargeDataChunk() []byte {
	result := make([]byte, 100000);
	for i := range result {
		result[i] = byte(int64(i) * int64(i) & 0xFF);
	}
	return result;
}

func TestDeflate(t *testing.T) {
	for _, h := range deflateTests {
		buffer := bytes.NewBuffer([]byte{});
		w := NewDeflater(buffer, h.level);
		w.Write(h.in);
		w.Close();
		if bytes.Compare(buffer.Bytes(), h.out) != 0 {
			t.Errorf("buffer is wrong; level = %v, buffer.Bytes() = %v, expected output = %v",
				h.level, buffer.Bytes(), h.out);
		}
	}
}

func testToFromWithLevel(t *testing.T, level int, input []byte) os.Error {
	buffer := bytes.NewBuffer([]byte{});
	w := NewDeflater(buffer, level);
	w.Write(input);
	w.Close();
	arr := buffer.Bytes();
	t.Logf("compressed: %v, %v", len(arr), arr);
	inflater := NewInflater(buffer);
	decompressed, err := io.ReadAll(inflater);
	if err != nil && err != os.EOF {
		t.Errorf("The error reading the buffer, %v", err);
		return err;
	}
	inflater.Close();
	if bytes.Compare(input, decompressed) != 0 {
		t.Errorf("the data was changed after deflate/inflate. Level: %v, input: %v, decompressed: %v",
			level, input, decompressed);
	}
	return nil;
}

func testToFrom(t * testing.T, input[] byte) {
	for i := 0; i < 10; i++ {
		testToFromWithLevel(t, i, input);
	}
}

func TestDeflateInflate(t *testing.T) {
	for _, h := range deflateInflateTests {
		testToFrom(t, h.in);
	}
}

func TestReverseBits(t *testing.T) {
	for _, h := range reverseBitsTests {
		if v := reverseBits(h.in, h.bitCount); v != h.out {
			t.Errorf("reverseBits(%v,%v) returned %v, %v expected",
				h.in, h.bitCount, v, h.out);
		}
	}
}

func TestDeflateInflateString(t *testing.T) {
	gold := bytes.NewBufferString(getEdata()).Bytes();
	testToFromWithLevel(t, 1, gold);
}

func getEdata() string {
	return "2.718281828459045235360287471352662497757247093699959574966967627724076630353547"+
	"59457138217852516642742746639193200305992181741359662904357290033429526059563073"+
	"81323286279434907632338298807531952510190115738341879307021540891499348841675092"+
	"44761460668082264800168477411853742345442437107539077744992069551702761838606261"+
	"33138458300075204493382656029760673711320070932870912744374704723069697720931014"+
	"16928368190255151086574637721112523897844250569536967707854499699679468644549059"+
	"87931636889230098793127736178215424999229576351482208269895193668033182528869398"+
	"49646510582093923982948879332036250944311730123819706841614039701983767932068328"+
	"23764648042953118023287825098194558153017567173613320698112509961818815930416903"+
	"51598888519345807273866738589422879228499892086805825749279610484198444363463244"+
	"96848756023362482704197862320900216099023530436994184914631409343173814364054625"+
	"31520961836908887070167683964243781405927145635490613031072085103837505101157477"+
	"04171898610687396965521267154688957035035402123407849819334321068170121005627880"+
	"23519303322474501585390473041995777709350366041699732972508868769664035557071622"+
	"68447162560798826517871341951246652010305921236677194325278675398558944896970964"+
	"09754591856956380236370162112047742722836489613422516445078182442352948636372141"+
	"74023889344124796357437026375529444833799801612549227850925778256209262264832627"+<commit_msg>clean up error output; do not print entire buffer on error, since it can be very large. instead print identifying string.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package flate

import (
	"bytes";
	"fmt";
	"io";
	"os";
	"testing";
)

type deflateTest struct {
	in []byte;
	level int;
	out []byte;
}

type deflateInflateTest struct {
	in [] byte;
}

type reverseBitsTest struct {
	in uint16;
	bitCount uint8;
	out uint16;
}

var deflateTests = []*deflateTest {
	&deflateTest { []byte{ }, 0, []byte{ 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, -1, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, DefaultCompression, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, 4, []byte{ 18, 4, 4, 0, 0, 255, 255 } },

	&deflateTest { []byte{ 0x11 }, 0, []byte { 0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x12 }, 0, []byte{ 0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, 0,
		[]byte{ 0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{}, 1, []byte{ 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, 1, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x12 }, 1, []byte{ 18, 20, 2, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, 1, []byte{ 18, 132, 2, 64, 0, 0, 0, 255, 255 } },
	&deflateTest { []byte{}, 9, []byte{ 1, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11 }, 9, []byte{ 18, 4, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x12 }, 9, []byte{ 18, 20, 2, 4, 0, 0, 255, 255 } },
	&deflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 }, 9, []byte{ 18, 132, 2, 64, 0, 0, 0, 255, 255 } },
}

var deflateInflateTests = []*deflateInflateTest {
	&deflateInflateTest { []byte{ } },
	&deflateInflateTest { []byte{ 0x11 } },
	&deflateInflateTest { []byte{ 0x11, 0x12 } },
	&deflateInflateTest { []byte{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 } },
	&deflateInflateTest { []byte{ 0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13 } },
	&deflateInflateTest { getLargeDataChunk() },
}

var reverseBitsTests = []*reverseBitsTest {
	&reverseBitsTest { 1, 1, 1 },
	&reverseBitsTest { 1, 2, 2 },
	&reverseBitsTest { 1, 3, 4 },
	&reverseBitsTest { 1, 4, 8 },
	&reverseBitsTest { 1, 5, 16 },
	&reverseBitsTest { 17, 5, 17 },
	&reverseBitsTest { 257, 9, 257 },
	&reverseBitsTest { 29, 5, 23 },
}

func getLargeDataChunk() []byte {
	result := make([]byte, 100000);
	for i := range result {
		result[i] = byte(int64(i) * int64(i) & 0xFF);
	}
	return result;
}

func TestDeflate(t *testing.T) {
	for _, h := range deflateTests {
		buffer := bytes.NewBuffer([]byte{});
		w := NewDeflater(buffer, h.level);
		w.Write(h.in);
		w.Close();
		if bytes.Compare(buffer.Bytes(), h.out) != 0 {
			t.Errorf("buffer is wrong; level = %v, buffer.Bytes() = %v, expected output = %v",
				h.level, buffer.Bytes(), h.out);
		}
	}
}

func testToFromWithLevel(t *testing.T, level int, input []byte, name string) os.Error {
	buffer := bytes.NewBuffer([]byte{});
	w := NewDeflater(buffer, level);
	w.Write(input);
	w.Close();
	inflater := NewInflater(buffer);
	decompressed, err := io.ReadAll(inflater);
	if err != nil {
		t.Errorf("reading inflater: %s", err);
		return err;
	}
	inflater.Close();
	if bytes.Compare(input, decompressed) != 0 {
		t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name);
	}
	return nil;
}

func testToFrom(t *testing.T, input[] byte, name string) {
	for i := 0; i < 10; i++ {
		testToFromWithLevel(t, i, input, name);
	}
}

func TestDeflateInflate(t *testing.T) {
	for i, h := range deflateInflateTests {
		testToFrom(t, h.in, fmt.Sprintf("#%d", i));
	}
}

func TestReverseBits(t *testing.T) {
	for _, h := range reverseBitsTests {
		if v := reverseBits(h.in, h.bitCount); v != h.out {
			t.Errorf("reverseBits(%v,%v) = %v, want %v",
				h.in, h.bitCount, v, h.out);
		}
	}
}

func TestDeflateInflateString(t *testing.T) {
	gold := bytes.NewBufferString(getEdata()).Bytes();
	testToFromWithLevel(t, 1, gold, "2.718281828...");
}

func getEdata() string {
	return "2.718281828459045235360287471352662497757247093699959574966967627724076630353547"+
	"59457138217852516642742746639193200305992181741359662904357290033429526059563073"+
	"81323286279434907632338298807531952510190115738341879307021540891499348841675092"+
	"44761460668082264800168477411853742345442437107539077744992069551702761838606261"+
	"33138458300075204493382656029760673711320070932870912744374704723069697720931014"+
	"16928368190255151086574637721112523897844250569536967707854499699679468644549059"+
	"87931636889230098793127736178215424999229576351482208269895193668033182528869398"+
	"49646510582093923982948879332036250944311730123819706841614039701983767932068328"+
	"23764648042953118023287825098194558153017567173613320698112509961818815930416903"+
	"51598888519345807273866738589422879228499892086805825749279610484198444363463244"+
	"96848756023362482704197862320900216099023530436994184914631409343173814364054625"+
	"31520961836908887070167683964243781405927145635490613031072085103837505101157477"+
	"04171898610687396965521267154688957035035402123407849819334321068170121005627880"+
	"23519303322474501585390473041995777709350366041699732972508868769664035557071622"+
	"68447162560798826517871341951246652010305921236677194325278675398558944896970964"+
	"09754591856956380236370162112047742722836489613422516445078182442352948636372141"+
	"74023889344124796357437026375529444833799801612549227850925778256209262264832627"+
{'fraction_non_alphanumeric': 0.15745161010328965, 'fraction_numerical': 0.3187223331308046, 'mean_word_length': 5.261956521739131, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2874146', 'n_tokens_mistral': 7405, 'n_tokens_neox': 4940, 'n_words': 1083}