<commit_before>package distro

import (
	"bytes"
	"testing"
)

const (
	clearString = "This is the test string used for testing"
)

var gzipArray = []byte{31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 10, 201, 200, 44, 86, 200, 44, 86, 40, 201, 72, 85, 40, 73, 45, 46, 81, 40, 46, 41, 202, 204, 75, 87, 40, 45, 78, 77, 81, 72, 203, 47, 2, 11, 102, 230, 165, 3, 2, 0, 0, 255, 255, 45, 117, 163, 29, 40, 0, 0, 0}

// Output from gzip
//0x1f, 0x8b, 0x08, 0x08, 0x0f, 0xb9, 0xed, 0x59, 0x00, 0x03, 0x6b, 0x6b, 0x00, 0x0b, 0xc9, 0xc8, 0x2c, 0x56, 0x00, 0xa2, 0x92, 0x8c, 0x54, 0x85, 0x92, 0xd4, 0xe2, 0x12, 0x85, 0xe2, 0x92, 0xa2, 0xcc, 0xbc, 0x74, 0x85, 0xd2, 0xe2, 0xd4, 0x14, 0x85, 0xb4, 0xfc, 0x22, 0xb0, 0x20, 0x50, 0x00, 0x00, 0x2d, 0x75, 0xa3, 0x1d, 0x28, 0x00, 0x00, 0x00}

//FIXME
var zlibArray = []byte{120, 156, 10, 201, 200, 44, 86, 200, 44, 86, 40, 201, 72, 85, 40, 73, 45, 46, 81, 40, 46, 41, 202, 204, 75, 87, 40, 45, 78, 77, 81, 72, 203, 47, 2, 11, 102, 230, 165, 3, 2, 0, 0, 255, 255, 42, 231, 14, 227}

func TestGzip(t *testing.T) {
	buf := bytes.Buffer{}
	buf.WriteString(clearString)

	comp := gzipTransformer{}

	enc := comp.Transform(buf)

	if bytes.Compare(enc.Bytes(), gzipArray) != 0 {
		t.Fatal("Encoded string ", enc.Bytes(), " is not ", gzipArray)
	}
}

func TestZlib(t *testing.T) {
	buf := bytes.Buffer{}
	buf.WriteString(clearString)

	comp := zlibTransformer{}

	enc := comp.Transform(buf)

	if bytes.Compare(enc.Bytes(), zlibArray) != 0 {
		t.Fatal("Encoded string ", enc.Bytes(), " is not ", gzipArray)
	}
}

var result bytes.Buffer

func BenchmarkGzip(b *testing.B) {
	buf := bytes.Buffer{}
	buf.WriteString(clearString)

	comp := gzipTransformer{}

	enc := bytes.Buffer{}
	for i := 0; i < 1000; i++ {
		enc = comp.Transform(buf)
	}
	result = enc
}

func BenchmarkZlib(b *testing.B) {
	buf := bytes.Buffer{}
	buf.WriteString(clearString)

	comp := zlibTransformer{}

	enc := bytes.Buffer{}
	for i := 0; i < 1000; i++ {
		enc = comp.Transform(buf)
	}
	result = enc
}<commit_msg>Update tests<commit_after>package distro

import (
	"testing"
)

const (
	clearString = "This is the test string used for testing"
	gzipString  = "H4sIAAAJbogA/wrJyCxWyCxWKMlIVShJLS5RKC4pysxLVygtTk1RSMsvAgtm5qUDAgAA//8tdaMdKAAAAA=="
	zlibString  = "eJwKycgsVsgsVijJSFUoSS0uUSguKcrMS1coLU5NUUjLLwILZualAwIAAP//KucO4w=="
)

func TestGzip(t *testing.T) {

	comp := gzipTransformer{}
	enc := comp.Transform([]byte(clearString))

	if string(enc) != gzipString {
		t.Fatal("Encoded string ", string(enc), " is not ", gzipString)
	}
}

func TestZlib(t *testing.T) {

	comp := zlibTransformer{}
	enc := comp.Transform([]byte(clearString))
	if string(enc) != zlibString {
		t.Fatal("Encoded string ", string(enc), " is not ", zlibString)
	}
}

var result []byte

func BenchmarkGzip(b *testing.B) {

	comp := gzipTransformer{}

	var enc []byte
	for i := 0; i < 1000; i++ {
		enc = comp.Transform([]byte(clearString))
	}
	result = enc
}

func BenchmarkZlib(b *testing.B) {

	comp := zlibTransformer{}

	var enc []byte
	for i := 0; i < 1000; i++ {
		enc = comp.Transform([]byte(clearString))
	}
	result = enc
}
{'fraction_non_alphanumeric': 0.17010974822466107, 'fraction_numerical': 0.131375080697224, 'mean_word_length': 4.080327868852459, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243481', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1359, 'n_words': 363}