<commit_before>package rollinghash_test

import (
	"hash"
	"io/ioutil"
	"math/rand"
	"testing"

	"github.com/chmduquesne/rollinghash"
	_adler32 "github.com/chmduquesne/rollinghash/adler32"
	"github.com/chmduquesne/rollinghash/bozo32"
	"github.com/chmduquesne/rollinghash/buzhash32"
	"github.com/chmduquesne/rollinghash/buzhash64"
	"github.com/chmduquesne/rollinghash/rabinkarp64"
)

var allHashes = []struct {
	name    string
	classic hash.Hash
	rolling rollinghash.Hash
}{
	{"adler32", _adler32.New(), _adler32.New()},
	{"buzhash32", buzhash32.New(), buzhash32.New()},
	{"buzhash64", buzhash64.New(), buzhash64.New()},
	{"bozo32", bozo32.New(), bozo32.New()},
	{"rabinkarp64", rabinkarp64.New(), rabinkarp64.New()},
}

// Gets the hash sum as a uint64
func sum64(h hash.Hash) (res uint64) {
	buf := make([]byte, 0, 8)
	s := h.Sum(buf)
	for _, b := range s {
		res <<= 8
		res |= uint64(b)
	}
	return
}

// Compute the hash by creating a byte slice with an additionnal '\0' at
// the beginning, writing the slice without the last byte, and then
// rolling the last byte.
func SumByWriteAndRoll(h rollinghash.Hash, b []byte) uint64 {
	q := []byte("\x00")
	q = append(q, b...)

	h.Reset()
	h.Write(q[:len(q)-1])
	h.Roll(q[len(q)-1])
	return sum64(h)
}

// Compute the hash the classic way
func SumByWriteOnly(h hash.Hash, b []byte) uint64 {
	h.Reset()
	h.Write(b)
	return sum64(h)
}

// Create some random slice (length betwen 0 and 8KB, random content)
func RandomBytes() (res []byte) {
	n := rand.Intn(8192)
	res = make([]byte, n)
	rand.Read(res)
	return res
}

// Verify that, on random inputs, the classic hash and the rollinghash
// return the same values
func blackBox(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	for i := 0; i < 1000; i++ {
		in := RandomBytes()
		if len(in) > 0 {
			sum := SumByWriteAndRoll(rolling, in)<commit_msg>Reducing the memory footprint of the test suite (rabinkarp64 likes a lot of cached data)<commit_after>package rollinghash_test

import (
	"hash"
	"io/ioutil"
	"math/rand"
	"testing"

	"github.com/chmduquesne/rollinghash"
	_adler32 "github.com/chmduquesne/rollinghash/adler32"
	"github.com/chmduquesne/rollinghash/bozo32"
	"github.com/chmduquesne/rollinghash/buzhash32"
	"github.com/chmduquesne/rollinghash/buzhash64"
	"github.com/chmduquesne/rollinghash/rabinkarp64"
)

var allHashes = []struct {
	name    string
	classic hash.Hash
	rolling rollinghash.Hash
}{
	{"adler32", _adler32.New(), _adler32.New()},
	{"buzhash32", buzhash32.New(), buzhash32.New()},
	{"buzhash64", buzhash64.New(), buzhash64.New()},
	{"bozo32", bozo32.New(), bozo32.New()},
	{"rabinkarp64", rabinkarp64.New(), rabinkarp64.New()},
}

// Gets the hash sum as a uint64
func sum64(h hash.Hash) (res uint64) {
	buf := make([]byte, 0, 8)
	s := h.Sum(buf)
	for _, b := range s {
		res <<= 8
		res |= uint64(b)
	}
	return
}

// Compute the hash by creating a byte slice with an additionnal '\0' at
// the beginning, writing the slice without the last byte, and then
// rolling the last byte.
func SumByWriteAndRoll(h rollinghash.Hash, b []byte) uint64 {
	q := []byte("\x00")
	q = append(q, b...)

	h.Reset()
	h.Write(q[:len(q)-1])
	h.Roll(q[len(q)-1])
	return sum64(h)
}

// Compute the hash the classic way
func SumByWriteOnly(h hash.Hash, b []byte) uint64 {
	h.Reset()
	h.Write(b)
	return sum64(h)
}

// Create some random slice (length betwen 0 and 1KB, random content)
func RandomBytes() (res []byte) {
	n := rand.Intn(1024)
	res = make([]byte, n)
	rand.Read(res)
	return res
}

// Verify that, on random inputs, the classic hash and the rollinghash
// return the same values
func blackBox(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	for i := 0; i < 100; i++ {
		in := RandomBytes()
		if len(in) > 0 {
			sum := SumByWriteAndRoll(rolling, in)
{'fraction_non_alphanumeric': 0.15654450261780103, 'fraction_numerical': 0.041099476439790575, 'mean_word_length': 4.170500676589986, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4850560', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1483, 'n_words': 385}