<commit_before>package ketama

import (
	"fmt"
	"math/rand"
	"testing"

	"github.com/stretchr/testify/assert"
)

type SimpleBucket struct {
	Labels  string
	Weights uint32
}

func (s *SimpleBucket) Label() string {
	return s.Labels
}

func (s *SimpleBucket) Weight() uint32 {
	return s.Weights
}

var _ Bucket = &SimpleBucket{}

func randStr(r *rand.Rand) string {
	s := ""
	for i := 0; i < 10; i++ {
		c := 'a' + r.Intn(26)
		s += string(c)
	}
	return s
}

func TestKetama(t *testing.T) {
	s := []Bucket{
		&SimpleBucket{
			Labels:  "w1",
			Weights: 1,
		},
		&SimpleBucket{
			Labels:  "w2",
			Weights: 2,
		},
	}
	c := New(s)
	assert.Equal(t, 2, len(c.Buckets()))
	r := rand.New(rand.NewSource(0))
	i0 := 0
	i1 := 0
	for i := 0; i < 10000; i++ {
		str := randStr(r)
		buck := c.Hash([]byte(str))
		if buck == s[0] {
			i0++
		} else if buck == s[1] {
			i1++
		} else {<commit_msg>fixing build breakage for ketama package<commit_after>package ketama

import (
	"fmt"
	"math/rand"
	"testing"

	"github.com/stretchr/testify/assert"
)

type SimpleBucket struct {
	Labels  string
	Weights uint32
}

func (s *SimpleBucket) Label() string {
	return s.Labels
}

func (s *SimpleBucket) Weight() uint32 {
	return s.Weights
}

var _ Bucket = &SimpleBucket{}

func randStr(r *rand.Rand) string {
	s := ""
	for i := 0; i < 10; i++ {
		c := 'a' + r.Intn(26)
		s += string(rune(c))
	}
	return s
}

func TestKetama(t *testing.T) {
	s := []Bucket{
		&SimpleBucket{
			Labels:  "w1",
			Weights: 1,
		},
		&SimpleBucket{
			Labels:  "w2",
			Weights: 2,
		},
	}
	c := New(s)
	assert.Equal(t, 2, len(c.Buckets()))
	r := rand.New(rand.NewSource(0))
	i0 := 0
	i1 := 0
	for i := 0; i < 10000; i++ {
		str := randStr(r)
		buck := c.Hash([]byte(str))
		if buck == s[0] {
			i0++
		} else if buck == s[1] {
			i1++
		} else {
{'fraction_non_alphanumeric': 0.18483674598782512, 'fraction_numerical': 0.032097399003873824, 'mean_word_length': 3.017777777777778, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3585895', 'n_tokens_mistral': 879, 'n_tokens_neox': 789, 'n_words': 195}