<commit_before>// Copyright (c) 2013, <NAME>.
// Use of this source code is governed by LICENSE file.

package bloom

import (
	"crypto/sha256"
	"math"
	"math/big"
)

type Filter struct {
	Capacity      int
	ErrorRate     float64
	bitTotal      uint64
	bitArray      []byte
	hashFunctions int
	chunkSize     int
}

func New(capacity int, errorRate float64) Filter {
	bitTotal := uint64(math.Pow(2, math.Ceil(math.Log2(-float64(capacity)*math.Log(errorRate)/math.Pow(math.Log(2), 2)))))
	bitArray := make([]byte, bitTotal>>3)
	hashFunctions := int(math.Floor(math.Log2(1.0 / errorRate)))
	chunkSize := int(math.Ceil(math.Log2(float64(bitTotal)) / 8))

	return Filter{
		capacity,
		errorRate,
		bitTotal,
		bitArray,
		hashFunctions,
		chunkSize,
	}
}

func (f *Filter) Add(key []byte) {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		f.bitArray[byteIndex] |= bitMask
	}

	// fmt.Printf("\n")
}

func (f *Filter) Contains(key []byte) bool {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		if f.bitArray[byteIndex]&bitMask == 0 {
			return false
		}
	}

	return true
}

func (f *Filter) generateHashForKey(key []byte) []byte {
	var completeHash []byte
	var hash = sha256.New()

	hash.Write(key)
	completeHash = append(completeHash, hash.Sum(nil)...)

	for f.chunkSize*f.hashFunctions > len(completeHash) {
		hash.Reset()
		hash.Write(completeHash)
		completeHash = append(completeHash, hash.Sum(nil)...)
	}<commit_msg>Returns pointer instead of a copy of the filter<commit_after>// Copyright (c) 2013, <NAME>.
// Use of this source code is governed by LICENSE file.

package bloom

import (
	"crypto/sha256"
	"math"
	"math/big"
)

type Filter struct {
	Capacity      int
	ErrorRate     float64
	bitTotal      uint64
	bitArray      []byte
	hashFunctions int
	chunkSize     int
}

func New(capacity int, errorRate float64) *Filter {
	bitTotal := uint64(math.Pow(2, math.Ceil(math.Log2(-float64(capacity)*math.Log(errorRate)/math.Pow(math.Log(2), 2)))))
	bitArray := make([]byte, bitTotal>>3)
	hashFunctions := int(math.Floor(math.Log2(1.0 / errorRate)))
	chunkSize := int(math.Ceil(math.Log2(float64(bitTotal)) / 8))

	return &Filter{
		capacity,
		errorRate,
		bitTotal,
		bitArray,
		hashFunctions,
		chunkSize,
	}
}

func (f *Filter) Add(key []byte) {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		f.bitArray[byteIndex] |= bitMask
	}
}

func (f *Filter) Contains(key []byte) bool {
	hash := f.generateHashForKey(key)

	for index := 0; index < f.hashFunctions; index++ {
		byteIndex, bitMask := f.findBitArrayIndexAndMask(hash, index)

		if f.bitArray[byteIndex]&bitMask == 0 {
			return false
		}
	}

	return true
}

func (f *Filter) generateHashForKey(key []byte) []byte {
	var completeHash []byte
	var hash = sha256.New()

	hash.Write(key)
	completeHash = append(completeHash, hash.Sum(nil)...)

	for f.chunkSize*f.hashFunctions > len(completeHash) {
		hash.Reset()
		hash.Write(completeHash)
		completeHash = append(completeHash, hash.Sum(nil)...)
	}
{'fraction_non_alphanumeric': 0.1446886446886447, 'fraction_numerical': 0.021367521367521368, 'mean_word_length': 4.096423017107309, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948049', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1167, 'n_words': 271}