<commit_before>package ratelimiter

import (
	"errors"
	"strconv"
)

var (
	errLimitReached = errors.New("Limit reached")
)

// RateLimiter represents a rate limiter
type RateLimiter interface {
	AddBucket(duration int, limit int) RateLimiter
	SetBuckets(buckets map[int]int) RateLimiter
	DelBucket(duration int) RateLimiter
	Validate(key string) error
}

// Cache represents the cache holding values typically a Memcache
type Cache interface {
	Set(k string, v []byte, ttl int32) error
	Decrement(key string, delta uint64) (uint64, error)
}

type rateLimiter struct {
	buckets map[int]int
	cache   Cache
	errCacheMiss error
}

// NewRateLimiter creates a new rate limiter
func NewRateLimiter(cache Cache, errCacheMiss error) RateLimiter {
	return &rateLimiter{
		buckets: make(map[int]int),
		cache:   cache,
		errCacheMiss: errCacheMiss,
	}
}

func (r *rateLimiter) AddBucket(duration int, limit int) RateLimiter {
	r.buckets[duration] = limit
	return r
}

func (r *rateLimiter) SetBuckets(buckets map[int]int) RateLimiter {
	r.buckets = buckets
	return r
}

func (r *rateLimiter) DelBucket(duration int) RateLimiter {
	delete(r.buckets, duration)
	return r
}

func (r rateLimiter) Validate(key string) error {
	// Loop through buckets
	for duration, limit := range r.buckets {
		// Decrement
		v, e := r.cache.Decrement(r.transformKey(key, duration), 1)

		// Process error
		if e != nil {
			// Cache miss
			if e.Error() == r.errCacheMiss.Error() {
				// Create key
				r.cache.Set(r.transformKey(key, duration), []byte(strconv.Itoa(limit)), int32(duration))
			} else {
				// Return
				return e
			}
		} else if v == 0 {
			return errLimitReached
		}
	}

	// Default return
	return nil
}

func (r rateLimiter) transformKey(key string, duration int) string {
	return key + "_" + strconv.Itoa(duration)
}<commit_msg>Update transform key<commit_after>package ratelimiter

import (
	"errors"
	"strconv"
)

var (
	errLimitReached = errors.New("Limit reached")
)

// RateLimiter represents a rate limiter
type RateLimiter interface {
	AddBucket(duration int, limit int) RateLimiter
	SetBuckets(buckets map[int]int) RateLimiter
	DelBucket(duration int) RateLimiter
	Validate(key string) error
}

// Cache represents the cache holding values typically a Memcache
type Cache interface {
	Set(k string, v []byte, ttl int32) error
	Decrement(key string, delta uint64) (uint64, error)
}

type rateLimiter struct {
	buckets map[int]int
	cache   Cache
	errCacheMiss error
}

// NewRateLimiter creates a new rate limiter
func NewRateLimiter(cache Cache, errCacheMiss error) RateLimiter {
	return &rateLimiter{
		buckets: make(map[int]int),
		cache:   cache,
		errCacheMiss: errCacheMiss,
	}
}

func (r *rateLimiter) AddBucket(duration int, limit int) RateLimiter {
	r.buckets[duration] = limit
	return r
}

func (r *rateLimiter) SetBuckets(buckets map[int]int) RateLimiter {
	r.buckets = buckets
	return r
}

func (r *rateLimiter) DelBucket(duration int) RateLimiter {
	delete(r.buckets, duration)
	return r
}

func (r rateLimiter) Validate(key string) error {
	// Loop through buckets
	for duration, limit := range r.buckets {
		// Decrement
		v, e := r.cache.Decrement(r.transformKey(key, duration), 1)

		// Process error
		if e != nil {
			// Cache miss
			if e.Error() == r.errCacheMiss.Error() {
				// Create key
				r.cache.Set(r.transformKey(key, duration), []byte(strconv.Itoa(limit)), int32(duration))
			} else {
				// Return
				return e
			}
		} else if v == 0 {
			return errLimitReached
		}
	}

	// Default return
	return nil
}

func (r rateLimiter) transformKey(key string, duration int) string {
	return "ratelimiter_" + key + "_" + strconv.Itoa(duration)
}
{'fraction_non_alphanumeric': 0.10871934604904632, 'fraction_numerical': 0.005449591280653951, 'mean_word_length': 4.077455048409405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1751027', 'n_tokens_mistral': 1362, 'n_tokens_neox': 1260, 'n_words': 359}