<commit_before>package cache

import (
	"sync"
	"time"
)

var expChan = make(chan expireNotifier, 1000) // TODO: configurable

type Getter interface {
	Get(key interface{}) (interface{}, bool)
}

type expireNotifier struct {
	key    interface{}
	expire time.Time
}

type cache struct {
	items map[interface{}]cacheItem
	mutex sync.RWMutex
}

type cacheItem struct {
	expire *time.Timer
	value  interface{}
}

func New(name string) *cache {
	c := &cache{
		items: make(map[interface{}]cacheItem),
		mutex: sync.RWMutex{},
	}
	go func() {
		for {
			select {
			case exp := <-expChan:
				c.mutex.Lock()
				delete(c.items, exp.key)
				c.mutex.Unlock()
			default:
			}
		}
	}()
	return c
}

func (c *cache) Set(key interface{}, item interface{}, expire time.Duration) error {
	timer := time.NewTimer(expire)
	c.mutex.Lock()
	c.items[key] = cacheItem{
		value:  item,
		expire: timer,<commit_msg>define interface<commit_after>package cache

import (
	"sync"
	"time"
)

var expChan = make(chan expireNotifier, 1000) // TODO: configurable

type Getter interface {
	Get(key interface{}) (interface{}, bool)
}

type expireNotifier struct {
	key    interface{}
	expire time.Time
}

type Cache interface {
	Getter
	Set(key interface{}, item interface{}, expire time.Duration) error
	SetExpire(key interface{}, expire time.Duration)
}

type cache struct {
	items map[interface{}]cacheItem
	mutex sync.RWMutex
}

var _ Cache = (*cache)(nil)

type cacheItem struct {
	expire *time.Timer
	value  interface{}
}

func New(name string) Cache {
	c := &cache{
		items: make(map[interface{}]cacheItem),
		mutex: sync.RWMutex{},
	}
	go func() {
		for {
			select {
			case exp := <-expChan:
				c.mutex.Lock()
				delete(c.items, exp.key)
				c.mutex.Unlock()
			default:
			}
		}
	}()
	return c
}

// TODO: implement MSet (multi set) for performance
func (c *cache) Set(key interface{}, item interface{}, expire time.Duration) error {
	timer := time.NewTimer(expire)
	c.mutex.Lock()
	c.items[key] = cacheItem{
		value:  item,
		expire: timer,
{'fraction_non_alphanumeric': 0.1397530864197531, 'fraction_numerical': 0.003950617283950617, 'mean_word_length': 3.767058823529412, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3324652', 'n_tokens_mistral': 778, 'n_tokens_neox': 718, 'n_words': 191}