<commit_before>package tokens

import (
	"github.com/dustin/go-humanize"
	"math/rand"
	"sync"
	"time"
	"log"
)

type Token struct {
	ValidTo         time.Time
	ExpiresReadable string
	Id              string
	VerifiedCount   int
}

type Tokens struct {
	sync.RWMutex
	tokens []Token
}

func Init() Tokens {
	t := Tokens{}
	return t
}

func (t *Tokens) Generate() string {
	t.Cleanup()

	var token Token
	token.Id = RandomString(8)
	now := time.Now().UTC()
	token.ValidTo = now.Add(1 * time.Minute)

	t.Lock()
	t.tokens = append([]Token{token}, t.tokens...)
	t.Unlock()
	return token.Id
}

func (t *Tokens) Verify(token string) bool {
	t.Lock()
	found := false
	now := time.Now().UTC()
	for i, data := range t.tokens {
		if data.Id == token {
			if now.Before(data.ValidTo) {
				t.tokens[i].VerifiedCount = t.tokens[i].VerifiedCount + 1
				found = true
			}
		}
	}
	t.Unlock()
	return found
}

func (t *Tokens) removeToken(token string) {
	t.Lock()
	for i, data := range t.tokens {
		if data.Id == token {<commit_msg>Tokens are valid for 5 minutes<commit_after>package tokens

import (
	"github.com/dustin/go-humanize"
	"math/rand"
	"sync"
	"time"
	"log"
)

type Token struct {
	ValidTo         time.Time
	ExpiresReadable string
	Id              string
	VerifiedCount   int
}

type Tokens struct {
	sync.RWMutex
	tokens []Token
}

func Init() Tokens {
	t := Tokens{}
	return t
}

func (t *Tokens) Generate() string {
	t.Cleanup()

	var token Token
	token.Id = RandomString(8)
	now := time.Now().UTC()
	token.ValidTo = now.Add(5 * time.Minute)

	t.Lock()
	t.tokens = append([]Token{token}, t.tokens...)
	t.Unlock()
	return token.Id
}

func (t *Tokens) Verify(token string) bool {
	t.Lock()
	found := false
	now := time.Now().UTC()
	for i, data := range t.tokens {
		if data.Id == token {
			if now.Before(data.ValidTo) {
				t.tokens[i].VerifiedCount = t.tokens[i].VerifiedCount + 1
				found = true
			}
		}
	}
	t.Unlock()
	return found
}

func (t *Tokens) removeToken(token string) {
	t.Lock()
	for i, data := range t.tokens {
		if data.Id == token {
{'fraction_non_alphanumeric': 0.15017064846416384, 'fraction_numerical': 0.0034129692832764505, 'mean_word_length': 3.0078125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '307980', 'n_tokens_mistral': 819, 'n_tokens_neox': 796, 'n_words': 201}