<commit_before>package cache

import (
	"bytes"
	"fmt"
	"testing"
	"time"
)

var (
	testData   = []byte(`{"hello":"world"}`)
	defaultExp = time.Duration(1 * time.Second)
)

func TestFetch(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))

	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	item, err := m.Fetch("greeting")
	if err != nil {
		t.Fatal(err)
	}

	if bytes.Compare(item, testData) > 0 {
		t.Fatalf("expected value: 'world' got: '%s'\n", item)
	}
}

func TestFetchExpired(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))

	err := m.Set("greeting", testData, 0)
	if err != nil {
		t.Fatal(err)
	}

	_, err = m.Fetch("greeting")
	if err != ErrNotFound {
		t.Fatalf("item expected to be expired")
	}
}

func TestSet(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))
	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	assertStoredItemCountIs(t, m.(*memcache), 1)
}

func TestSetExisting(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))
	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	testData = []byte(`{"hello":"universe"}`)
	err = m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	assertStoredItemCountIs(t, m.(*memcache), 1)
}

func TestSetOverItemLimit(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))
	for i := 0; i < 20; i++ {
		key := fmt.Sprint(i)
		value := []byte(key)
		err := m.Set(key, value, defaultExp)
		if err != nil {
			t.Fatal(err)
		}
	}

	// check count
	assertStoredItemCountIs(t, m.(*memcache), 10)

	// check content
	for i := 10; i < 20; i++ {
		key := fmt.Sprint(i)
		expected := []byte(key)
		v, err := m.Fetch(key)
		if err != nil || bytes.Compare(v, expected) > 0 {
			t.Fatal(err)
		}
	}
}

func TestSetOverBytesLimit(t *testing.T) {
	m := NewMemCache(MaxBytesLimit(8))

	// total of 9 bytes
	testItems := [][]byte{
		[]byte("123"),
		[]byte("456"),
		[]byte("789"),
	}

	for i, item := range testItems {
		key := fmt.Sprint(i)
		err := m.Set(key, item, defaultExp)
		if err != nil {
			t.Fatal(err)
		}
	}

	assertStoredItemCountIs(t, m.(*memcache), 2)
}

func TestDelete(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))

	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	m.Delete("greeting")
	assertStoredItemCountIs(t, m.(*memcache), 0)
	if m.(*memcache).currentBytes != 0 {
		t.Fatal("currentBytes expected to be 0")
	}
}

func assertStoredItemCountIs(t *testing.T, m *memcache, expected uint) {
	msg := "expected %v items to be stored, got %v\n"

	if l := uint(m.list.Len()); l != expected {
		t.Fatalf(msg, expected, l)
	}

	if l := uint(len(m.items)); l != expected {
		t.Fatalf(msg, expected, l)
	}
}<commit_msg>update test<commit_after>package cache

import (
	"bytes"
	"fmt"
	"testing"
	"time"
)

var (
	testData   = []byte(`{"hello":"world"}`)
	defaultExp = time.Duration(1 * time.Second)
)

func TestFetch(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))

	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	item, err := m.Fetch("greeting")
	if err != nil {
		t.Fatal(err)
	}

	if bytes.Compare(item, testData) > 0 {
		t.Fatalf("expected value: 'world' got: '%s'\n", item)
	}
}

func TestFetchExpired(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))

	err := m.Set("greeting", testData, 0)
	if err != nil {
		t.Fatal(err)
	}

	_, err = m.Fetch("greeting")
	if err != ErrNotFound {
		t.Fatalf("item expected to be expired")
	}
}

func TestSet(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))
	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	assertStoredItemCountIs(t, m, 1)
}

func TestSetExisting(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))
	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	testData = []byte(`{"hello":"universe"}`)
	err = m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	assertStoredItemCountIs(t, m, 1)
}

func TestSetOverItemLimit(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))
	for i := 0; i < 20; i++ {
		key := fmt.Sprint(i)
		value := []byte(key)
		err := m.Set(key, value, defaultExp)
		if err != nil {
			t.Fatal(err)
		}
	}

	// check count
	assertStoredItemCountIs(t, m, 10)

	// check content
	for i := 10; i < 20; i++ {
		key := fmt.Sprint(i)
		expected := []byte(key)
		v, err := m.Fetch(key)
		if err != nil || bytes.Compare(v, expected) > 0 {
			t.Fatal(err)
		}
	}
}

func TestSetOverBytesLimit(t *testing.T) {
	m := NewMemCache(MaxBytesLimit(8))

	// total of 9 bytes
	testItems := [][]byte{
		[]byte("123"),
		[]byte("456"),
		[]byte("789"),
	}

	for i, item := range testItems {
		key := fmt.Sprint(i)
		err := m.Set(key, item, defaultExp)
		if err != nil {
			t.Fatal(err)
		}
	}

	assertStoredItemCountIs(t, m, 2)
}

func TestDelete(t *testing.T) {
	m := NewMemCache(MaxItemLimit(10))

	err := m.Set("greeting", testData, defaultExp)
	if err != nil {
		t.Fatal(err)
	}

	m.Delete("greeting")
	assertStoredItemCountIs(t, m, 0)
	if m.currentBytes != 0 {
		t.Fatal("currentBytes expected to be 0")
	}
}

func assertStoredItemCountIs(t *testing.T, m *MemCache, expected uint) {
	msg := "expected %v items to be stored, got %v\n"

	if l := uint(m.list.Len()); l != expected {
		t.Fatalf(msg, expected, l)
	}

	if l := uint(len(m.items)); l != expected {
		t.Fatalf(msg, expected, l)
	}
}
{'fraction_non_alphanumeric': 0.1893491124260355, 'fraction_numerical': 0.015532544378698224, 'mean_word_length': 3.6790657439446366, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2521235', 'n_tokens_mistral': 2400, 'n_tokens_neox': 2232, 'n_words': 495}