<commit_before>package dskvs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type Data struct {
	Descr string
}

/*
	Boilerplate
*/

func setUp(t *testing.T) *Store {
	store, err := NewStore("./db")
	if err != nil {
		t.Fatalf("Error creating store, %v", err)
	}

	err = store.Load()
	if err != nil {
		t.Fatalf("Error loading store, %v", err)
	}
	return store
}

func tearDown(store *Store, t *testing.T) {
	err := store.Close()
	if err != nil {
		t.Fatalf("Error closing store, %v", err)
	}
	err = os.RemoveAll(store.storagePath)
	if err != nil {
		t.Fatalf("Error deleting storage path, %v", err)
	}
}

func generateData(d Data, t *testing.T) []byte {
	dataBytes, err := json.Marshal(d)
	if err != nil {
		t.Fatal("Error with test data, %v", err)
	}
	return dataBytes
}

/*
	Common checks
*/

func checkGetIsEmpty(store *Store, key string, t *testing.T) {
	_, err := store.Get(key)
	if err == nil {
		t.Fatalf("Expected to receive KeyError but no error")
	}
	switch err := err.(type) {
	case KeyError:
		// Expected
		break
	default:
		t.Fatalf("Expected to receive KeyError but got <%s>", err)
	}
}

/*
	Test cases, single goroutine
*/

func TestCreatingStore(t *testing.T) {
	store := setUp(t)
	tearDown(store, t)
}

func TestSingleOperation(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	key := "artist/daftpunk"
	expected := generateData(Data{"The peak of awesome"}, t)

	err := store.Put(key, expected)
	if err != nil {
		t.Fatalf("Error putting data in", err)
	}

	actual, err := store.Get(key)
	if err != nil {
		t.Fatalf("Error getting data back", err)
	}

	if !bytes.Equal(expected, actual) {
		t.Fatalf("Expected <%s> but was <%s>",
			expected,
			actual)
	}

	err = store.Delete(key)
	if err != nil {
		t.Fatalf("Error deleting data we just put", err)
	}

	checkGetIsEmpty(store, key, t)

}

func TestMultipleOperations(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	coll := "artist"
	baseKey := "daftpunk"

	var key string
	var expected []byte
	var expectedList [][]byte
	var actual []byte
	for i := int(0); i < 10; i++ {
		key = coll + CollKeySep + baseKey + strconv.Itoa(i)
		expected = generateData(Data{"The peak of awesome" + strconv.Itoa(i)}, t)

		err := store.Put(key, expected)
		if err != nil {
			t.Errorf("Error putting data in", err)
		}
		actual, err = store.Get(key)
		if err != nil {
			t.Errorf("Error getting data back", err)
		}

		if !bytes.Equal(expected, actual) {
			t.Errorf("Expected <%s> but was <%s>",
				expected,
				actual)
		}

		expectedList = append(expectedList, expected)
	}

	actuaList, err := store.GetAll(coll)
	if err != nil {
		t.Fatalf("Error getting all keys with coll <%s>", coll)
	}
	if len(actuaList) != len(expectedList) {
		t.Fatalf("Expected to read %d items, but read %d",
			len(expectedList),
			len(actuaList))
	}

	err = store.DeleteAll("artist")
	if err != nil {
		t.Fatal("Error deleting key", err)
	}
	for i := int(0); i < 10; i++ {
		key = coll + CollKeySep + baseKey + strconv.Itoa(i)
		checkGetIsEmpty(store, key, t)
	}
}

/*
	Test cases, multiple goroutine
*/

func TestMultipleGoroutine(t *testing.T) {
	var kvCount int
	if testing.Short() {
		kvCount = 1000
	} else {
		kvCount = 10000
	}
	coll := "games"
	baseKey := "total annihilation #"

	type KeyValue struct {
		Key   string
		Value []byte
	}<commit_msg>Add failing test for persistence artifact reload.<commit_after>package dskvs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type Data struct {
	Descr string
}

/*
	Boilerplate
*/

func setUp(t *testing.T) *Store {
	store, err := NewStore("./db")
	if err != nil {
		t.Fatalf("Error creating store, %v", err)
	}

	err = store.Load()
	if err != nil {
		t.Fatalf("Error loading store, %v", err)
	}
	return store
}

func tearDown(store *Store, t *testing.T) {
	err := store.Close()
	if err != nil {
		t.Fatalf("Error closing store, %v", err)
	}
	err = os.RemoveAll(store.storagePath)
	if err != nil {
		t.Fatalf("Error deleting storage path, %v", err)
	}
}

func generateData(d Data, t *testing.T) []byte {
	dataBytes, err := json.Marshal(d)
	if err != nil {
		t.Fatal("Error with test data, %v", err)
	}
	return dataBytes
}

/*
	Common checks
*/

func checkGetIsEmpty(store *Store, key string, t *testing.T) {
	_, err := store.Get(key)
	if err == nil {
		t.Fatalf("Expected to receive KeyError but no error")
	}
	switch err := err.(type) {
	case KeyError:
		// Expected
		break
	default:
		t.Fatalf("Expected to receive KeyError but got <%s>", err)
	}
}

/*
	Test cases, single goroutine
*/

func TestCreatingStore(t *testing.T) {
	store := setUp(t)
	tearDown(store, t)
}

func TestSingleOperation(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	key := "artist/daftpunk"
	expected := generateData(Data{"The peak of awesome"}, t)

	err := store.Put(key, expected)
	if err != nil {
		t.Fatalf("Error putting data in, %v", err)
	}

	actual, err := store.Get(key)
	if err != nil {
		t.Fatalf("Error getting data back, %v", err)
	}

	if !bytes.Equal(expected, actual) {
		t.Fatalf("Expected <%s> but was <%s>",
			expected,
			actual)
	}

	err = store.Delete(key)
	if err != nil {
		t.Fatalf("Error deleting data we just put, %v", err)
	}

	checkGetIsEmpty(store, key, t)

}

func TestMultipleOperations(t *testing.T) {

	store := setUp(t)
	defer tearDown(store, t)

	coll := "artist"
	baseKey := "daftpunk"

	var key string
	var expected []byte
	var expectedList [][]byte
	var actual []byte
	for i := int(0); i < 10; i++ {
		key = coll + CollKeySep + baseKey + strconv.Itoa(i)
		expected = generateData(Data{"The peak of awesome" + strconv.Itoa(i)}, t)

		err := store.Put(key, expected)
		if err != nil {
			t.Errorf("Error putting data in, %v", err)
		}
		actual, err = store.Get(key)
		if err != nil {
			t.Errorf("Error getting data back, %v", err)
		}

		if !bytes.Equal(expected, actual) {
			t.Errorf("Expected <%s> but was <%s>",
				expected,
				actual)
		}

		expectedList = append(expectedList, expected)
	}

	actuaList, err := store.GetAll(coll)
	if err != nil {
		t.Fatalf("Error getting all keys with coll <%s>", coll)
	}
	if len(actuaList) != len(expectedList) {
		t.Fatalf("Expected to read %d items, but read %d",
			len(expectedList),
			len(actuaList))
	}

	err = store.DeleteAll("artist")
	if err != nil {
		t.Fatal("Error deleting key, %v", err)
	}
	for i := int(0); i < 10; i++ {
		key = coll + CollKeySep + baseKey + strconv.Itoa(i)
		checkGetIsEmpty(store, key, t)
	}
}

func TestStorePersistPutAfterClose(t *testing.T) {
	store := setUp(t)

	key := "artist/the prodigy"
	expected := generateData(Data{"Beyond the peak"}, t)

	if err := store.Put(key, expected); err != nil {
		tearDown(store, t)
		t.Fatalf("Error putting data in, %v", err)
	}

	// Don't use tearDown as it deletes the storage after use
	if err := store.Close(); err != nil {
		t.Fatalf("Error closing store, %v", err)
	}

	otherStore := setUp(t)
	defer tearDown(otherStore, t)

	actual, err := otherStore.Get(key)
	if err != nil {
		t.Fatalf("Error getting data back, %v", err)
	}
	if !bytes.Equal(expected, actual) {
		t.Fatalf("Expected <%s> but was <%s>",
			expected,
			actual)
	}
}

func TestStorePersistDeleteAfterClose(t *testing.T) {
	store := setUp(t)

	key := "artist/the prodigy"
	expected := generateData(Data{"Beyond the peak"}, t)

	if err := store.Put(key, expected); err != nil {
		tearDown(store, t)
		t.Fatalf("Error putting value in, %v", err)
	}

	if err := store.Delete(key); err != nil {
		tearDown(store, t)
		t.Fatalf("Error deleting value, %v", err)
	}

	// Don't use tearDown as it deletes the storage after use
	if err := store.Close(); err != nil {
		t.Fatalf("Error closing store, %v", err)
	}

	otherStore := setUp(t)
	defer tearDown(otherStore, t)

	checkGetIsEmpty(otherStore, key, t)

}

/*
	Test cases, multiple goroutine
*/

func TestMultipleGoroutine(t *testing.T) {
	var kvCount int
	if testing.Short() {
		kvCount = 1000
	} else {
		kvCount = 10000
	}
	coll := "games"
	baseKey := "total annihilation #"

	type KeyValue struct {
		Key   string
		Value []byte
	}
{'fraction_non_alphanumeric': 0.15829453565302623, 'fraction_numerical': 0.0036755697133055622, 'mean_word_length': 3.522437673130194, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2073620', 'n_tokens_mistral': 3302, 'n_tokens_neox': 3106, 'n_words': 871}