<commit_before>package db

import (
	"encoding/json"
	"math/rand"
	"os"
	"strconv"
	"testing"
	"time"
)

const (
	COL_BENCH_SIZE = 1000000 // Number of documents made available for collection benchmark
)

func TestInsertRead(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}
	docs := []string{`{"a": 1}`, `{"b": 2}`}
	var jsonDoc [2]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])

	ids := [2]uint64{}
	if ids[0], err = col.Insert(jsonDoc[0]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if ids[1], err = col.Insert(jsonDoc[1]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if col.Read(ids[0]).(map[string]interface{})[string('a')].(float64) != 1.0 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[0]))
	}
	if col.Read(ids[1]).(map[string]interface{})[string('b')].(float64) != 2.0 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[1]))
	}
	keys, vals := col.IdIndex.GetAll()
	if !(keys[0] == ids[0] && ids[0] == vals[0] && keys[1] == ids[1] && ids[1] == vals[1] && len(keys) == 2 && len(vals) == 2) {
		t.Errorf("ID Index was not set correctly")
	}
	col.Close()
}

func TestInsertUpdateRead(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}

	docs := []string{`{"a": 1}`, `{"b": 2}`}
	var jsonDoc [2]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])

	updatedDocs := []string{`{"a": 2}`, `{"b": "abcdefghijklmnopqrstuvwxyz"}`}
	var updatedJsonDoc [2]interface{}
	json.Unmarshal([]byte(updatedDocs[0]), &updatedJsonDoc[0])
	json.Unmarshal([]byte(updatedDocs[1]), &updatedJsonDoc[1])

	ids := [2]uint64{}
	if ids[0], err = col.Insert(jsonDoc[0]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if ids[1], err = col.Insert(jsonDoc[1]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}

	if ids[0], err = col.Update(ids[0], updatedJsonDoc[0]); err != nil {
		t.Errorf("Failed to update: %v", err)
	}
	if ids[1], err = col.Update(ids[1], updatedJsonDoc[1]); err != nil {
		t.Errorf("Failed to update: %v", err)
	}

	if col.Read(ids[0]).(map[string]interface{})[string('a')].(float64) != 2.0 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[0]))
	}
	if col.Read(ids[1]).(map[string]interface{})[string('b')].(string) != string("abcdefghijklmnopqrstuvwxyz") {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[1]))
	}
	keys, vals := col.IdIndex.GetAll()
	if !(keys[0] == ids[0] && ids[0] == vals[0] && keys[1] == ids[1] && ids[1] == vals[1] && len(keys) == 2 && len(vals) == 2) {
		t.Errorf("ID Index was not set correctly")
	}
	col.Close()
}

func TestInsertDeleteRead(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}
	docs := []string{`{"a": 1}`, `{"b": 2}`}
	var jsonDoc [2]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])
	ids := [2]uint64{}
	if ids[0], err = col.Insert(jsonDoc[0]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if ids[1], err = col.Insert(jsonDoc[1]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	col.Delete(ids[0])
	if col.Read(ids[0]) != nil {
		t.Errorf("Did not delete document, still read %v", col.Read(ids[0]))
	}
	if col.Read(ids[1]).(map[string]interface{})[string('b')].(float64) != 2 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[1]))
	}
	keys, vals := col.IdIndex.GetAll()
	if !(keys[0] == ids[1] && ids[1] == vals[0] && len(keys) == 1 && len(vals) == 1) {
		t.Errorf("ID Index was not set correctly")
	}
	col.Close()
}

func TestIndex(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}
	docs := []string{`{"a": {"b": {"c": 1}}, "d": 0}`, `{"a": {"b": {"c": 2}}, "d": 0}`, `{"a": {"b": {"c": 3}}, "d": 0}`, `{"a": {"b": {"c": 4}}, "d": 0}`}
	var jsonDoc [4]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])
	json.Unmarshal([]byte(docs[2]), &jsonDoc[2])
	json.Unmarshal([]byte(docs[3]), &jsonDoc[3])
	ids := [3]uint64{}
	ids[0], _ = col.Insert(jsonDoc[0])
	if err = col.Index([]string{"a", "b", "c"}); err != nil {
		t.Error(err)
		return
	}
	if err = col.Index([]string{"d"}); err != nil {
		t.Error(err)
		return
	}
	for _, first := range col.StrHT {
		keys, vals := first.GetAll()
		if !(len(keys) == 1 && len(vals) == 1 && vals[0] == ids[0]) {
			t.Errorf("Did not index existing document, got %v, %v", keys, vals)
		}
		break
	}
	ids[1], _ = col.Insert(jsonDoc[1])
	ids[2], _ = col.Insert(jsonDoc[2])
	ids[2], _ = col.Update(ids[2], jsonDoc[3])
	col.Delete(ids[1])
	// jsonDoc[0,3], ids[0, 2] are the ones left
	index1 := col.StrHT["a,b,c"]
	index2 := col.StrHT["d"]
	k0, v0 := index2.Get(StrHash(0), 0, func(k, v uint64) bool {
		return true
	})
	k1, v1 := index1.Get(StrHash(1), 0, func(k, v uint64) bool {
		return true
	})
	k4, v4 := index1.Get(StrHash(4), 0, func(k, v uint64) bool {
		return true
	})
	if !(len(k0) == 2 && len(v0) == 2 && k0[0] == StrHash(0) && v0[0] == ids[0] && k0[1] == StrHash(0) && v0[1] == ids[2]) {
		t.Errorf("Index fault, %v, %v", k0, v0)
	}
	if !(len(k1) == 1 && len(v1) == 1 && k1[0] == StrHash(1) && v1[0] == ids[0]) {
		t.Errorf("Index fault, %v, %v", k1, v1)
	}
	if !(len(k4) == 1 && len(v4) == 1 && k4[0] == StrHash(4) && v4[0] == ids[2]) {
		t.Errorf("Index fault, %v, %v", k4, v4)
	}
	// now remove a,b,c index
	if err = col.Unindex([]string{"a", "b", "c"}); err != nil {
		t.Error(err)
		return
	}
	if _, ok := col.StrHT["a,b,c"]; ok {
		t.Error("did not delete index")
	}
	if _, ok := col.StrIC["a,b,c"]; ok {
		t.Error("did not delete index")
	}
	newID, err := col.Insert(jsonDoc[0])
	if err != nil {
		t.Error(err)
	}
	k0, v0 = col.StrHT["d"].Get(StrHash(0), 0, func(k, v uint64) bool {
		return true<commit_msg>create two previously missed index cases<commit_after>package db

import (
	"encoding/json"
	"math/rand"
	"os"
	"strconv"
	"testing"
	"time"
)

const (
	COL_BENCH_SIZE = 1000000 // Number of documents made available for collection benchmark
)

func TestInsertRead(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}
	docs := []string{`{"a": 1}`, `{"b": 2}`}
	var jsonDoc [2]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])

	ids := [2]uint64{}
	if ids[0], err = col.Insert(jsonDoc[0]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if ids[1], err = col.Insert(jsonDoc[1]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if col.Read(ids[0]).(map[string]interface{})[string('a')].(float64) != 1.0 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[0]))
	}
	if col.Read(ids[1]).(map[string]interface{})[string('b')].(float64) != 2.0 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[1]))
	}
	keys, vals := col.IdIndex.GetAll()
	if !(keys[0] == ids[0] && ids[0] == vals[0] && keys[1] == ids[1] && ids[1] == vals[1] && len(keys) == 2 && len(vals) == 2) {
		t.Errorf("ID Index was not set correctly")
	}
	col.Close()
}

func TestInsertUpdateRead(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}

	docs := []string{`{"a": 1}`, `{"b": 2}`}
	var jsonDoc [2]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])

	updatedDocs := []string{`{"a": 2}`, `{"b": "abcdefghijklmnopqrstuvwxyz"}`}
	var updatedJsonDoc [2]interface{}
	json.Unmarshal([]byte(updatedDocs[0]), &updatedJsonDoc[0])
	json.Unmarshal([]byte(updatedDocs[1]), &updatedJsonDoc[1])

	ids := [2]uint64{}
	if ids[0], err = col.Insert(jsonDoc[0]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if ids[1], err = col.Insert(jsonDoc[1]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}

	if ids[0], err = col.Update(ids[0], updatedJsonDoc[0]); err != nil {
		t.Errorf("Failed to update: %v", err)
	}
	if ids[1], err = col.Update(ids[1], updatedJsonDoc[1]); err != nil {
		t.Errorf("Failed to update: %v", err)
	}

	if col.Read(ids[0]).(map[string]interface{})[string('a')].(float64) != 2.0 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[0]))
	}
	if col.Read(ids[1]).(map[string]interface{})[string('b')].(string) != string("abcdefghijklmnopqrstuvwxyz") {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[1]))
	}
	keys, vals := col.IdIndex.GetAll()
	if !(keys[0] == ids[0] && ids[0] == vals[0] && keys[1] == ids[1] && ids[1] == vals[1] && len(keys) == 2 && len(vals) == 2) {
		t.Errorf("ID Index was not set correctly")
	}
	col.Close()
}

func TestInsertDeleteRead(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}
	docs := []string{`{"a": 1}`, `{"b": 2}`}
	var jsonDoc [2]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])
	ids := [2]uint64{}
	if ids[0], err = col.Insert(jsonDoc[0]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	if ids[1], err = col.Insert(jsonDoc[1]); err != nil {
		t.Errorf("Failed to insert: %v", err)
	}
	col.Delete(ids[0])
	if col.Read(ids[0]) != nil {
		t.Errorf("Did not delete document, still read %v", col.Read(ids[0]))
	}
	if col.Read(ids[1]).(map[string]interface{})[string('b')].(float64) != 2 {
		t.Errorf("Failed to read back document, got %v", col.Read(ids[1]))
	}
	keys, vals := col.IdIndex.GetAll()
	if !(keys[0] == ids[1] && ids[1] == vals[0] && len(keys) == 1 && len(vals) == 1) {
		t.Errorf("ID Index was not set correctly")
	}
	col.Close()
}

func TestIndex(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Errorf("Failed to open: %v", err)
		return
	}
	docs := []string{
		`{"a": {"b": {"c": 1}}, "d": 0}`,
		`{"a": {"b": {"c": 2}}, "d": 0}`,
		`{"a": {"b": {"c": 3}}, "d": 0}`,
		`{"a": {"b": {"c": 4}}, "d": [0, 9]}`,
		`{"a": {"b": {"c": null}}, "d": null}`}
	var jsonDoc [4]interface{}
	json.Unmarshal([]byte(docs[0]), &jsonDoc[0])
	json.Unmarshal([]byte(docs[1]), &jsonDoc[1])
	json.Unmarshal([]byte(docs[2]), &jsonDoc[2])
	json.Unmarshal([]byte(docs[3]), &jsonDoc[3])
	ids := [3]uint64{}
	ids[0], _ = col.Insert(jsonDoc[0])
	if err = col.Index([]string{"a", "b", "c"}); err != nil {
		t.Error(err)
		return
	}
	if err = col.Index([]string{"d"}); err != nil {
		t.Error(err)
		return
	}
	for _, first := range col.StrHT {
		keys, vals := first.GetAll()
		if !(len(keys) == 1 && len(vals) == 1 && vals[0] == ids[0]) {
			t.Errorf("Did not index existing document, got %v, %v", keys, vals)
		}
		break
	}
	ids[1], _ = col.Insert(jsonDoc[1])
	ids[2], _ = col.Insert(jsonDoc[2])
	ids[2], _ = col.Update(ids[2], jsonDoc[3])
	col.Delete(ids[1])
	// jsonDoc[0,3], ids[0, 2] are the ones left
	index1 := col.StrHT["a,b,c"]
	index2 := col.StrHT["d"]
	// d index
	k0, v0 := index2.Get(StrHash(0), 0, func(k, v uint64) bool {
		return true
	})
	k9, v9 := index2.Get(StrHash(9), 0, func(k, v uint64) bool {
		return true
	})
	if !(len(k0) == 2 && len(v0) == 2 && k0[0] == StrHash(0) && v0[0] == ids[0] && k0[1] == StrHash(0) && v0[1] == ids[2]) {
		t.Errorf("Index fault on key 0, %v, %v", k0, v0)
	}
	if !(len(k9) == 1 && len(v9) == 1 && k9[0] == StrHash(9) && v9[0] == ids[2]) {
		t.Errorf("Index fault on key 9, %v, %v", k9, v9)
	}
	// abc index
	k1, v1 := index1.Get(StrHash(1), 0, func(k, v uint64) bool {
		return true
	})
	k4, v4 := index1.Get(StrHash(4), 0, func(k, v uint64) bool {
		return true
	})
	if !(len(k1) == 1 && len(v1) == 1 && k1[0] == StrHash(1) && v1[0] == ids[0]) {
		t.Errorf("Index fault, %v, %v", k1, v1)
	}
	if !(len(k4) == 1 && len(v4) == 1 && k4[0] == StrHash(4) && v4[0] == ids[2]) {
		t.Errorf("Index fault, %v, %v", k4, v4)
	}
	// now remove a,b,c index
	if err = col.Unindex([]string{"a", "b", "c"}); err != nil {
		t.Error(err)
		return
	}
	if _, ok := col.StrHT["a,b,c"]; ok {
		t.Error("did not delete index")
	}
	if _, ok := col.StrIC["a,b,c"]; ok {
		t.Error("did not delete index")
	}
	newID, err := col.Insert(jsonDoc[0])
	if err != nil {
		t.Error(err)
	}
	k0, v0 = col.StrHT["d"].Get(StrHash(0), 0, func(k, v uint64) bool {
		return true
{'fraction_non_alphanumeric': 0.2479095695261691, 'fraction_numerical': 0.03654382161659957, 'mean_word_length': 4.214775938635446, 'pattern_counts': {'":': 52, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2483668', 'n_tokens_mistral': 5814, 'n_tokens_neox': 5564, 'n_words': 1249}