<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"
	"runtime"
	"sync"
	"time"

	"github.com/gholt/brimstore/valuesstore"
	"github.com/gholt/brimutil"
)

func main() {
	seed := int64(1)
	bytesPerValue := 128
	targetBytes := 4 * 1024 * 1024 * 1024
	cores := runtime.GOMAXPROCS(0)
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
		cores = runtime.GOMAXPROCS(0)
	}
	clients := cores * cores
	valuesPerClient := targetBytes / bytesPerValue / clients
	totalValues := clients * valuesPerClient
	totalValueBytes := totalValues * bytesPerValue
	value := make([]byte, bytesPerValue)
	brimutil.NewSeededScrambled(seed).Read(value)
	keys := createKeys(seed, clients, valuesPerClient)
	keys2 := createKeys(seed+int64(totalValues), clients, valuesPerClient)

	fmt.Println(cores, "cores")
	fmt.Println(bytesPerValue, "bytes per value")
	fmt.Println(clients, "clients")
	fmt.Println(valuesPerClient, "values per client")
	fmt.Printf("%d %0.2fm total values\n", totalValues, float64(totalValues)/1000000)
	fmt.Printf("%d %0.2fG total value bytes\n", totalValueBytes, float64(totalValueBytes)/1024/1024/1024)
	var st runtime.MemStats
	runtime.ReadMemStats(&st)
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)

	fmt.Println()
	start := time.Now()
	vs := valuesstore.NewValuesStore(nil)
	dur := time.Now().Sub(start)
	fmt.Println(dur, "to start ValuesStore")
	runtime.ReadMemStats(&st)
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	writeValues(vs, keys, value, clients, valuesPerClient)
	dur = time.Now().Sub(start)
	fmt.Printf("%s %.0f/s %0.2fG/s to add %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), float64(totalValueBytes)/(float64(dur)/float64(time.Second))/1024/1024/1024, totalValues)
	runtime.ReadMemStats(&st)
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	m := readValues(vs, [][][]byte{keys}, value, clients, valuesPerClient)
	dur = time.Now().Sub(start)
	fmt.Printf("%s %.0f/s %dns each, to read %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), int(dur)/totalValues, totalValues)
	if m != 0 {
		fmt.Println(m, "MISSING KEYS!")
	}
	runtime.ReadMemStats(&st)
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	readChan, writeChan := readAndWriteValues(vs, keys, keys2, value, clients, valuesPerClient)
	readDone := false
	writeDone := false
	for !readDone || !writeDone {
		if readDone {
			<-writeChan
			dur = time.Now().Sub(start)
			fmt.Printf("%s %.0f/s %0.2fG/s to add %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), float64(totalValueBytes)/(float64(dur)/float64(time.Second))/1024/1024/1024, totalValues)
			writeDone = true
		} else if writeDone {
			m = <-readChan
			dur = time.Now().Sub(start)
			fmt.Printf("%s %.0f/s %dns each, to read %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), int(dur)/totalValues, totalValues)
			if m != 0 {
				fmt.Println(m, "MISSING KEYS!")
			}
			readDone = true
		} else {
			select {
			case m = <-readChan:
				dur = time.Now().Sub(start)
				fmt.Printf("%s %.0f/s %dns each, to read %d values while writing other values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), int(dur)/totalValues, totalValues)
				if m != 0 {
					fmt.Println(m, "MISSING KEYS!")
				}
				readDone = true
			case <-writeChan:
				dur = time.Now().Sub(start)
				fmt.Printf("%s %.0f/s %0.2fG/s to add %d values while reading other values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), float64(totalValueBytes)/(float64(dur)/float64(time.Second))/1024/1024/1024, totalValues)
				writeDone = true
			}
		}
	}
	runtime.ReadMemStats(&st)
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	m = readValues(vs, [][][]byte{keys, keys2}, value, clients, valuesPerClient)
	dur = time.Now().Sub(start)
	fmt.Printf("%s %.0f/s %dns each, to read %d values\n", dur, float64(totalValues*2)/(float64(dur)/float64(time.Second)), int(dur)/(totalValues*2), totalValues*2)
	if m != 0 {
		fmt.Println(m, "MISSING KEYS!")
	}
	runtime.ReadMemStats(&st)
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	vs.Close()
	dur = time.Now().Sub(start)
	fmt.Println(dur, "to close ValuesStore")
	runtime.ReadMemStats(&st)
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)
}

func createKeys(seed int64, clients int, valuesPerClient int) [][]byte {
	wg := &sync.WaitGroup{}
	keys := make([][]byte, clients)
	wg.Add(clients)
	for i := 0; i < clients; i++ {
		keys[i] = make([]byte, valuesPerClient*16)
		go func(i int) {
			brimutil.NewSeededScrambled(seed + int64(valuesPerClient*i)).Read(keys[i])<commit_msg>alloc deltas for test exe<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"
	"runtime"
	"sync"
	"time"

	"github.com/gholt/brimstore/valuesstore"
	"github.com/gholt/brimutil"
)

func main() {
	seed := int64(1)
	bytesPerValue := 128
	targetBytes := 4 * 1024 * 1024 * 1024
	cores := runtime.GOMAXPROCS(0)
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
		cores = runtime.GOMAXPROCS(0)
	}
	clients := cores * cores
	valuesPerClient := targetBytes / bytesPerValue / clients
	totalValues := clients * valuesPerClient
	totalValueBytes := totalValues * bytesPerValue
	value := make([]byte, bytesPerValue)
	brimutil.NewSeededScrambled(seed).Read(value)
	keys := createKeys(seed, clients, valuesPerClient)
	keys2 := createKeys(seed+int64(totalValues), clients, valuesPerClient)

	fmt.Println(cores, "cores")
	fmt.Println(bytesPerValue, "bytes per value")
	fmt.Println(clients, "clients")
	fmt.Println(valuesPerClient, "values per client")
	fmt.Printf("%d %0.2fm total values\n", totalValues, float64(totalValues)/1000000)
	fmt.Printf("%d %0.2fG total value bytes\n", totalValueBytes, float64(totalValueBytes)/1024/1024/1024)
	var st runtime.MemStats
	runtime.ReadMemStats(&st)
	lastAlloc := st.TotalAlloc
	fmt.Printf("%0.2fG total alloc\n", float64(st.TotalAlloc)/1024/1024/1024)

	fmt.Println()
	start := time.Now()
	vs := valuesstore.NewValuesStore(nil)
	dur := time.Now().Sub(start)
	fmt.Println(dur, "to start ValuesStore")
	runtime.ReadMemStats(&st)
	deltaAlloc := st.TotalAlloc - lastAlloc
	lastAlloc = st.TotalAlloc
	fmt.Printf("%0.2fG total alloc, %0.2fG delta\n", float64(st.TotalAlloc)/1024/1024/1024, float64(deltaAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	writeValues(vs, keys, value, clients, valuesPerClient)
	dur = time.Now().Sub(start)
	fmt.Printf("%s %.0f/s %0.2fG/s to add %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), float64(totalValueBytes)/(float64(dur)/float64(time.Second))/1024/1024/1024, totalValues)
	runtime.ReadMemStats(&st)
	deltaAlloc = st.TotalAlloc - lastAlloc
	lastAlloc = st.TotalAlloc
	fmt.Printf("%0.2fG total alloc, %0.2fG delta\n", float64(st.TotalAlloc)/1024/1024/1024, float64(deltaAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	m := readValues(vs, [][][]byte{keys}, value, clients, valuesPerClient)
	dur = time.Now().Sub(start)
	fmt.Printf("%s %.0f/s %dns each, to read %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), int(dur)/totalValues, totalValues)
	if m != 0 {
		fmt.Println(m, "MISSING KEYS!")
	}
	runtime.ReadMemStats(&st)
	deltaAlloc = st.TotalAlloc - lastAlloc
	lastAlloc = st.TotalAlloc
	fmt.Printf("%0.2fG total alloc, %0.2fG delta\n", float64(st.TotalAlloc)/1024/1024/1024, float64(deltaAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	readChan, writeChan := readAndWriteValues(vs, keys, keys2, value, clients, valuesPerClient)
	readDone := false
	writeDone := false
	for !readDone || !writeDone {
		if readDone {
			<-writeChan
			dur = time.Now().Sub(start)
			fmt.Printf("%s %.0f/s %0.2fG/s to add %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), float64(totalValueBytes)/(float64(dur)/float64(time.Second))/1024/1024/1024, totalValues)
			writeDone = true
		} else if writeDone {
			m = <-readChan
			dur = time.Now().Sub(start)
			fmt.Printf("%s %.0f/s %dns each, to read %d values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), int(dur)/totalValues, totalValues)
			if m != 0 {
				fmt.Println(m, "MISSING KEYS!")
			}
			readDone = true
		} else {
			select {
			case m = <-readChan:
				dur = time.Now().Sub(start)
				fmt.Printf("%s %.0f/s %dns each, to read %d values while writing other values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), int(dur)/totalValues, totalValues)
				if m != 0 {
					fmt.Println(m, "MISSING KEYS!")
				}
				readDone = true
			case <-writeChan:
				dur = time.Now().Sub(start)
				fmt.Printf("%s %.0f/s %0.2fG/s to add %d values while reading other values\n", dur, float64(totalValues)/(float64(dur)/float64(time.Second)), float64(totalValueBytes)/(float64(dur)/float64(time.Second))/1024/1024/1024, totalValues)
				writeDone = true
			}
		}
	}
	runtime.ReadMemStats(&st)
	deltaAlloc = st.TotalAlloc - lastAlloc
	lastAlloc = st.TotalAlloc
	fmt.Printf("%0.2fG total alloc, %0.2fG delta\n", float64(st.TotalAlloc)/1024/1024/1024, float64(deltaAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	m = readValues(vs, [][][]byte{keys, keys2}, value, clients, valuesPerClient)
	dur = time.Now().Sub(start)
	fmt.Printf("%s %.0f/s %dns each, to read %d values\n", dur, float64(totalValues*2)/(float64(dur)/float64(time.Second)), int(dur)/(totalValues*2), totalValues*2)
	if m != 0 {
		fmt.Println(m, "MISSING KEYS!")
	}
	runtime.ReadMemStats(&st)
	deltaAlloc = st.TotalAlloc - lastAlloc
	lastAlloc = st.TotalAlloc
	fmt.Printf("%0.2fG total alloc, %0.2fG delta\n", float64(st.TotalAlloc)/1024/1024/1024, float64(deltaAlloc)/1024/1024/1024)

	fmt.Println()
	start = time.Now()
	vs.Close()
	dur = time.Now().Sub(start)
	fmt.Println(dur, "to close ValuesStore")
	runtime.ReadMemStats(&st)
	deltaAlloc = st.TotalAlloc - lastAlloc
	lastAlloc = st.TotalAlloc
	fmt.Printf("%0.2fG total alloc, %0.2fG delta\n", float64(st.TotalAlloc)/1024/1024/1024, float64(deltaAlloc)/1024/1024/1024)
}

func createKeys(seed int64, clients int, valuesPerClient int) [][]byte {
	wg := &sync.WaitGroup{}
	keys := make([][]byte, clients)
	wg.Add(clients)
	for i := 0; i < clients; i++ {
		keys[i] = make([]byte, valuesPerClient*16)
		go func(i int) {
			brimutil.NewSeededScrambled(seed + int64(valuesPerClient*i)).Read(keys[i])
{'fraction_non_alphanumeric': 0.17136435995879765, 'fraction_numerical': 0.06292724037831257, 'mean_word_length': 4.772972972972973, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7174205', 'n_tokens_mistral': 4633, 'n_tokens_neox': 4032, 'n_words': 777}