<commit_before>package brimstore

import (
	"encoding/binary"
	"fmt"
	"github.com/gholt/brimutil"
	"github.com/spaolacci/murmur3"
	"io"
	"math"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"
)

const CHECKSUM_INTERVAL = 65532

type ReadValue struct {
	KeyHashA uint64
	KeyHashB uint64
	Value    []byte
	Seq      uint64
	ReadChan chan error
	offset   uint32
}

type WriteValue struct {
	KeyHashA    uint64
	KeyHashB    uint64
	Value       []byte
	Seq         uint64
	WrittenChan chan error
}

type Store struct {
	clearableMemBlockChan    chan *memBlock
	clearedMemBlockChan      chan *memBlock
	writeValueChans          []chan *WriteValue
	diskWritableMemBlockChan chan *memBlock
	freeTOCBlockChan         chan []byte
	pendingTOCBlockChan      chan []byte
	memWriterDoneChans       []chan struct{}
	memClearerDoneChans      []chan struct{}
	diskWriterDoneChan       chan struct{}
	tocWriterDoneChan        chan struct{}
	keyLocationBlocks        []keyLocationBlock
	keyLocationBlockLock     sync.RWMutex
	keyLocationMap           *keyLocationMap
	cores                    int
	maxValueSize             int
	allocSize                int
	diskWriterBytes          uint64
	tocWriterBytes           uint64
}

func NewStore() *Store {
	cores := runtime.GOMAXPROCS(0)
	var maxValueSize int
	if env := os.Getenv("BRIMSTORE_MAX_VALUE_SIZE"); env != "" {
		if val, err := strconv.Atoi(env); err == nil {
			maxValueSize = val
		}
	}
	if maxValueSize <= 0 {
		maxValueSize = 4194304
	}
	allocSize := 1 << PowerOfTwoNeeded(uint64(maxValueSize))
	if allocSize < 4096 {
		allocSize = 4096
	}
	s := &Store{
		keyLocationBlocks: make([]keyLocationBlock, 0),
		keyLocationMap:    newKeyLocationMap(),
		cores:             cores,
		allocSize:         allocSize,
		maxValueSize:      maxValueSize,
	}
	return s
}

func (s *Store) MaxValueSize() int {
	return s.maxValueSize
}

func (s *Store) Start() {
	s.clearableMemBlockChan = make(chan *memBlock, s.cores)
	s.clearedMemBlockChan = make(chan *memBlock, s.cores)
	s.writeValueChans = make([]chan *WriteValue, s.cores)<commit_msg>gotta have that extra 4 bytes<commit_after>package brimstore

import (
	"encoding/binary"
	"fmt"
	"github.com/gholt/brimutil"
	"github.com/spaolacci/murmur3"
	"io"
	"math"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"
)

const CHECKSUM_INTERVAL = 65532

type ReadValue struct {
	KeyHashA uint64
	KeyHashB uint64
	Value    []byte
	Seq      uint64
	ReadChan chan error
	offset   uint32
}

type WriteValue struct {
	KeyHashA    uint64
	KeyHashB    uint64
	Value       []byte
	Seq         uint64
	WrittenChan chan error
}

type Store struct {
	clearableMemBlockChan    chan *memBlock
	clearedMemBlockChan      chan *memBlock
	writeValueChans          []chan *WriteValue
	diskWritableMemBlockChan chan *memBlock
	freeTOCBlockChan         chan []byte
	pendingTOCBlockChan      chan []byte
	memWriterDoneChans       []chan struct{}
	memClearerDoneChans      []chan struct{}
	diskWriterDoneChan       chan struct{}
	tocWriterDoneChan        chan struct{}
	keyLocationBlocks        []keyLocationBlock
	keyLocationBlockLock     sync.RWMutex
	keyLocationMap           *keyLocationMap
	cores                    int
	maxValueSize             int
	allocSize                int
	diskWriterBytes          uint64
	tocWriterBytes           uint64
}

func NewStore() *Store {
	cores := runtime.GOMAXPROCS(0)
	var maxValueSize int
	if env := os.Getenv("BRIMSTORE_MAX_VALUE_SIZE"); env != "" {
		if val, err := strconv.Atoi(env); err == nil {
			maxValueSize = val
		}
	}
	if maxValueSize <= 0 {
		maxValueSize = 4194304
	}
	allocSize := 1 << PowerOfTwoNeeded(uint64(maxValueSize+4))
	if allocSize < 4096 {
		allocSize = 4096
	}
	s := &Store{
		keyLocationBlocks: make([]keyLocationBlock, 0),
		keyLocationMap:    newKeyLocationMap(),
		cores:             cores,
		allocSize:         allocSize,
		maxValueSize:      maxValueSize,
	}
	return s
}

func (s *Store) MaxValueSize() int {
	return s.maxValueSize
}

func (s *Store) Start() {
	s.clearableMemBlockChan = make(chan *memBlock, s.cores)
	s.clearedMemBlockChan = make(chan *memBlock, s.cores)
	s.writeValueChans = make([]chan *WriteValue, s.cores)
{'fraction_non_alphanumeric': 0.08869018849685839, 'fraction_numerical': 0.022232962783953602, 'mean_word_length': 3.0578431372549018, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4678213', 'n_tokens_mistral': 1552, 'n_tokens_neox': 1409, 'n_words': 338}