<commit_before>package db

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"syscall"
)

type returnChanMessage struct {
	err  error
	json string
}

type readChanMessage struct {
	key        string
	returnChan chan returnChanMessage
}

type db struct {
	data      []byte
	dataMap   map[string]string
	log       *log
	fd        int
	filename  string
	file      *os.File
	writeChan chan map[string]string
	readChan  chan readChanMessage
}

func (db *db) load() {
	err := json.Unmarshal(db.data, &db.dataMap)
	if err != nil {
		fmt.Println("Error unmarshalling initial data into map: ", err)
	}
	fmt.Println(db.dataMap)
}

func (db *db) mmap(size int) {
	fmt.Println("mmapping db file: ", size)
	data, err := syscall.Mmap(db.fd, 0, size, syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		fmt.Println("Error mmapping: ", err)
	}
	db.data = data
}

func (db *db) resize(size int) {
	fmt.Println("Resizing db file: ", size)
	err := syscall.Ftruncate(db.fd, int64(size))
	if err != nil {
		fmt.Println("Error resizing: ", err)
	}
}

func (db *db) open() {
	fmt.Println("Getting db file descriptor")
	f, err := os.OpenFile(db.filename, os.O_CREATE|os.O_RDWR, 0)
	if err != nil {
		fmt.Println("Could not open file: ", err)
	}
	db.fd = int(f.Fd())
	db.file = f
}

func (db *db) extend(size int) {
	db.file.Close()
	db.open()
	db.resize(size)
	db.mmap(size)
}

func (db *db) listener() {
	for {
		select {
		case writeReq := <-db.writeChan:
			key := writeReq["key"]
			value := writeReq["value"]
			db.dataMap[key] = value
			b, err := json.Marshal(db.dataMap)
			if err != nil {
				fmt.Println("Error marshalling db: ", err)
			}
			if len(b) > len(db.data) {
				db.extend(len(b))
			}
			copy(db.data, b)
			s := "SET " + key + ": " + value + "\n"
			b = append(db.log.data, []byte(s)...)
			if len(b) > len(db.log.data) {
				db.log.extend(len(b))
			}
			copy(db.log.data, b)

		case readReq := <-db.readChan:
			key := readReq.key
			returnChan := readReq.returnChan
			if value, ok := db.dataMap[key]; ok {
				m := &returnChanMessage{nil, value}
				returnChan <- *m
			} else {
				m := &returnChanMessage{errors.New("Invalid Key"), ""}
				returnChan <- *m
			}
		}
	}
}

func initDB(dbFilename string, logFilename string) *db {
	log := initLog(logFilename)
	writeChan := make(chan map[string]string, 250)
	readChan := make(chan readChanMessage)
	dataMap := make(map[string]string)
	var data []byte
	var fd int
	var file *os.File
	db := &db{data, dataMap, log, fd, dbFilename, file, writeChan, readChan}
	db.open()
	f, err := os.Stat(dbFilename)
	if err != nil {
		fmt.Println("Could not stat file: ", err)
	}

	db.mmap(int(f.Size()))
	db.load()
	go db.listener()
	return db
}<commit_msg>switch type name to DB<commit_after>package db

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"syscall"
)

type returnChanMessage struct {
	err  error
	json string
}

type readChanMessage struct {
	key        string
	returnChan chan returnChanMessage
}

type DB struct {
	data      []byte
	dataMap   map[string]string
	log       *log
	fd        int
	filename  string
	file      *os.File
	writeChan chan map[string]string
	readChan  chan readChanMessage
}

func (db *DB) load() {
	err := json.Unmarshal(db.data, &db.dataMap)
	if err != nil {
		fmt.Println("Error unmarshalling initial data into map: ", err)
	}
	fmt.Println(db.dataMap)
}

func (db *DB) mmap(size int) {
	fmt.Println("mmapping db file: ", size)
	data, err := syscall.Mmap(db.fd, 0, size, syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		fmt.Println("Error mmapping: ", err)
	}
	db.data = data
}

func (db *DB) resize(size int) {
	fmt.Println("Resizing db file: ", size)
	err := syscall.Ftruncate(db.fd, int64(size))
	if err != nil {
		fmt.Println("Error resizing: ", err)
	}
}

func (db *DB) open() {
	fmt.Println("Getting db file descriptor")
	f, err := os.OpenFile(db.filename, os.O_CREATE|os.O_RDWR, 0)
	if err != nil {
		fmt.Println("Could not open file: ", err)
	}
	db.fd = int(f.Fd())
	db.file = f
}

func (db *DB) extend(size int) {
	db.file.Close()
	db.open()
	db.resize(size)
	db.mmap(size)
}

func (db *DB) listener() {
	for {
		select {
		case writeReq := <-db.writeChan:
			key := writeReq["key"]
			value := writeReq["value"]
			db.dataMap[key] = value
			b, err := json.Marshal(db.dataMap)
			if err != nil {
				fmt.Println("Error marshalling db: ", err)
			}
			if len(b) > len(db.data) {
				db.extend(len(b))
			}
			copy(db.data, b)
			s := "SET " + key + ": " + value + "\n"
			b = append(db.log.data, []byte(s)...)
			if len(b) > len(db.log.data) {
				db.log.extend(len(b))
			}
			copy(db.log.data, b)

		case readReq := <-db.readChan:
			key := readReq.key
			returnChan := readReq.returnChan
			if value, ok := db.dataMap[key]; ok {
				m := &returnChanMessage{nil, value}
				returnChan <- *m
			} else {
				m := &returnChanMessage{errors.New("Invalid Key"), ""}
				returnChan <- *m
			}
		}
	}
}

func initDB(dbFilename string, logFilename string) *DB {
	log := initLog(logFilename)
	writeChan := make(chan map[string]string, 250)
	readChan := make(chan readChanMessage)
	dataMap := make(map[string]string)
	var data []byte
	var fd int
	var file *os.File
	db := &DB{data, dataMap, log, fd, dbFilename, file, writeChan, readChan}
	db.open()
	f, err := os.Stat(dbFilename)
	if err != nil {
		fmt.Println("Could not stat file: ", err)
	}

	db.mmap(int(f.Size()))
	db.load()
	go db.listener()
	return db
}
{'fraction_non_alphanumeric': 0.16235078053259872, 'fraction_numerical': 0.0025711662075298437, 'mean_word_length': 3.264682850430697, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4677562', 'n_tokens_mistral': 2375, 'n_tokens_neox': 2127, 'n_words': 522}