<commit_before>package storage

import (
	"log"
	"os"
	"pkg/util"
)

type NeedleMap struct {
	indexFile *os.File
	m         CompactMap

	//transient
	bytes           []byte
	deletionCounter int
	fileCounter     int
}

func NewNeedleMap(file *os.File) *NeedleMap {
	nm := &NeedleMap{
		m:         NewCompactMap(),
		bytes:     make([]byte, 16),
		indexFile: file,
	}
	return nm
}

const (
	RowsToRead = 1024
)

func LoadNeedleMap(file *os.File) *NeedleMap {
	nm := NewNeedleMap(file)
	bytes := make([]byte, 16*RowsToRead)
	count, e := nm.indexFile.Read(bytes)
	if count > 0 {
		fstat, _ := file.Stat()
		log.Println("Loading index file", fstat.Name(), "size", fstat.Size())
	}
	for count > 0 && e == nil {
		for i := 0; i < count; i += 16 {
			key := util.BytesToUint64(bytes[i : i+8])
			offset := util.BytesToUint32(bytes[i+8 : i+12])
			size := util.BytesToUint32(bytes[i+12 : i+16])
			if offset > 0 {
				nm.m.Set(Key(key), offset, size)
				//log.Println("reading key", key, "offset", offset, "size", size)
				nm.fileCounter++
			} else {
				nm.m.Delete(Key(key))
        log.Println("removing key", key)
				nm.deletionCounter++
			}<commit_msg>remove debug message<commit_after>package storage

import (
	"log"
	"os"
	"pkg/util"
)

type NeedleMap struct {
	indexFile *os.File
	m         CompactMap

	//transient
	bytes           []byte
	deletionCounter int
	fileCounter     int
}

func NewNeedleMap(file *os.File) *NeedleMap {
	nm := &NeedleMap{
		m:         NewCompactMap(),
		bytes:     make([]byte, 16),
		indexFile: file,
	}
	return nm
}

const (
	RowsToRead = 1024
)

func LoadNeedleMap(file *os.File) *NeedleMap {
	nm := NewNeedleMap(file)
	bytes := make([]byte, 16*RowsToRead)
	count, e := nm.indexFile.Read(bytes)
	if count > 0 {
		fstat, _ := file.Stat()
		log.Println("Loading index file", fstat.Name(), "size", fstat.Size())
	}
	for count > 0 && e == nil {
		for i := 0; i < count; i += 16 {
			key := util.BytesToUint64(bytes[i : i+8])
			offset := util.BytesToUint32(bytes[i+8 : i+12])
			size := util.BytesToUint32(bytes[i+12 : i+16])
			if offset > 0 {
				nm.m.Set(Key(key), offset, size)
				//log.Println("reading key", key, "offset", offset, "size", size)
				nm.fileCounter++
			} else {
				nm.m.Delete(Key(key))
        //log.Println("removing key", key)
				nm.deletionCounter++
			}
{'fraction_non_alphanumeric': 0.1632829373650108, 'fraction_numerical': 0.024190064794816415, 'mean_word_length': 3.070298769771529, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6944674', 'n_tokens_mistral': 1010, 'n_tokens_neox': 890, 'n_words': 206}