<commit_before>package garbage4

import (
	"os"
	"strconv"
	"syscall"
	"time"
	"unsafe"
)

const SEGMENT_HEADER_SIZE = 32

type Segment struct {
	*Header
	ref  []byte
	file *os.File
	data *[MAX_QUEUE_SIZE]byte
}

type Header struct {
	version uint32
	flag    uint32
	size    uint32 //PADDED
	id      uint64
	nextId  uint64
}

func newSegment(t *Topic) *Segment {
	id := uint64(time.Now().UnixNano())
	segment := openSegment(t, id, true)
	segment.id = id
	segment.size = SEGMENT_HEADER_SIZE
	return segment
}

func openSegment(t *Topic, id uint64, truncate bool) *Segment {
	name := PATH + t.name + "/" + strconv.FormatUint(id, 10) + ".q"
	file, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		panic(err)
	}
	if truncate {
		file.Truncate(MAX_QUEUE_SIZE)
	}
	ref, err := syscall.Mmap(int(file.Fd()), 0, MAX_QUEUE_SIZE, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
	if err != nil {
		panic(err)
	}

	s := &Segment{
		ref:  ref,
		file: file,
		data: (*[MAX_QUEUE_SIZE]byte)(unsafe.Pointer(&ref[0])),
	}
	s.Header = (*Header)(unsafe.Pointer(&s.data[0]))<commit_msg>Header's size might not reflect the actual size (maybe it crashed before flushing). However, the data is always properly flushed, so, on open, find the proper size.<commit_after>package garbage4

import (
	"os"
	"strconv"
	"syscall"
	"time"
	"unsafe"
)

const SEGMENT_HEADER_SIZE = 32

type Segment struct {
	*Header
	ref  []byte
	file *os.File
	data *[MAX_QUEUE_SIZE]byte
}

type Header struct {
	version uint32
	flag    uint32
	size    uint32 //PADDED
	id      uint64
	nextId  uint64
}

func newSegment(t *Topic) *Segment {
	id := uint64(time.Now().UnixNano())
	segment := openSegment(t, id, true)
	segment.id = id
	segment.size = SEGMENT_HEADER_SIZE
	return segment
}

func openSegment(t *Topic, id uint64, isNew bool) *Segment {
	name := PATH + t.name + "/" + strconv.FormatUint(id, 10) + ".q"
	file, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		panic(err)
	}
	if isNew {
		file.Truncate(MAX_QUEUE_SIZE)
	}
	ref, err := syscall.Mmap(int(file.Fd()), 0, MAX_QUEUE_SIZE, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
	if err != nil {
		panic(err)
	}

	s := &Segment{
		ref:  ref,
		file: file,
		data: (*[MAX_QUEUE_SIZE]byte)(unsafe.Pointer(&ref[0])),
	}
	s.Header = (*Header)(unsafe.Pointer(&s.data[0]))
	if isNew == false {
		offset := s.size
		for {
			o4 := offset + 4
			if o4 >= MAX_QUEUE_SIZE {
				break
			}
			l := encoder.Uint32(s.data[offset:])
			if l == 0 {
				break
			}
			offset = l + o4
		}
		s.size = offset
	}
{'fraction_non_alphanumeric': 0.14469078179696615, 'fraction_numerical': 0.022948269155970438, 'mean_word_length': 3.6763636363636363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '163923', 'n_tokens_mistral': 1137, 'n_tokens_neox': 1028, 'n_words': 245}