<commit_before>package sej

import (
	"io"
	"os"
	"path"
)

// Offset is used to manage a disk-persisted offset
type Offset struct {
	dir      *os.File
	file     string
	fileLock *fileLock
	value    uint64

	Syncing bool
}

type DefaultOffset int

const (
	FirstOffset DefaultOffset = iota
	LastOffset
)

// NewOffset creates a new Offset object persisted to dir/ofs/name.ofs
func NewOffset(dir, name string, defaultOffset DefaultOffset) (*Offset, error) {
	jnlDir, dir := JournalDirPath(dir), OffsetDirPath(dir)
	_ = os.MkdirAll(dir, 0755)
	filePrefix := path.Join(dir, name)
	fileLock, err := openFileLock(filePrefix + ".lck")
	if err != nil {
		return nil, err
	}
	d, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	o := &Offset{dir: d, file: filePrefix + ".ofs", fileLock: fileLock}
	f, err := openOrCreate(o.file)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	o.value, err = ReadOffset(f)
	if err == io.EOF {
		jd, err := OpenJournalDir(jnlDir)
		if err != nil {
			return nil, err
		}
		if defaultOffset == LastOffset {
			o.value, err = jd.Last().LastReadableOffset()
			if err != nil {
				return nil, err
			}
		} else {
			o.value = jd.First().FirstOffset
		}
	} else if err != nil {
		return nil, err
	}
	return o, nil
}

func OpenReadonlyOffset(dir, name string) (*Offset, error) {
	filePrefix := path.Join(dir, name)
	d, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	o := &Offset{dir: d, file: filePrefix + ".ofs"}
	f, err := os.Open(o.file)
	if err != nil {
		return nil, err
	}
	defer f.Close()<commit_msg>fix offset dir<commit_after>package sej

import (
	"io"
	"os"
	"path"
)

// Offset is used to manage a disk-persisted offset
type Offset struct {
	dir      *os.File
	file     string
	fileLock *fileLock
	value    uint64

	Syncing bool
}

type DefaultOffset int

const (
	FirstOffset DefaultOffset = iota
	LastOffset
)

// NewOffset creates a new Offset object persisted to dir/ofs/name.ofs
func NewOffset(dir, name string, defaultOffset DefaultOffset) (*Offset, error) {
	jnlDir, dir := JournalDirPath(dir), OffsetDirPath(dir)
	_ = os.MkdirAll(dir, 0755)
	filePrefix := path.Join(dir, name)
	fileLock, err := openFileLock(filePrefix + ".lck")
	if err != nil {
		return nil, err
	}
	d, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	o := &Offset{dir: d, file: filePrefix + ".ofs", fileLock: fileLock}
	f, err := openOrCreate(o.file)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	o.value, err = ReadOffset(f)
	if err == io.EOF {
		jd, err := OpenJournalDir(jnlDir)
		if err != nil {
			return nil, err
		}
		if defaultOffset == LastOffset {
			o.value, err = jd.Last().LastReadableOffset()
			if err != nil {
				return nil, err
			}
		} else {
			o.value = jd.First().FirstOffset
		}
	} else if err != nil {
		return nil, err
	}
	return o, nil
}

func OpenReadonlyOffset(dir, name string) (*Offset, error) {
	dir = OffsetDirPath(dir)
	filePrefix := path.Join(dir, name)
	d, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	o := &Offset{dir: d, file: filePrefix + ".ofs"}
	f, err := os.Open(o.file)
	if err != nil {
		return nil, err
	}
	defer f.Close()
{'fraction_non_alphanumeric': 0.13675486133248327, 'fraction_numerical': 0.003825310806503028, 'mean_word_length': 3.246278755074425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3647549', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1172, 'n_words': 351}