<commit_before>package spool

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"

	"github.com/vektra/cypress"
	"github.com/vektra/tai64n"
)

type Spool struct {

	// The size of each file will get before it's rotated
	PerFileSize int64

	// How many rotate files to keep
	MaxFiles int

	OnRotate func(string) error

	root      string
	current   string
	file      *os.File
	startSize int64

	enc *cypress.StreamEncoder
}

const PerFileSize = (1024 * 1024) // 1 meg per file
const MaxFiles = 10

// So we'll spool 10 megs worth of logs max

const DefaultSpoolDir = "/var/lib/cypress/spool"

func (sf *Spool) openCurrent() error {
	fd, err := os.OpenFile(sf.current, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}

	sf.file = fd

	fi, err := fd.Stat()
	if err != nil {
		return err
	}

	sf.startSize = fi.Size()

	enc := cypress.NewStreamEncoder(fd)

	if sf.startSize == 0 {
		err = enc.Init(cypress.SNAPPY)<commit_msg>Properly reopen the spool file<commit_after>package spool

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"

	"github.com/vektra/cypress"
	"github.com/vektra/tai64n"
)

type Spool struct {

	// The size of each file will get before it's rotated
	PerFileSize int64

	// How many rotate files to keep
	MaxFiles int

	OnRotate func(string) error

	root      string
	current   string
	file      *os.File
	startSize int64

	enc *cypress.StreamEncoder
}

const PerFileSize = (1024 * 1024) // 1 meg per file
const MaxFiles = 10

// So we'll spool 10 megs worth of logs max

const DefaultSpoolDir = "/var/lib/cypress/spool"

func (sf *Spool) openCurrent() error {
	fd, err := os.OpenFile(sf.current, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}

	sf.file = fd

	fi, err := fd.Stat()
	if err != nil {
		return err
	}

	sf.startSize = fi.Size()

	enc := cypress.NewStreamEncoder(fd)

	if sf.startSize == 0 {
		err = enc.Init(cypress.SNAPPY)
		if err != nil {
			return err
		}
	} else {
		err = enc.OpenFile(fd)
{'fraction_non_alphanumeric': 0.11956521739130435, 'fraction_numerical': 0.023715415019762844, 'mean_word_length': 3.13265306122449, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3062864', 'n_tokens_mistral': 876, 'n_tokens_neox': 796, 'n_words': 221}