<commit_before>package sej

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"time"
)

var (
	errMessageCorrupted   = errors.New("last message of the journal file is courrupted")
	errJournalFileIsEmpty = errors.New("the journal file is empty")
)

// Message in a segmented journal file
type Message struct {
	Offset    uint64
	Timestamp time.Time
	Type      byte
	Key       []byte
	Value     []byte
}

type readSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

func (m *Message) IsNull() bool {
	return m.Offset == 0 &&
		m.Timestamp.IsZero() &&
		m.Type == 0 &&
		m.Key == nil &&
		m.Value == nil
}

// WriteMessage writes the message
// buf should be at least 8 bytes and is used to avoid allocation
func WriteMessage(w io.Writer, buf []byte, m *Message) (int64, error) {
	cnt := int64(0) // total bytes written

	n, err := writeUint64(w, buf, m.Offset)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	ts := m.Timestamp
	var nano int64
	if ts.IsZero() {
		nano = math.MinInt64
	} else {
		nano = ts.UnixNano()
	}
	n, err = writeInt64(w, buf, nano)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeByte(w, buf, m.Type)<commit_msg>copy method<commit_after>package sej

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"math"
	"os"
	"time"
)

var (
	errMessageCorrupted   = errors.New("last message of the journal file is courrupted")
	errJournalFileIsEmpty = errors.New("the journal file is empty")
)

// Message in a segmented journal file
type Message struct {
	Offset    uint64
	Timestamp time.Time
	Type      byte
	Key       []byte
	Value     []byte
}

type readSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

func (m *Message) IsNull() bool {
	return m.Offset == 0 &&
		m.Timestamp.IsZero() &&
		m.Type == 0 &&
		m.Key == nil &&
		m.Value == nil
}

func (m *Message) Copy() Message {
	return Message{
		Offset:    m.Offset,
		Timestamp: m.Timestamp,
		Type:      m.Type,
		Key:       append([]byte(nil), m.Key...),
		Value:     append([]byte(nil), m.Value...),
	}
}

// WriteMessage writes the message
// buf should be at least 8 bytes and is used to avoid allocation
func WriteMessage(w io.Writer, buf []byte, m *Message) (int64, error) {
	cnt := int64(0) // total bytes written

	n, err := writeUint64(w, buf, m.Offset)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	ts := m.Timestamp
	var nano int64
	if ts.IsZero() {
		nano = math.MinInt64
	} else {
		nano = ts.UnixNano()
	}
	n, err = writeInt64(w, buf, nano)
	cnt += int64(n)
	if err != nil {
		return cnt, err
	}

	n, err = writeByte(w, buf, m.Type)
{'fraction_non_alphanumeric': 0.13734567901234568, 'fraction_numerical': 0.016975308641975308, 'mean_word_length': 3.0263975155279503, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5339204', 'n_tokens_mistral': 1031, 'n_tokens_neox': 978, 'n_words': 299}