<commit_before>package sse

import (
	"bytes"
	"fmt"
	"io"
)

const (
	id   = "id: %d\n"
	data = "data: %s\n"
)

type encoder struct {
	reader io.Reader // stores the original reader
	offset int64     // offset for Seek purposes
}

func NewEncoder(r io.Reader) io.Reader {
	return &encoder{reader: r}
}

func (r *encoder) Seek(offset int64, whence int) (n int64, err error) {
	r.offset, err = r.reader.(io.ReadSeeker).Seek(offset, whence)
	return r.offset, err
}

// FIXME: this version is simplified and assumes
// that len(p) is always greater than the potential
// length of data to be read.
func (r *encoder) Read(p []byte) (n int, err error) {
	n, err = r.reader.Read(p)

	if n > 0 {
		buf := format(r.offset, p[:n])
		r.offset += int64(n)
		n = copy(p, buf)
	}

	return n, err
}

func format(pos int64, msg []byte) []byte {
	buf := bytes.NewBufferString(fmt.Sprintf(id, pos+int64(len(msg))))

	for _, line := range bytes.Split(msg, []byte{'\n'}) {
		buf.WriteString(fmt.Sprintf(data, line))
	}
	buf.Write([]byte{'\n'})

	return buf.Bytes()
}
<commit_msg>Fix panic in SSE for non seekable readers<commit_after>package sse

import (
	"bytes"
	"fmt"
	"io"
)

const (
	id   = "id: %d\n"
	data = "data: %s\n"
)

type encoder struct {
	reader io.Reader // stores the original reader
	offset int64     // offset for Seek purposes
}

func NewEncoder(r io.Reader) io.Reader {
	return &encoder{reader: r}
}

func (r *encoder) Seek(offset int64, whence int) (n int64, err error) {
	if seeker, ok := r.reader.(io.ReadSeeker); ok {
		r.offset, err = seeker.Seek(offset, whence)
	}
	return r.offset, err
}

// FIXME: this version is simplified and assumes
// that len(p) is always greater than the potential
// length of data to be read.
func (r *encoder) Read(p []byte) (n int, err error) {
	n, err = r.reader.Read(p)

	if n > 0 {
		buf := format(r.offset, p[:n])
		r.offset += int64(n)
		n = copy(p, buf)
	}

	return n, err
}

func format(pos int64, msg []byte) []byte {
	buf := bytes.NewBufferString(fmt.Sprintf(id, pos+int64(len(msg))))

	for _, line := range bytes.Split(msg, []byte{'\n'}) {
		buf.WriteString(fmt.Sprintf(data, line))
	}
	buf.Write([]byte{'\n'})

	return buf.Bytes()
}

{'fraction_non_alphanumeric': 0.16399450297755383, 'fraction_numerical': 0.011910215300045808, 'mean_word_length': 3.5217391304347827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1614516', 'n_tokens_mistral': 879, 'n_tokens_neox': 839, 'n_words': 250}