<commit_before>package runeio

// RuneReadUnreader is the underlying interface Reader will use.
type RuneReadUnreader interface {
	ReadRune() (r rune, size int, err error)
	UnreadRune() error
	String() string
}

type Reader struct {
	rr    RuneReadUnreader
	Runes []rune
}

func NewRuneio(r RuneReadUnreader) *Reader {
	return &Reader{r, []rune{}}
}

func (r *Reader) Discard(n uint) (discarded uint, err error) {
	runes, err := r.ReadRunes(n)
	return uint(len(runes)), err
}

func (r *Reader) ReadRunes(n uint) (runes []rune, err error) {
	if err = r.readFromReader(n); err != nil {
		n = uint(len(r.Runes))
	}

	runes = r.Runes[0:n]
	r.Runes = r.Runes[n:]

	return runes, err
}

func (r *Reader) PeekRunes(n uint) (runes []rune, err error) {
	if err := r.readFromReader(n); err != nil {
		return r.Runes, err
	}

	return r.Runes[0:n], nil
}

func (r *Reader) String() string {
	return string(r.Runes) + r.rr.String()
}

func (r *Reader) Reset(bufReader RuneReadUnreader) {
	r.rr = bufReader
}

func (r *Reader) readFromReader(n uint) error {
	l := int(n) - len(r.Runes)

	// check if we've already read enough runes
	if l <= 0 {
		return nil
	}

	// if not, read the remaining amount of runes
	for i := 0; i < l; i++ {
		ru, _, err := r.rr.ReadRune()
		if err != nil {
			return err
		}
		r.Runes = append(r.Runes, ru)
	}

	return nil
}<commit_msg>runeio: embed RuneReadUnreader in Reader; rename RuneReadUnreader to RuneReader<commit_after>package runeio

// RuneReader is the underlying interface Reader will use.
type RuneReader interface {
	ReadRune() (r rune, size int, err error)
	String() string
}

type Reader struct {
	RuneReader
	Runes []rune
}

func NewRuneio(r RuneReader) *Reader {
	return &Reader{r, []rune{}}
}

func (r *Reader) Discard(n uint) (discarded uint, err error) {
	runes, err := r.ReadRunes(n)
	return uint(len(runes)), err
}

func (r *Reader) ReadRunes(n uint) (runes []rune, err error) {
	if err = r.readFromReader(n); err != nil {
		n = uint(len(r.Runes))
	}

	runes = r.Runes[0:n]
	r.Runes = r.Runes[n:]

	return runes, err
}

func (r *Reader) PeekRunes(n uint) (runes []rune, err error) {
	if err := r.readFromReader(n); err != nil {
		return r.Runes, err
	}

	return r.Runes[0:n], nil
}

func (r *Reader) String() string {
	return string(r.Runes) + r.RuneReader.String()
}

func (r *Reader) Reset(bufReader RuneReader) {
	r.RuneReader = bufReader
}

func (r *Reader) readFromReader(n uint) error {
	l := int(n) - len(r.Runes)

	// check if we've already read enough runes
	if l <= 0 {
		return nil
	}

	// if not, read the remaining amount of runes
	for i := 0; i < l; i++ {
		ru, _, err := r.ReadRune()
		if err != nil {
			return err
		}
		r.Runes = append(r.Runes, ru)
	}

	return nil
}
{'fraction_non_alphanumeric': 0.14652956298200515, 'fraction_numerical': 0.0029379360998898272, 'mean_word_length': 3.6091370558375635, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4845672', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1091, 'n_words': 323}