<commit_before>package walnut

import (
	"errors"
	"io"
)

var (
	ErrLineTooLong = errors.New("line too long")
)

// Defines a "key = value" assignment.
type def struct {
	key   string
	value string
	line  int
}

// Generates a slice of lines from a reader. Each line must fit in `buf`, or
// a `ErrLineTooLong` error will be returned.
func readLines(r io.Reader, buf []byte) ([][]byte, error) {
	lines := make([][]byte, 0)
	start, cont := 0, 0

	for {
		if cont == len(buf) {
			return nil, ErrLineTooLong
		}

		n, err := r.Read(buf[cont:])
		end := cont + n

		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}

		for {
			nl := byteIndex(buf[cont:end], '\n')
			if nl == -1 {
				break
			}

			lines = push(lines, buf[start:cont+nl])

			start = cont + 1
			cont += nl + 1
		}

		if start == 0 {
			cont = end
		} else {
			cont = copy(buf, buf[start:end])
			start = 0
		}
	}

	return push(lines, buf[start:cont]), nil
}

// Safely appends a line to a slice of lines.
func push(lines [][]byte, line []byte) [][]byte {
	dup := make([]byte, len(line))
	copy(dup, line)

	return append(lines, dup)
}
<commit_msg>Change error reporting in parse.go<commit_after>package walnut

import (
	"fmt"
	"io"
)

const (
	_ErrTooLong = "line %d too long (buffer overflow thwarted)"
)

// Defines a "key = value" assignment.
type def struct {
	key   string
	value string
	line  int
}

// Generates a slice of lines from a reader. Each line must fit in `buf`, or
// a `ErrLineTooLong` error will be returned.
func readLines(r io.Reader, buf []byte) ([][]byte, error) {
	lines := make([][]byte, 0)
	start, cont := 0, 0

	for {
		if cont == len(buf) {
			return nil, fmt.Errorf(_ErrTooLong, len(lines)+1)
		}

		n, err := r.Read(buf[cont:])
		end := cont + n

		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}

		for {
			nl := byteIndex(buf[cont:end], '\n')
			if nl == -1 {
				break
			}

			lines = push(lines, buf[start:cont+nl])

			start = cont + 1
			cont += nl + 1
		}

		if start == 0 {
			cont = end
		} else {
			cont = copy(buf, buf[start:end])
			start = 0
		}
	}

	return push(lines, buf[start:cont]), nil
}

// Safely appends a line to a slice of lines.
func push(lines [][]byte, line []byte) [][]byte {
	dup := make([]byte, len(line))
	copy(dup, line)

	return append(lines, dup)
}

{'fraction_non_alphanumeric': 0.1585261353898886, 'fraction_numerical': 0.00728363324764353, 'mean_word_length': 2.7722132471728593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '386938', 'n_tokens_mistral': 990, 'n_tokens_neox': 880, 'n_words': 283}