<commit_before>package buffer

import (
	"io"
)

const (
	maxUint   = ^uint32(0)
	maxInt    = int32(maxUint >> 1)
	maxUint64 = ^uint64(0)
	maxInt64  = int64(maxUint64 >> 1)
	maxAlloc  = int64(1024 * 32)
)

type Buffer interface {
	Len() int64
	Cap() int64
	io.Reader
	io.Writer
}

type buffer struct {
	head []byte
	Buffer
}

func MaxCap() int64 {
	return maxInt64
}

func LimitAlloc(n int64) int64 {
	if n > maxAlloc {
		return maxAlloc
	}
	return n
}

func Gap(buf Buffer) int64 {
	return buf.Cap() - buf.Len()
}

func Full(buf Buffer) bool {
	return Gap(buf) == 0
}

type Lener interface {
	Len() int64
}

func Empty(l Lener) bool {
	return l.Len() == 0
}

func RoomFor(buf Buffer, p []byte) bool {
	return Gap(buf) >= int64(len(p))
}

func ShrinkToRead(buf Buffer, p []byte) []byte {
	if buf.Len() < int64(len(p)) {
		return p[:buf.Len()]
	}
	return p
}

func ShrinkToFit(buf Buffer, p []byte) []byte {
	if !RoomFor(buf, p) {
		p = p[:Gap(buf)]
	}
	return p
}

func TotalLen(buffers []Buffer) (n int64) {
	for _, buffer := range buffers {
		if n > MaxCap()-buffer.Len() {
			return MaxCap()
		} else {
			n += buffer.Len()
		}
	}
	return n
}

func TotalCap(buffers []Buffer) (n int64) {
	for _, buffer := range buffers {
		if n > MaxCap()-buffer.Cap() {
			return MaxCap()
		} else {<commit_msg>Made buffer functions more generic<commit_after>package buffer

import (
	"io"
)

const (
	maxUint   = ^uint32(0)
	maxInt    = int32(maxUint >> 1)
	maxUint64 = ^uint64(0)
	maxInt64  = int64(maxUint64 >> 1)
	maxAlloc  = int64(1024 * 32)
)

type Lener interface {
	Len() int64
}

type Caper interface {
	Cap() int64
}

type LenCaper interface {
	Lener
	Caper
}

type Buffer interface {
	Lener
	Caper
	io.Reader
	io.Writer
}

type buffer struct {
	head []byte
	Buffer
}

func MaxCap() int64 {
	return maxInt64
}

func LimitAlloc(n int64) int64 {
	if n > maxAlloc {
		return maxAlloc
	}
	return n
}

func Gap(buf LenCaper) int64 {
	return buf.Cap() - buf.Len()
}

func Full(buf LenCaper) bool {
	return Gap(buf) == 0
}

func Empty(l Lener) bool {
	return l.Len() == 0
}

func RoomFor(buf LenCaper, p []byte) bool {
	return Gap(buf) >= int64(len(p))
}

func ShrinkToRead(buf Lener, p []byte) []byte {
	if buf.Len() < int64(len(p)) {
		return p[:buf.Len()]
	}
	return p
}

func ShrinkToFit(buf LenCaper, p []byte) []byte {
	if !RoomFor(buf, p) {
		p = p[:Gap(buf)]
	}
	return p
}

func TotalLen(buffers []Buffer) (n int64) {
	for _, buffer := range buffers {
		if n > MaxCap()-buffer.Len() {
			return MaxCap()
		} else {
			n += buffer.Len()
		}
	}
	return n
}

func TotalCap(buffers []Buffer) (n int64) {
	for _, buffer := range buffers {
		if n > MaxCap()-buffer.Cap() {
			return MaxCap()
		} else {
{'fraction_non_alphanumeric': 0.14985163204747776, 'fraction_numerical': 0.037833827893175076, 'mean_word_length': 3.3221153846153846, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6452161', 'n_tokens_mistral': 1205, 'n_tokens_neox': 1110, 'n_words': 300}