<commit_before>package lzb

import "errors"

type buffer struct {
	data   []byte
	bottom int64
	top    int64
}

var (
	errOffset = errors.New("offset outside buffer range")
	errAgain  = errors.New("buffer overflow; repeat")
	errNegLen = errors.New("length is negative")
)

func initBuffer(b *buffer, capacity int) {
	*b = buffer{data: make([]byte, capacity)}
}

func newBuffer(capacity int) *buffer {
	return &buffer{data: make([]byte, capacity)}
}

func (b *buffer) capacity() int {
	return len(b.data)
}

func (b *buffer) length() int {
	return int(b.top - b.bottom)
}

func (b *buffer) setTop(off int64) {
	if off < 0 {
		panic("b.Top overflow?")
	}
	b.top = off
	b.bottom = off - int64(len(b.data))
	if b.bottom < 0 {
		b.bottom = 0
	}
}

func (b *buffer) index(off int64) int {
	if off < 0 {
		panic("negative offset?")
	}
	return int(off % int64(len(b.data)))
}

func (b *buffer) writeSlice(p []byte) {
	if len(p) == 0 {
		return
	}
	off := b.top
	m := len(p) - len(b.data)
	if m > 0 {
		off += int64(m)
		if off < 0 {
			panic("overflow b.top")
		}
		p = p[m:]
	}
	for len(p) > 0 {
		m = copy(b.data[b.index(off):], p)
		off += int64(m)
		if off < 0 {
			panic("overflow b.top")
		}
		p = p[m:]
	}
	b.setTop(off)
}

func (b *buffer) Write(p []byte) (n int, err error) {
	b.writeSlice(p)
	return len(p), nil
}

func (b *buffer) WriteByte(c byte) error {
	b.data[b.index(b.top)] = c
	b.setTop(b.top + 1)
	return nil
}

func (b *buffer) writeRep(off int64, n int) (written int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	if !(b.bottom <= off && off <= b.top) {
		return 0, errOffset
	}
	end := off + int64(n)
	if !(end <= b.top) {
		return 0, errAgain
	}
	e := b.index(end)
	for off < end {
		s := b.index(off)
		var q []byte
		if s < e {
			q = b.data[s:e]
		} else {
			q = b.data[s:]
		}
		b.writeSlice(q)
		off += int64(len(q))
	}
	return n, nil
}

// equalBytes count the equal bytes at off1 and off2 until max is reached.
func (b *buffer) equalBytes(off1, off2 int64, max int) int {
	if off1 < b.top || off2 < b.top || max <= 0 {
		return 0
	}
	n := b.top - off1<commit_msg>buffer.go: support for a write limit<commit_after>package lzb

import "errors"

type buffer struct {
	data       []byte
	bottom     int64 // bottom == max(top - len(data), 0)
	top        int64
	writeLimit int64
}

const maxWriteLimit = 1<<63 - 1

var (
	errOffset = errors.New("offset outside buffer range")
	errAgain  = errors.New("buffer overflow; repeat")
	errNegLen = errors.New("length is negative")
	errLimit  = errors.New("write limit reached")
)

func initBuffer(b *buffer, capacity int) {
	*b = buffer{data: make([]byte, capacity), writeLimit: maxWriteLimit}
}

func newBuffer(capacity int) *buffer {
	b := new(buffer)
	initBuffer(b, capacity)
	return b
}

func (b *buffer) capacity() int {
	return len(b.data)
}

func (b *buffer) length() int {
	return int(b.top - b.bottom)
}

func (b *buffer) setTop(off int64) {
	if off < 0 {
		panic("b.Top overflow?")
	}
	if off > b.writeLimit {
		panic("off exceeds writeLimit")
	}
	b.top = off
	b.bottom = off - int64(len(b.data))
	if b.bottom < 0 {
		b.bottom = 0
	}
}

func (b *buffer) index(off int64) int {
	if off < 0 {
		panic("negative offset?")
	}
	return int(off % int64(len(b.data)))
}

func (b *buffer) Write(p []byte) (n int, err error) {
	if len(p) == 0 {
		return
	}
	var m int
	off := b.top
	if off+int64(len(p)) > b.writeLimit {
		m = int(b.writeLimit - off)
		p = p[:m]
		err = errors.New("write limit reached")
	}
	m = len(p) - len(b.data)
	if m > 0 {
		n += m
		p = p[m:]
	}
	for len(p) > 0 {
		m = copy(b.data[b.index(off):], p)
		n += m
		p = p[m:]
	}
	b.setTop(off + int64(n))
	return n, err
}

func (b *buffer) WriteByte(c byte) error {
	if b.top >= b.writeLimit {
		return errLimit
	}
	b.data[b.index(b.top)] = c
	b.setTop(b.top + 1)
	return nil
}

func (b *buffer) writeRep(off int64, n int) (written int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	if !(b.bottom <= off && off <= b.top) {
		return 0, errOffset
	}
	start, end := off, off + int64(n)
	if !(end <= b.top) {
		return 0, errAgain
	}
	e := b.index(end)
	for off < end {
		s := b.index(off)
		var q []byte
		if s < e {
			q = b.data[s:e]
		} else {
			q = b.data[s:]
		}
		n, err := b.Write(q)
		off += int64(n)
		if err != nil {
			break
		}
		off += int64(n)
	}
	b.setTop(off)
	return int(off - start), nil
}

// equalBytes count the equal bytes at off1 and off2 until max is reached.
func (b *buffer) equalBytes(off1, off2 int64, max int) int {
	if off1 < b.top || off2 < b.top || max <= 0 {
		return 0
	}
	n := b.top - off1
{'fraction_non_alphanumeric': 0.1698816308636563, 'fraction_numerical': 0.022139412538360367, 'mean_word_length': 2.9302325581395348, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1400919', 'n_tokens_mistral': 2055, 'n_tokens_neox': 1921, 'n_words': 549}