<commit_before>package buf

import (
	"fmt"
	"io"

	"github.com/fagongzi/util/hack"
)

const (
	defaultMinGrowSize      = 256
	defaultIOCopyBufferSize = 1024 * 4
)

// Option bytebuf option
type Option func(*ByteBuf)

// WithMemAllocator Set the memory allocator, when Bytebuf is initialized, it needs to
// allocate a []byte of the size specified by capacity from memory. When ByteBuf.Release
// is called, the memory will be freed back to the allocator.
func WithMemAllocator(alloc Allocator) Option {
	return func(bb *ByteBuf) {
		bb.options.alloc = alloc
	}
}

// WithMinGowSize set minimum Grow size. When there is not enough space left
// in the Bytebuf, write data needs to be expanded.
func WithMinGowSize(minGrowSize int) Option {
	return func(bb *ByteBuf) {
		bb.options.minGrowSize = minGrowSize
	}
}

// WithIOCopyBufferSize set io copy buffer used to control how much data will written
// at a time.
func WithIOCopyBufferSize(value int) Option {
	return func(bb *ByteBuf) {
		bb.options.ioCopyBufferSize = value
	}
}

// disableCompactAfterGrow set Set whether the buffer should be compressed,
// if it is grow, it will reset the reader and writer index. Default is true.
func WithDisableCompactAfterGrow(value bool) Option {
	return func(bb *ByteBuf) {
		bb.options.disableCompactAfterGrow = value
	}
}

// Slice the slice of byte buf
type Slice struct {
	from, to int // [from, to)
	buf      *ByteBuf
}

// Data data
func (s Slice) Data() []byte {
	return s.buf.buf[s.from:s.to]
}

var (
	_ io.WriterTo   = (*ByteBuf)(nil)
	_ io.Writer     = (*ByteBuf)(nil)
	_ io.Reader     = (*ByteBuf)(nil)
	_ io.ReaderFrom = (*ByteBuf)(nil)
)

// ByteBuf is a reusable buffer that holds an internal []byte and maintains 2 indexes for
// read and write data.
//
// | discardable bytes  |   readable bytes   |   writeable bytes  |
// |                    |                    |                    |
// |                    |                    |                    |
// 0      <=       readerIndex    <=     writerIndex    <=     capacity
//
// The ByteBuf implemented io.Reader, io.Writer, io.WriterTo, io.ReaderFrom interface
type ByteBuf struct {
	buf         []byte // buf data, auto +/- size
	readerIndex int
	writerIndex int
	markedIndex int

	options struct {
		alloc                   Allocator
		minGrowSize             int
		ioCopyBufferSize        int
		disableCompactAfterGrow bool
	}
}

// NewByteBuf create bytebuf with options
func NewByteBuf(capacity int, opts ...Option) *ByteBuf {
	b := &ByteBuf{
		readerIndex: 0,
		writerIndex: 0,
	}
	for _, opt := range opts {
		opt(b)
	}
	b.adjust()
	b.buf = b.options.alloc.Alloc(capacity)
	return b
}

func (b *ByteBuf) adjust() {
	if b.options.alloc == nil {
		b.options.alloc = newNonReusableAllocator()
	}
	if b.options.minGrowSize == 0 {
		b.options.minGrowSize = defaultMinGrowSize
	}
	if b.options.ioCopyBufferSize == 0 {
		b.options.ioCopyBufferSize = defaultIOCopyBufferSize
	}
}

// Close close the ByteBuf
func (b *ByteBuf) Close() {
	b.options.alloc.Free(b.buf)
	b.buf = nil
}

// Reset reset to reuse.
func (b *ByteBuf) Reset() {
	b.readerIndex = 0
	b.writerIndex = 0
	b.markedIndex = 0
}

// SetReadIndex set the reader index. The data in the [readIndex, writeIndex] that can be read.
func (b *ByteBuf) SetReadIndex(readIndex int) {
	if readIndex < 0 || readIndex > b.writerIndex {
		panic(fmt.Sprintf("invalid readIndex %d, writeIndex %d", readIndex, b.writerIndex))
	}

	b.readerIndex = readIndex
}

// GetReadIndex returns the read index
func (b *ByteBuf) GetReadIndex() int {
	return b.readerIndex
}

// SetWriteIndex set the write index. The data can write into range [writeIndex, len(buf)).
// Note, since the underlying buf will expand, the previously held writeIndex will become
// invalid, and in most cases this method should use SetWriteIndexByOffset instead.
func (b *ByteBuf) SetWriteIndex(writeIndex int) {
	if writeIndex < b.readerIndex || writeIndex > b.capacity() {
		panic(fmt.Sprintf("invalid writeIndex %d, capacity %d, readIndex %d",
			writeIndex, b.capacity(), b.readerIndex))
	}

	b.writerIndex = writeIndex
}

// GetWriteIndex get the write index
func (b *ByteBuf) GetWriteIndex() int {
	return b.writerIndex
}

// GetWriteOffset returns the offset of the current writeIndex relative to the ReadIndex.
// GetWriteIndex returns an absolute position, which will fail when the underlying buf is
// expanded.
func (b *ByteBuf) GetWriteOffset() int {
	return b.Readable()
}

// SetWriteIndexByOffset Use writeOffset to reset writeIndex, since offset is a relative
// position to readIndex, so it won't affect correctness when the underlying buf is expanded.
func (b *ByteBuf) SetWriteIndexByOffset(writeOffset int) {
	index := b.Readable() + writeOffset
	b.SetWriteIndex(index)
}<commit_msg>fix set write index by offset (#45)<commit_after>package buf

import (
	"fmt"
	"io"

	"github.com/fagongzi/util/hack"
)

const (
	defaultMinGrowSize      = 256
	defaultIOCopyBufferSize = 1024 * 4
)

// Option bytebuf option
type Option func(*ByteBuf)

// WithMemAllocator Set the memory allocator, when Bytebuf is initialized, it needs to
// allocate a []byte of the size specified by capacity from memory. When ByteBuf.Release
// is called, the memory will be freed back to the allocator.
func WithMemAllocator(alloc Allocator) Option {
	return func(bb *ByteBuf) {
		bb.options.alloc = alloc
	}
}

// WithMinGowSize set minimum Grow size. When there is not enough space left
// in the Bytebuf, write data needs to be expanded.
func WithMinGowSize(minGrowSize int) Option {
	return func(bb *ByteBuf) {
		bb.options.minGrowSize = minGrowSize
	}
}

// WithIOCopyBufferSize set io copy buffer used to control how much data will written
// at a time.
func WithIOCopyBufferSize(value int) Option {
	return func(bb *ByteBuf) {
		bb.options.ioCopyBufferSize = value
	}
}

// disableCompactAfterGrow set Set whether the buffer should be compressed,
// if it is grow, it will reset the reader and writer index. Default is true.
func WithDisableCompactAfterGrow(value bool) Option {
	return func(bb *ByteBuf) {
		bb.options.disableCompactAfterGrow = value
	}
}

// Slice the slice of byte buf
type Slice struct {
	from, to int // [from, to)
	buf      *ByteBuf
}

// Data data
func (s Slice) Data() []byte {
	return s.buf.buf[s.from:s.to]
}

var (
	_ io.WriterTo   = (*ByteBuf)(nil)
	_ io.Writer     = (*ByteBuf)(nil)
	_ io.Reader     = (*ByteBuf)(nil)
	_ io.ReaderFrom = (*ByteBuf)(nil)
)

// ByteBuf is a reusable buffer that holds an internal []byte and maintains 2 indexes for
// read and write data.
//
// | discardable bytes  |   readable bytes   |   writeable bytes  |
// |                    |                    |                    |
// |                    |                    |                    |
// 0      <=       readerIndex    <=     writerIndex    <=     capacity
//
// The ByteBuf implemented io.Reader, io.Writer, io.WriterTo, io.ReaderFrom interface
type ByteBuf struct {
	buf         []byte // buf data, auto +/- size
	readerIndex int
	writerIndex int
	markedIndex int

	options struct {
		alloc                   Allocator
		minGrowSize             int
		ioCopyBufferSize        int
		disableCompactAfterGrow bool
	}
}

// NewByteBuf create bytebuf with options
func NewByteBuf(capacity int, opts ...Option) *ByteBuf {
	b := &ByteBuf{
		readerIndex: 0,
		writerIndex: 0,
	}
	for _, opt := range opts {
		opt(b)
	}
	b.adjust()
	b.buf = b.options.alloc.Alloc(capacity)
	return b
}

func (b *ByteBuf) adjust() {
	if b.options.alloc == nil {
		b.options.alloc = newNonReusableAllocator()
	}
	if b.options.minGrowSize == 0 {
		b.options.minGrowSize = defaultMinGrowSize
	}
	if b.options.ioCopyBufferSize == 0 {
		b.options.ioCopyBufferSize = defaultIOCopyBufferSize
	}
}

// Close close the ByteBuf
func (b *ByteBuf) Close() {
	b.options.alloc.Free(b.buf)
	b.buf = nil
}

// Reset reset to reuse.
func (b *ByteBuf) Reset() {
	b.readerIndex = 0
	b.writerIndex = 0
	b.markedIndex = 0
}

// SetReadIndex set the reader index. The data in the [readIndex, writeIndex] that can be read.
func (b *ByteBuf) SetReadIndex(readIndex int) {
	if readIndex < 0 || readIndex > b.writerIndex {
		panic(fmt.Sprintf("invalid readIndex %d, writeIndex %d", readIndex, b.writerIndex))
	}

	b.readerIndex = readIndex
}

// GetReadIndex returns the read index
func (b *ByteBuf) GetReadIndex() int {
	return b.readerIndex
}

// SetWriteIndex set the write index. The data can write into range [writeIndex, len(buf)).
// Note, since the underlying buf will expand, the previously held writeIndex will become
// invalid, and in most cases this method should use SetWriteIndexByOffset instead.
func (b *ByteBuf) SetWriteIndex(writeIndex int) {
	if writeIndex < b.readerIndex || writeIndex > b.capacity() {
		panic(fmt.Sprintf("invalid writeIndex %d, capacity %d, readIndex %d",
			writeIndex, b.capacity(), b.readerIndex))
	}

	b.writerIndex = writeIndex
}

// GetWriteIndex get the write index
func (b *ByteBuf) GetWriteIndex() int {
	return b.writerIndex
}

// GetWriteOffset returns the offset of the current writeIndex relative to the ReadIndex.
// GetWriteIndex returns an absolute position, which will fail when the underlying buf is
// expanded.
func (b *ByteBuf) GetWriteOffset() int {
	return b.Readable()
}

// SetWriteIndexByOffset Use writeOffset to reset writeIndex, since offset is a relative
// position to readIndex, so it won't affect correctness when the underlying buf is expanded.
func (b *ByteBuf) SetWriteIndexByOffset(writeOffset int) {
	b.SetWriteIndex(b.readerIndex + writeOffset)
}
{'fraction_non_alphanumeric': 0.10082541009298923, 'fraction_numerical': 0.003970327029568488, 'mean_word_length': 3.178088171104321, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3332025', 'n_tokens_mistral': 3119, 'n_tokens_neox': 2998, 'n_words': 1008}