<commit_before>package oslice

import (
	"bytes"
	"log"
	"sort"
)

type OSlice struct {
	buf        *bytes.Buffer
	regionList []int
	idList     []RegionID
	isSorted   bool
}

type RegionID int

func New() *OSlice {
	return &OSlice{
		buf: new(bytes.Buffer),
	}
}

// 如果是结构体内部的一个项，而且不是指针，需要用到Init()
func (o *OSlice) Init() {
	if o.buf == nil {
		o.buf = new(bytes.Buffer)
	}
}

func (o *OSlice) Len() int {
	if len(o.regionList) != len(o.idList) {
		panic("regionList is not sync with idList")
	}
	return len(o.regionList)
}

func (o *OSlice) Less(i, j int) bool {
	return bytes.Compare(o.Query(o.idList[i]), o.Query(o.idList[j])) < 0
}

func (o *OSlice) Swap(i, j int) {
	o.idList[i], o.idList[j] = o.idList[j], o.idList[i]
}

func (o *OSlice) Sort() {
	if !o.isSorted {
		sort.Sort(o)
		o.isSorted = true
	}
}

// reserve <= 0 表示不预留空间
// 如果要求预留空间比实际cap-len大，也就直接返回实际可预留空间cap-len
func (o *OSlice) Shrink(reserve int) (reserved int) {
	length := o.buf.Len()
	capacity := o.buf.Cap()

	if reserve < 0 {
		reserve = 0
	}

	if capacity-length > reserve {
		src := o.buf.Bytes()[:length : length+reserve]
		o.buf = bytes.NewBuffer(src)
		return reserve
	}

	return capacity - length
}

func (o *OSlice) Append(words []byte) (id RegionID) {
	begin := o.buf.Len()

	_, err := o.buf.Write(words)
	if err != nil {
		log.Panicln("bytes.Buffer write []byte failed: ", err)
	}
<commit_msg>退回修改shrink错误<commit_after>package oslice

import (
	"bytes"
	"log"
	"sort"
)

type OSlice struct {
	buf        *bytes.Buffer
	regionList []int
	idList     []RegionID
	isSorted   bool
}

type RegionID int

func New() *OSlice {
	return &OSlice{
		buf: new(bytes.Buffer),
	}
}

// 如果是结构体内部的一个项，而且不是指针，需要用到Init()
func (o *OSlice) Init() {
	if o.buf == nil {
		o.buf = new(bytes.Buffer)
	}
}

func (o *OSlice) Len() int {
	if len(o.regionList) != len(o.idList) {
		panic("regionList is not sync with idList")
	}
	return len(o.regionList)
}

func (o *OSlice) Less(i, j int) bool {
	return bytes.Compare(o.Query(o.idList[i]), o.Query(o.idList[j])) < 0
}

func (o *OSlice) Swap(i, j int) {
	o.idList[i], o.idList[j] = o.idList[j], o.idList[i]
}

func (o *OSlice) Sort() {
	if !o.isSorted {
		sort.Sort(o)
		o.isSorted = true
	}
}

// reserve <= 0 表示不预留空间
// 如果要求预留空间比实际cap-len大，也就直接返回实际可预留空间cap-len
func (o *OSlice) Shrink(reserve int) (reserved int) {
	length := o.buf.Len()
	capacity := o.buf.Cap()

	if reserve < 0 {
		reserve = 0
	}

	if capacity-length > reserve {
		src := o.buf.Bytes()
		dst := make([]byte, len(src), len(src)+reserve)
		copy(dst, src)

		o.buf = bytes.NewBuffer(dst)
		return reserve
	}

	return capacity - length
}

func (o *OSlice) Append(words []byte) (id RegionID) {
	begin := o.buf.Len()

	_, err := o.buf.Write(words)
	if err != nil {
		log.Panicln("bytes.Buffer write []byte failed: ", err)
	}

{'fraction_non_alphanumeric': 0.17209302325581396, 'fraction_numerical': 0.0028622540250447226, 'mean_word_length': 3.4031496062992126, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1711063', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1300, 'n_words': 275}