<commit_before>package readline

import (
	"bufio"
	"container/list"
	"os"
	"strings"

	"github.com/chzyer/readline/runes"
)

type hisItem struct {
	Source  []rune
	Version int64
	Tmp     []rune
}

func (h *hisItem) Clean() {
	h.Source = nil
	h.Tmp = nil
}

type opHistory struct {
	cfg        *Config
	history    *list.List
	historyVer int64
	current    *list.Element
	fd         *os.File
}

func newOpHistory(cfg *Config) (o *opHistory) {
	o = &opHistory{
		cfg:     cfg,
		history: list.New(),
	}
	if o.cfg.HistoryFile != "" {
		o.historyUpdatePath(o.cfg.HistoryFile)
	}
	return
}

// only called by newOpHistory
func (o *opHistory) historyUpdatePath(path string) {
	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return
	}
	o.fd = f
	r := bufio.NewReader(o.fd)
	total := 0
	for ; ; total++ {
		line, err := r.ReadSlice('\n')
		if err != nil {
			break
		}
		o.PushHistory([]rune(strings.TrimSpace(string(line))))
		o.CompactHistory()
	}
	if total > o.cfg.HistoryLimit {
		o.HistoryRewrite()
	}
	o.historyVer++
	o.PushHistory(nil)
	return
}

func (o *opHistory) CompactHistory() {
	for o.history.Len() > o.cfg.HistoryLimit {
		o.history.Remove(o.history.Front())
	}
}

func (o *opHistory) HistoryRewrite() {
	if o.cfg.HistoryFile == "" {
		return
	}

	tmpFile := o.cfg.HistoryFile + ".tmp"
	fd, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0666)
	if err != nil {
		return
	}
	defer fd.Close()

	buf := bufio.NewWriter(fd)
	for elem := o.history.Front(); elem != nil; elem = elem.Next() {
		buf.WriteString(string(elem.Value.(*hisItem).Source))
	}
	buf.Flush()

	// replace history file
	if err = os.Rename(tmpFile, o.cfg.HistoryFile); err != nil {
		return
	}

	if o.fd != nil {
		o.fd.Close()
	}
	// fd is write only, just satisfy what we need.
	o.fd = fd
}

func (o *opHistory) CloseHistory() {
	if o.fd != nil {
		o.fd.Close()
	}
}
<commit_msg>fix history rewrite fd close<commit_after>package readline

import (
	"bufio"
	"container/list"
	"os"
	"strings"

	"github.com/chzyer/readline/runes"
)

type hisItem struct {
	Source  []rune
	Version int64
	Tmp     []rune
}

func (h *hisItem) Clean() {
	h.Source = nil
	h.Tmp = nil
}

type opHistory struct {
	cfg        *Config
	history    *list.List
	historyVer int64
	current    *list.Element
	fd         *os.File
}

func newOpHistory(cfg *Config) (o *opHistory) {
	o = &opHistory{
		cfg:     cfg,
		history: list.New(),
	}
	if o.cfg.HistoryFile != "" {
		o.historyUpdatePath(o.cfg.HistoryFile)
	}
	return
}

// only called by newOpHistory
func (o *opHistory) historyUpdatePath(path string) {
	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return
	}
	o.fd = f
	r := bufio.NewReader(o.fd)
	total := 0
	for ; ; total++ {
		line, err := r.ReadSlice('\n')
		if err != nil {
			break
		}
		o.PushHistory([]rune(strings.TrimSpace(string(line))))
		o.CompactHistory()
	}
	if total > o.cfg.HistoryLimit {
		o.HistoryRewrite()
	}
	o.historyVer++
	o.PushHistory(nil)
	return
}

func (o *opHistory) CompactHistory() {
	for o.history.Len() > o.cfg.HistoryLimit {
		o.history.Remove(o.history.Front())
	}
}

func (o *opHistory) HistoryRewrite() {
	if o.cfg.HistoryFile == "" {
		return
	}

	tmpFile := o.cfg.HistoryFile + ".tmp"
	fd, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0666)
	if err != nil {
		return
	}

	buf := bufio.NewWriter(fd)
	for elem := o.history.Front(); elem != nil; elem = elem.Next() {
		buf.WriteString(string(elem.Value.(*hisItem).Source))
	}
	buf.Flush()

	// replace history file
	if err = os.Rename(tmpFile, o.cfg.HistoryFile); err != nil {
		fd.Close()
		return
	}

	if o.fd != nil {
		o.fd.Close()
	}
	// fd is write only, just satisfy what we need.
	o.fd = fd
}

func (o *opHistory) CloseHistory() {
	if o.fd != nil {
		o.fd.Close()
	}
}

{'fraction_non_alphanumeric': 0.16139076284379866, 'fraction_numerical': 0.006746237675142709, 'mean_word_length': 3.1765980498374864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2033363', 'n_tokens_mistral': 1652, 'n_tokens_neox': 1573, 'n_words': 339}