<commit_before>package scommons

import (
	"bytes"

	"github.com/celrenheit/sandflake"
	"github.com/celrenheit/sandglass/sgproto"
	"github.com/celrenheit/sandglass/storage"
)

var (
	MsgPrefix = []byte("m")
	WalPrefix = []byte("w")
)

type StorageCommons struct {
	storage.Storage
}

func (s *StorageCommons) LastKeyForPrefix(prefix []byte) []byte {
	it := s.Iter(&storage.IterOptions{
		Reverse:     true,
		FetchValues: false,
	})
	if prefix == nil || len(prefix) == 0 {
		it.Rewind()
	} else {
		it.Seek(append(prefix, 0xFF, 0xFF, 0xFF))
	}

	if !it.ValidForPrefix(prefix) {
		return nil
	}

	return it.Item().Key
}

func (s *StorageCommons) LastKVForPrefix(prefix, suffix []byte) []byte {
	it := s.Iter(&storage.IterOptions{
		Reverse:     true,
		FetchValues: true,
	})

	// FIXME: use maxid instead of '~'
	if prefix == nil || len(prefix) == 0 {
		it.Rewind()
	} else {
		it.Seek(append(prefix, bytes.Repeat([]byte{0xFF}, 3)...))
	}

	for ; it.ValidForPrefix(prefix); it.Next() {
		if suffix == nil || bytes.HasSuffix(it.Item().Key, suffix) {
			return it.Item().Value
		}
	}

	return nil
}

func (s *StorageCommons) ForEach(fn func(msg *sgproto.Message) error) error {
	return s.ForRange(sandflake.Nil, sandflake.MaxID, fn)
}
<commit_msg>do not forget to close iterators<commit_after>package scommons

import (
	"bytes"

	"github.com/celrenheit/sandflake"
	"github.com/celrenheit/sandglass/sgproto"
	"github.com/celrenheit/sandglass/storage"
)

var (
	MsgPrefix = []byte("m")
	WalPrefix = []byte("w")
)

type StorageCommons struct {
	storage.Storage
}

func (s *StorageCommons) LastKeyForPrefix(prefix []byte) []byte {
	it := s.Iter(&storage.IterOptions{
		Reverse:     true,
		FetchValues: false,
	})
	defer it.Close()

	if prefix == nil || len(prefix) == 0 {
		it.Rewind()
	} else {
		it.Seek(append(prefix, 0xFF, 0xFF, 0xFF))
	}

	if !it.ValidForPrefix(prefix) {
		return nil
	}

	return it.Item().Key
}

func (s *StorageCommons) LastKVForPrefix(prefix, suffix []byte) []byte {
	it := s.Iter(&storage.IterOptions{
		Reverse:     true,
		FetchValues: true,
	})
	defer it.Close()

	// FIXME: use maxid instead of '~'
	if prefix == nil || len(prefix) == 0 {
		it.Rewind()
	} else {
		it.Seek(append(prefix, bytes.Repeat([]byte{0xFF}, 3)...))
	}

	for ; it.ValidForPrefix(prefix); it.Next() {
		if suffix == nil || bytes.HasSuffix(it.Item().Key, suffix) {
			return it.Item().Value
		}
	}

	return nil
}

func (s *StorageCommons) ForEach(fn func(msg *sgproto.Message) error) error {
	return s.ForRange(sandflake.Nil, sandflake.MaxID, fn)
}

{'fraction_non_alphanumeric': 0.16653574234092694, 'fraction_numerical': 0.0054988216810683424, 'mean_word_length': 3.9456310679611653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2066725', 'n_tokens_mistral': 1046, 'n_tokens_neox': 978, 'n_words': 203}