<commit_before>//+build rocksdb

package rocksdb

import (
	"time"

	"github.com/tecbot/gorocksdb"

	"github.com/chrislusf/seaweedfs/weed/filer"
)

type TTLFilter struct {
	skipLevel0 bool
}

func NewTTLFilter() gorocksdb.CompactionFilter {
	return &TTLFilter{
		skipLevel0: true,
	}
}

func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []byte) {
	// decode could be slow, causing write stall
	// level >0 sst can run compaction in parallel
	if t.skipLevel0 && level == 0 {
		return false, val
	}
	entry := filer.Entry{}
	if err := entry.DecodeAttributesAndChunks(val); err == nil {
		if entry.TtlSec == 0 ||
			entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).After(time.Now()) {
			return false, val
		}
	}
	return true, nil
}

func (t *TTLFilter) Name() string {
	return "TTLFilter"
}
<commit_msg>ignore decode error for non-entry data<commit_after>//+build rocksdb

package rocksdb

import (
	"time"

	"github.com/tecbot/gorocksdb"

	"github.com/chrislusf/seaweedfs/weed/filer"
)

type TTLFilter struct {
	skipLevel0 bool
}

func NewTTLFilter() gorocksdb.CompactionFilter {
	return &TTLFilter{
		skipLevel0: true,
	}
}

func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []byte) {
	// decode could be slow, causing write stall
	// level >0 sst can run compaction in parallel
	if !t.skipLevel0 || level > 0 {
		entry := filer.Entry{}
		if err := entry.DecodeAttributesAndChunks(val); err == nil {
			if entry.TtlSec > 0 &&
				entry.Crtime.Add(time.Duration(entry.TtlSec)*time.Second).Before(time.Now()) {
				return true, nil
			}
		}
	}
	return false, val
}

func (t *TTLFilter) Name() string {
	return "TTLFilter"
}

{'fraction_non_alphanumeric': 0.133769322235434, 'fraction_numerical': 0.007134363852556481, 'mean_word_length': 3.9067055393586005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4606033', 'n_tokens_mistral': 661, 'n_tokens_neox': 626, 'n_words': 158}