<commit_before>// Copyright (c) 2012, <NAME> <<EMAIL>>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package block

import (
	"bytes"
	"encoding/binary"

	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/filter"
)

var (
	// Generate new filter every 2KB of data
	filterBaseLg byte = 11
	filterBase   int  = 1 << filterBaseLg
)

// FilterWriter represent filter block writer.
type FilterWriter struct {
	filter filter.Filter

	buf     *bytes.Buffer
	keys    [][]byte
	offsets []uint32
}

// NewFilterWriter create new initialized filter block writer.
func NewFilterWriter(filter filter.Filter) *FilterWriter {
	return &FilterWriter{
		filter: filter,
		buf:    new(bytes.Buffer),
	}
}

// Generate generate filter up to given offset.
func (b *FilterWriter) Generate(offset int) {
	idx := offset / filterBase
	for idx > len(b.offsets) {
		b.generateFilter()
	}
}

// Add add key to the filter block.
func (b *FilterWriter) Add(key []byte) {
	b.keys = append(b.keys, key)
}

// Finish finalize the filter block.
func (b *FilterWriter) Finish() []byte {
	if len(b.keys) > 0 {
		b.generateFilter()
	}

	// Append array of per-filter offsets
	offsetsOffset := uint32(b.buf.Len())
	for _, offset := range b.offsets {
		binary.Write(b.buf, binary.LittleEndian, offset)
	}

	binary.Write(b.buf, binary.LittleEndian, offsetsOffset)
	b.buf.WriteByte(filterBaseLg)

	return b.buf.Bytes()
}

func (b *FilterWriter) generateFilter() {
	b.offsets = append(b.offsets, uint32(b.buf.Len()))

	// fast path if there are no keys for this filter
	if len(b.keys) == 0 {
		return
	}

	// generate filter for current set of keys and append to buffer
	b.filter.CreateFilter(b.keys, b.buf)

	b.keys = nil
}

// FilterReader represent a filter block reader.
type FilterReader struct {
	filter filter.Filter
	buf    []byte

	baseLg       uint
	offsetsStart uint32
	length       uint

	ob []byte // offset buffer
}

// NewFilterReader create new initialized filter block reader.
func NewFilterReader(buf []byte, filter filter.Filter) (b *FilterReader, err error) {
	// 4 bytes for offset start and 1 byte for baseLg
	if len(buf) < 5 {
		err = errors.ErrCorrupt("filter block to short")
		return
	}

	offsetsStart := binary.LittleEndian.Uint32(buf[len(buf)-5:])
	if offsetsStart > uint32(len(buf))-5 {
		err = errors.ErrCorrupt("bad restart offset in filter block")
		return
	}

	b = &FilterReader{
		filter:       filter,
		buf:          buf,
		baseLg:       uint(buf[len(buf)-1]),
		offsetsStart: offsetsStart,
		length:       (uint(len(buf)) - 5 - uint(offsetsStart)) / 4,
		ob:           buf[offsetsStart : len(buf)-1],
	}
	return
}

// KeyMayMatch test whether given key at given offset may match.
func (b *FilterReader) KeyMayMatch(offset uint, key []byte) bool {
	idx := offset >> b.baseLg
	if idx < b.length {
		ob := b.ob[idx*4:]
		start := binary.LittleEndian.Uint32(ob)
		end := binary.LittleEndian.Uint32(ob[4:])
		if start <= end && end <= b.offsetsStart {
			filter := b.buf[start:end]
			return b.filter.KeyMayMatch(key, filter)
		} else if start == end {
			// Empty filters do not match any keys
			return false
		}
	}
	// Errors are treated as potential matches
	return true
}<commit_msg>Remove naked returns from leveldb/block/filter.go.<commit_after>// Copyright (c) 2012, <NAME> <<EMAIL>>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package block

import (
	"bytes"
	"encoding/binary"

	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/filter"
)

var (
	// Generate new filter every 2KB of data
	filterBaseLg byte = 11
	filterBase   int  = 1 << filterBaseLg
)

// FilterWriter represent filter block writer.
type FilterWriter struct {
	filter filter.Filter

	buf     *bytes.Buffer
	keys    [][]byte
	offsets []uint32
}

// NewFilterWriter create new initialized filter block writer.
func NewFilterWriter(filter filter.Filter) *FilterWriter {
	return &FilterWriter{
		filter: filter,
		buf:    new(bytes.Buffer),
	}
}

// Generate generate filter up to given offset.
func (b *FilterWriter) Generate(offset int) {
	idx := offset / filterBase
	for idx > len(b.offsets) {
		b.generateFilter()
	}
}

// Add add key to the filter block.
func (b *FilterWriter) Add(key []byte) {
	b.keys = append(b.keys, key)
}

// Finish finalize the filter block.
func (b *FilterWriter) Finish() []byte {
	if len(b.keys) > 0 {
		b.generateFilter()
	}

	// Append array of per-filter offsets
	offsetsOffset := uint32(b.buf.Len())
	for _, offset := range b.offsets {
		binary.Write(b.buf, binary.LittleEndian, offset)
	}

	binary.Write(b.buf, binary.LittleEndian, offsetsOffset)
	b.buf.WriteByte(filterBaseLg)

	return b.buf.Bytes()
}

func (b *FilterWriter) generateFilter() {
	b.offsets = append(b.offsets, uint32(b.buf.Len()))

	// fast path if there are no keys for this filter
	if len(b.keys) == 0 {
		return
	}

	// generate filter for current set of keys and append to buffer
	b.filter.CreateFilter(b.keys, b.buf)

	b.keys = nil
}

// FilterReader represent a filter block reader.
type FilterReader struct {
	filter filter.Filter
	buf    []byte

	baseLg       uint
	offsetsStart uint32
	length       uint

	ob []byte // offset buffer
}

// NewFilterReader create new initialized filter block reader.
func NewFilterReader(buf []byte, filter filter.Filter) (*FilterReader, error) {
	// 4 bytes for offset start and 1 byte for baseLg
	if len(buf) < 5 {
		return nil, errors.ErrCorrupt("filter block to short")
	}

	offsetsStart := binary.LittleEndian.Uint32(buf[len(buf)-5:])
	if offsetsStart > uint32(len(buf))-5 {
		return nil, errors.ErrCorrupt("bad restart offset in filter block")
	}

	return &FilterReader{
		filter:       filter,
		buf:          buf,
		baseLg:       uint(buf[len(buf)-1]),
		offsetsStart: offsetsStart,
		length:       (uint(len(buf)) - 5 - uint(offsetsStart)) / 4,
		ob:           buf[offsetsStart : len(buf)-1],
	}, nil
}

// KeyMayMatch test whether given key at given offset may match.
func (b *FilterReader) KeyMayMatch(offset uint, key []byte) bool {
	idx := offset >> b.baseLg
	if idx < b.length {
		ob := b.ob[idx*4:]
		start := binary.LittleEndian.Uint32(ob)
		end := binary.LittleEndian.Uint32(ob[4:])
		if start <= end && end <= b.offsetsStart {
			filter := b.buf[start:end]
			return b.filter.KeyMayMatch(key, filter)
		} else if start == end {
			// Empty filters do not match any keys
			return false
		}
	}
	// Errors are treated as potential matches
	return true
}
{'fraction_non_alphanumeric': 0.12267714156216951, 'fraction_numerical': 0.011179936546306089, 'mean_word_length': 3.3870112657389, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7284412', 'n_tokens_mistral': 2374, 'n_tokens_neox': 2222, 'n_words': 678}