<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vellum

import (
	"io"

	"github.com/willf/bitset"
)

// FST is an in-memory representation of a finite state transducer,
// capable of returning the uint64 value associated with
// each []byte key stored, as well as enumerating all of the keys
// in order.
type FST struct {
	f       io.Closer
	ver     int
	len     int
	typ     int
	data    []byte
	decoder decoder
}

func new(data []byte, f io.Closer) (rv *FST, err error) {
	rv = &FST{
		data: data,
		f:    f,
	}

	rv.ver, rv.typ, err = decodeHeader(data)
	if err != nil {
		return nil, err
	}

	rv.decoder, err = loadDecoder(rv.ver, rv.data)
	if err != nil {
		return nil, err
	}

	rv.len = rv.decoder.getLen()

	return rv, nil
}

// Contains returns true if this FST contains the specified key.
func (f *FST) Contains(val []byte) (bool, error) {
	_, exists, err := f.Get(val)
	return exists, err
}

// Get returns the value associated with the key.  NOTE: a value of zero
// does not imply the key does not exist, you must consult the second
// return value as well.
func (f *FST) Get(input []byte) (uint64, bool, error) {
	return f.get(input, nil)
}

func (f *FST) get(input []byte, prealloc fstState) (uint64, bool, error) {
	var total uint64
	curr := f.decoder.getRoot()
	state, err := f.decoder.stateAt(curr, prealloc)
	if err != nil {
		return 0, false, err
	}
	for i := range input {
		_, curr, output := state.TransitionFor(input[i])
		if curr == noneAddr {
			return 0, false, nil
		}<commit_msg>optimize FST.get() input slice access<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vellum

import (
	"io"

	"github.com/willf/bitset"
)

// FST is an in-memory representation of a finite state transducer,
// capable of returning the uint64 value associated with
// each []byte key stored, as well as enumerating all of the keys
// in order.
type FST struct {
	f       io.Closer
	ver     int
	len     int
	typ     int
	data    []byte
	decoder decoder
}

func new(data []byte, f io.Closer) (rv *FST, err error) {
	rv = &FST{
		data: data,
		f:    f,
	}

	rv.ver, rv.typ, err = decodeHeader(data)
	if err != nil {
		return nil, err
	}

	rv.decoder, err = loadDecoder(rv.ver, rv.data)
	if err != nil {
		return nil, err
	}

	rv.len = rv.decoder.getLen()

	return rv, nil
}

// Contains returns true if this FST contains the specified key.
func (f *FST) Contains(val []byte) (bool, error) {
	_, exists, err := f.Get(val)
	return exists, err
}

// Get returns the value associated with the key.  NOTE: a value of zero
// does not imply the key does not exist, you must consult the second
// return value as well.
func (f *FST) Get(input []byte) (uint64, bool, error) {
	return f.get(input, nil)
}

func (f *FST) get(input []byte, prealloc fstState) (uint64, bool, error) {
	var total uint64
	curr := f.decoder.getRoot()
	state, err := f.decoder.stateAt(curr, prealloc)
	if err != nil {
		return 0, false, err
	}
	for _, c := range input {
		_, curr, output := state.TransitionFor(c)
		if curr == noneAddr {
			return 0, false, nil
		}
{'fraction_non_alphanumeric': 0.11435406698564593, 'fraction_numerical': 0.00861244019138756, 'mean_word_length': 3.143706640237859, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6654344', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1359, 'n_words': 564}