<commit_before>package indexer

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/couchbase/indexing/secondary/collatejson"
	"github.com/couchbase/indexing/secondary/common"
)

var (
	ErrSecKeyNil = errors.New("Secondary key array is empty")
)

var (
	jsonEncoder *collatejson.Codec
	encBufPool  *common.BytesBufPool
)

const (
	maxIndexEntrySize = MAX_SEC_KEY_BUFFER_LEN + MAX_DOCID_LEN + 2
)

func init() {
	jsonEncoder = collatejson.NewCodec(16)
	encBufPool = common.NewByteBufferPool(maxIndexEntrySize)
}

// Generic index entry abstraction (primary or secondary)
// Represents a row in the index
type IndexEntry interface {
	ReadDocId([]byte) ([]byte, error)
	ReadSecKey([]byte) ([]byte, error)

	Bytes() []byte
	String() string
}

// Generic index key abstraction (primary or secondary)
// Represents a key supplied by the user for scan operation
type IndexKey interface {
	Compare(IndexEntry) int
	ComparePrefixFields(IndexEntry) int
	Bytes() []byte
	String() string
}

// Storage encoding for primary index entry
// Raw docid bytes are stored as the key
type primaryIndexEntry []byte

func NewPrimaryIndexEntry(docid []byte) (*primaryIndexEntry, error) {
	buf := append([]byte(nil), docid...)
	e := primaryIndexEntry(buf)
	return &e, nil
}

func BytesToPrimaryIndexEntry(b []byte) (*primaryIndexEntry, error) {
	e := primaryIndexEntry(b)
	return &e, nil
}

func (e *primaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	buf = append(buf, []byte(*e)...)
	return buf, nil
}

func (e *primaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	return nil, nil
}

func (e *primaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *primaryIndexEntry) String() string {
	return string(*e)
}

// Storage encoding for secondary index entry
// Format:
// [collate_json_encoded_sec_key][raw_docid_bytes][len_of_docid_2_bytes]
type secondaryIndexEntry []byte

func NewSecondaryIndexEntry(key []byte, docid []byte) (*secondaryIndexEntry, error) {
	var err error
	var buf []byte

	if isNilJsonKey(key) {
		return nil, ErrSecKeyNil
	}

	poolBuf := encBufPool.Get()
	defer encBufPool.Put(poolBuf)
	if buf, err = jsonEncoder.Encode(key, (*poolBuf)[:0]); err != nil {
		return nil, err
	}

	buf = append(buf, docid...)
	buf = buf[:len(buf)+2]
	offset := len(buf) - 2
	binary.LittleEndian.PutUint16(buf[offset:offset+2], uint16(len(docid)))

	buf = append([]byte(nil), buf[:len(buf)]...)
	e := secondaryIndexEntry(buf)
	return &e, nil
}

func BytesToSecondaryIndexEntry(b []byte) (*secondaryIndexEntry, error) {
	e := secondaryIndexEntry(b)
	return &e, nil
}

func (e *secondaryIndexEntry) lenDocId() int {
	rbuf := []byte(*e)
	offset := len(rbuf) - 2
	l := binary.LittleEndian.Uint16(rbuf[offset : offset+2])
	return int(l)
}

func (e *secondaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	rbuf := []byte(*e)
	doclen := e.lenDocId()
	offset := len(rbuf) - doclen - 2
	buf = append(buf, rbuf[offset:offset+doclen]...)

	return buf, nil
}

func (e *secondaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	var err error
	rbuf := []byte(*e)
	doclen := e.lenDocId()

	encoded := rbuf[0 : len(rbuf)-doclen-2]
	if buf, err = jsonEncoder.Decode(encoded, buf); err != nil {
		return nil, err
	}

	return buf, nil
}

func (e *secondaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *secondaryIndexEntry) String() string {
	buf := make([]byte, MAX_SEC_KEY_LEN*4)
	buf, _ = e.ReadSecKey(buf)
	buf = append(buf, ':')
	buf, _ = e.ReadDocId(buf)<commit_msg>MB-14611 index_entry: Minor code rearrangement<commit_after>package indexer

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/couchbase/indexing/secondary/collatejson"
	"github.com/couchbase/indexing/secondary/common"
)

var (
	ErrSecKeyNil = errors.New("Secondary key array is empty")
)

var (
	jsonEncoder *collatejson.Codec
	encBufPool  *common.BytesBufPool
)

const (
	maxIndexEntrySize = MAX_SEC_KEY_BUFFER_LEN + MAX_DOCID_LEN + 2
)

func init() {
	jsonEncoder = collatejson.NewCodec(16)
	encBufPool = common.NewByteBufferPool(maxIndexEntrySize)
}

// Generic index entry abstraction (primary or secondary)
// Represents a row in the index
type IndexEntry interface {
	ReadDocId([]byte) ([]byte, error)
	ReadSecKey([]byte) ([]byte, error)

	Bytes() []byte
	String() string
}

// Generic index key abstraction (primary or secondary)
// Represents a key supplied by the user for scan operation
type IndexKey interface {
	Compare(IndexEntry) int
	ComparePrefixFields(IndexEntry) int
	Bytes() []byte
	String() string
}

// Storage encoding for primary index entry
// Raw docid bytes are stored as the key
type primaryIndexEntry []byte

func NewPrimaryIndexEntry(docid []byte) (*primaryIndexEntry, error) {
	buf := append([]byte(nil), docid...)
	e := primaryIndexEntry(buf)
	return &e, nil
}

func BytesToPrimaryIndexEntry(b []byte) (*primaryIndexEntry, error) {
	e := primaryIndexEntry(b)
	return &e, nil
}

func (e *primaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	buf = append(buf, []byte(*e)...)
	return buf, nil
}

func (e *primaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	return nil, nil
}

func (e *primaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *primaryIndexEntry) String() string {
	return string(*e)
}

// Storage encoding for secondary index entry
// Format:
// [collate_json_encoded_sec_key][raw_docid_bytes][len_of_docid_2_bytes]
type secondaryIndexEntry []byte

func NewSecondaryIndexEntry(key []byte, docid []byte) (*secondaryIndexEntry, error) {
	var err error
	var buf []byte

	if isNilJsonKey(key) {
		return nil, ErrSecKeyNil
	}

	poolBuf := encBufPool.Get()
	defer encBufPool.Put(poolBuf)
	if buf, err = jsonEncoder.Encode(key, (*poolBuf)[:0]); err != nil {
		return nil, err
	}

	buf = append(buf, docid...)
	buf = buf[:len(buf)+2]
	offset := len(buf) - 2
	binary.LittleEndian.PutUint16(buf[offset:offset+2], uint16(len(docid)))

	buf = append([]byte(nil), buf[:len(buf)]...)
	e := secondaryIndexEntry(buf)
	return &e, nil
}

func BytesToSecondaryIndexEntry(b []byte) (*secondaryIndexEntry, error) {
	e := secondaryIndexEntry(b)
	return &e, nil
}

func (e *secondaryIndexEntry) lenDocId() int {
	rbuf := []byte(*e)
	offset := len(rbuf) - 2
	l := binary.LittleEndian.Uint16(rbuf[offset : offset+2])
	return int(l)
}

func (e secondaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	doclen := e.lenDocId()
	offset := len(e) - doclen - 2
	buf = append(buf, e[offset:offset+doclen]...)

	return buf, nil
}

func (e secondaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	var err error
	doclen := e.lenDocId()

	encoded := e[0 : len(e)-doclen-2]
	if buf, err = jsonEncoder.Decode(encoded, buf); err != nil {
		return nil, err
	}

	return buf, nil
}

func (e *secondaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *secondaryIndexEntry) String() string {
	buf := make([]byte, MAX_SEC_KEY_LEN*4)
	buf, _ = e.ReadSecKey(buf)
	buf = append(buf, ':')
	buf, _ = e.ReadDocId(buf)
{'fraction_non_alphanumeric': 0.1481374946066446, 'fraction_numerical': 0.006472026463397095, 'mean_word_length': 4.923339011925043, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4018705', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2463, 'n_words': 639}