<commit_before>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"errors"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs/builder"
	"github.com/btcsuite/fastsha256"
)

const (
	// cfIndexName is the human-readable name for the index.
	cfIndexName = "committed filter index"
)

// Committed filters come in two flavours: basic and extended. They are
// generated and dropped in pairs, and both are indexed by a block's hash.
// Besides holding different content, they also live in different buckets.
var (
	// cfBasicIndexKey is the name of the db bucket used to house the
	// block hash -> basic cf index (cf#0).
	cfBasicIndexKey = []byte("cf0byhashidx")
	// cfBasicHeaderKey is the name of the db bucket used to house the
	// block hash -> basic cf header index (cf#0).
	cfBasicHeaderKey = []byte("cf0headerbyhashidx")
	// cfExtendedIndexKey is the name of the db bucket used to house the
	// block hash -> extended cf index (cf#1).
	cfExtendedIndexKey = []byte("cf1byhashidx")
	// cfExtendedHeaderKey is the name of the db bucket used to house the
	// block hash -> extended cf header index (cf#1).
	cfExtendedHeaderKey = []byte("cf1headerbyhashidx")
)

// dbFetchFilter retrieves a block's basic or extended filter. A filter's
// absence is not considered an error.
func dbFetchFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Get(h[:]), nil
}

// dbFetchFilterHeader retrieves a block's basic or extended filter header.
// A filter's absence is not considered an error.
func dbFetchFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(key)
	fh := idx.Get(h[:])
	if len(fh) != fastsha256.Size {
		return nil, errors.New("invalid filter header length")
	}
	return fh, nil
}

// dbStoreFilter stores a block's basic or extended filter.
func dbStoreFilter(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Put(h[:], f)
}

// dbStoreFilterHeader stores a block's basic or extended filter header.
func dbStoreFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash, fh []byte) error {
	if len(fh) != fastsha256.Size {
		return errors.New("invalid filter header length")
	}
	idx := dbTx.Metadata().Bucket(key)
	return idx.Put(h[:], fh)
}

// dbDeleteFilter deletes a filter's basic or extended filter.
func dbDeleteFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Delete(h[:])
}

// dbDeleteFilterHeader deletes a filter's basic or extended filter header.
func dbDeleteFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Delete(h[:])
}

// CfIndex implements a committed filter (cf) by hash index.
type CfIndex struct {
	db          database.DB
	chainParams *chaincfg.Params
}

// Ensure the CfIndex type implements the Indexer interface.
var _ Indexer = (*CfIndex)(nil)

// Init initializes the hash-based cf index. This is part of the Indexer
// interface.
func (idx *CfIndex) Init() error {
	return nil // Nothing to do.
}

// Key returns the database key to use for the index as a byte slice. This is
// part of the Indexer interface.
func (idx *CfIndex) Key() []byte {
	return cfBasicIndexKey
}

// Name returns the human-readable name of the index. This is part of the
// Indexer interface.
func (idx *CfIndex) Name() string {
	return cfIndexName
}

// Create is invoked when the indexer manager determines the index needs to
// be created for the first time. It creates buckets for the two hash-based cf
// indexes (simple, extended).
func (idx *CfIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()
	_, err := meta.CreateBucket(cfBasicIndexKey)
	if err != nil {
		return err
	}
	_, err = meta.CreateBucket(cfBasicHeaderKey)
	if err != nil {
		return err
	}
	_, err = meta.CreateBucket(cfExtendedIndexKey)
	if err != nil {
		return err
	}
	_, err = meta.CreateBucket(cfExtendedHeaderKey)
	if err != nil {
		return err
	}
	firstHeader := make([]byte, chainhash.HashSize)
	err = dbStoreFilterHeader(
		dbTx,
		cfBasicHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
	if err != nil {
		return err
	}
	err = dbStoreFilterHeader(
		dbTx,
		cfExtendedHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
	return err
}

// makeBasicFilterForBlock builds a block's basic filter, which consists of
// all outpoints and pkscript data pushes referenced by transactions within the
// block.
func makeBasicFilterForBlock(block *btcutil.Block) ([]byte, error) {
	b := builder.WithKeyHash(block.Hash())
	_, err := b.Key()
	if err != nil {
		return nil, err
	}
	for _, tx := range block.Transactions() {
		for _, txIn := range tx.MsgTx().TxIn {
			b.AddOutPoint(txIn.PreviousOutPoint)
		}
		for _, txOut := range tx.MsgTx().TxOut {
			b.AddScript(txOut.PkScript)
		}
	}
	f, err := b.Build()
	if err != nil {
		return nil, err
	}
	return f.Bytes(), nil
}

// makeExtendedFilterForBlock builds a block's extended filter, which consists
// of all tx hashes and sigscript data pushes contained in the block.
func makeExtendedFilterForBlock(block *btcutil.Block) ([]byte, error) {
	b := builder.WithKeyHash(block.Hash())
	_, err := b.Key()
	if err != nil {
		return nil, err
	}
	for _, tx := range block.Transactions() {
		b.AddHash(tx.Hash())
		for _, txIn := range tx.MsgTx().TxIn {
			b.AddScript(txIn.SignatureScript)
		}
	}
	f, err := b.Build()
	if err != nil {
		return nil, err
	}
	return f.Bytes(), nil
}

// makeHeaderForFilter implements the chaining logic between filters, where
// a filter's header is defined as sha256(sha256(filter) + previousFilterHeader).
func makeHeaderForFilter(f, pfh []byte) []byte {
	fhash := fastsha256.Sum256(f)
	chain := make([]byte, 0, 2*fastsha256.Size)
	chain = append(chain, fhash[:]...)
	chain = append(chain, pfh...)
	fh := fastsha256.Sum256(chain)
	return fh[:]
}

// storeFilter stores a given filter, and performs the steps needed to
// generate the filter's header.
func storeFilter(dbTx database.Tx, block *btcutil.Block, f []byte, extended bool) error {
	// Figure out which buckets to use.
	fkey := cfBasicIndexKey
	hkey := cfBasicHeaderKey
	if extended {
		fkey = cfExtendedIndexKey
		hkey = cfExtendedHeaderKey
	}<commit_msg>Skip TxIn on coinbase transaction when indexing<commit_after>// Copyright (c) 2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"errors"

	"github.com/btcsuite/btcd/blockchain"
	"github.com/btcsuite/btcd/chaincfg"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/database"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/gcs/builder"
	"github.com/btcsuite/fastsha256"
)

const (
	// cfIndexName is the human-readable name for the index.
	cfIndexName = "committed filter index"
)

// Committed filters come in two flavours: basic and extended. They are
// generated and dropped in pairs, and both are indexed by a block's hash.
// Besides holding different content, they also live in different buckets.
var (
	// cfBasicIndexKey is the name of the db bucket used to house the
	// block hash -> basic cf index (cf#0).
	cfBasicIndexKey = []byte("cf0byhashidx")
	// cfBasicHeaderKey is the name of the db bucket used to house the
	// block hash -> basic cf header index (cf#0).
	cfBasicHeaderKey = []byte("cf0headerbyhashidx")
	// cfExtendedIndexKey is the name of the db bucket used to house the
	// block hash -> extended cf index (cf#1).
	cfExtendedIndexKey = []byte("cf1byhashidx")
	// cfExtendedHeaderKey is the name of the db bucket used to house the
	// block hash -> extended cf header index (cf#1).
	cfExtendedHeaderKey = []byte("cf1headerbyhashidx")
)

// dbFetchFilter retrieves a block's basic or extended filter. A filter's
// absence is not considered an error.
func dbFetchFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Get(h[:]), nil
}

// dbFetchFilterHeader retrieves a block's basic or extended filter header.
// A filter's absence is not considered an error.
func dbFetchFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
	idx := dbTx.Metadata().Bucket(key)
	fh := idx.Get(h[:])
	if len(fh) != fastsha256.Size {
		return nil, errors.New("invalid filter header length")
	}
	return fh, nil
}

// dbStoreFilter stores a block's basic or extended filter.
func dbStoreFilter(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Put(h[:], f)
}

// dbStoreFilterHeader stores a block's basic or extended filter header.
func dbStoreFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash, fh []byte) error {
	if len(fh) != fastsha256.Size {
		return errors.New("invalid filter header length")
	}
	idx := dbTx.Metadata().Bucket(key)
	return idx.Put(h[:], fh)
}

// dbDeleteFilter deletes a filter's basic or extended filter.
func dbDeleteFilter(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Delete(h[:])
}

// dbDeleteFilterHeader deletes a filter's basic or extended filter header.
func dbDeleteFilterHeader(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
	idx := dbTx.Metadata().Bucket(key)
	return idx.Delete(h[:])
}

// CfIndex implements a committed filter (cf) by hash index.
type CfIndex struct {
	db          database.DB
	chainParams *chaincfg.Params
}

// Ensure the CfIndex type implements the Indexer interface.
var _ Indexer = (*CfIndex)(nil)

// Init initializes the hash-based cf index. This is part of the Indexer
// interface.
func (idx *CfIndex) Init() error {
	return nil // Nothing to do.
}

// Key returns the database key to use for the index as a byte slice. This is
// part of the Indexer interface.
func (idx *CfIndex) Key() []byte {
	return cfBasicIndexKey
}

// Name returns the human-readable name of the index. This is part of the
// Indexer interface.
func (idx *CfIndex) Name() string {
	return cfIndexName
}

// Create is invoked when the indexer manager determines the index needs to
// be created for the first time. It creates buckets for the two hash-based cf
// indexes (simple, extended).
func (idx *CfIndex) Create(dbTx database.Tx) error {
	meta := dbTx.Metadata()
	_, err := meta.CreateBucket(cfBasicIndexKey)
	if err != nil {
		return err
	}
	_, err = meta.CreateBucket(cfBasicHeaderKey)
	if err != nil {
		return err
	}
	_, err = meta.CreateBucket(cfExtendedIndexKey)
	if err != nil {
		return err
	}
	_, err = meta.CreateBucket(cfExtendedHeaderKey)
	if err != nil {
		return err
	}
	firstHeader := make([]byte, chainhash.HashSize)
	err = dbStoreFilterHeader(
		dbTx,
		cfBasicHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
	if err != nil {
		return err
	}
	err = dbStoreFilterHeader(
		dbTx,
		cfExtendedHeaderKey,
		&idx.chainParams.GenesisBlock.Header.PrevBlock,
		firstHeader,
	)
	return err
}

// makeBasicFilterForBlock builds a block's basic filter, which consists of
// all outpoints and pkscript data pushes referenced by transactions within the
// block.
func makeBasicFilterForBlock(block *btcutil.Block) ([]byte, error) {
	b := builder.WithKeyHash(block.Hash())
	_, err := b.Key()
	if err != nil {
		return nil, err
	}
	for i, tx := range block.Transactions() {
		// Skip the inputs for the coinbase transaction
		if i != 0 {
			for _, txIn := range tx.MsgTx().TxIn {
				b.AddOutPoint(txIn.PreviousOutPoint)
			}
		}
		for _, txOut := range tx.MsgTx().TxOut {
			b.AddScript(txOut.PkScript)
		}
	}
	f, err := b.Build()
	if err != nil {
		return nil, err
	}
	return f.Bytes(), nil
}

// makeExtendedFilterForBlock builds a block's extended filter, which consists
// of all tx hashes and sigscript data pushes contained in the block.
func makeExtendedFilterForBlock(block *btcutil.Block) ([]byte, error) {
	b := builder.WithKeyHash(block.Hash())
	_, err := b.Key()
	if err != nil {
		return nil, err
	}
	for i, tx := range block.Transactions() {
		b.AddHash(tx.Hash())
		// Skip the inputs for the coinbase transaction
		if i != 0 {
			for _, txIn := range tx.MsgTx().TxIn {
				b.AddScript(txIn.SignatureScript)
			}
		}
	}
	f, err := b.Build()
	if err != nil {
		return nil, err
	}
	return f.Bytes(), nil
}

// makeHeaderForFilter implements the chaining logic between filters, where
// a filter's header is defined as sha256(sha256(filter) + previousFilterHeader).
func makeHeaderForFilter(f, pfh []byte) []byte {
	fhash := fastsha256.Sum256(f)
	chain := make([]byte, 0, 2*fastsha256.Size)
	chain = append(chain, fhash[:]...)
	chain = append(chain, pfh...)
	fh := fastsha256.Sum256(chain)
	return fh[:]
}

// storeFilter stores a given filter, and performs the steps needed to
// generate the filter's header.
func storeFilter(dbTx database.Tx, block *btcutil.Block, f []byte, extended bool) error {
	// Figure out which buckets to use.
	fkey := cfBasicIndexKey
	hkey := cfBasicHeaderKey
	if extended {
		fkey = cfExtendedIndexKey
		hkey = cfExtendedHeaderKey
	}
{'fraction_non_alphanumeric': 0.11920627869095217, 'fraction_numerical': 0.006663705019991115, 'mean_word_length': 3.927763589930682, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6249670', 'n_tokens_mistral': 4599, 'n_tokens_neox': 4376, 'n_words': 1466}