<commit_before>package indexer

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"

	"github.com/deepfabric/bkdtree"
	"github.com/deepfabric/indexer/cql"
	"github.com/deepfabric/pilosa"
	"github.com/pkg/errors"
)

//Conf originate from config file. It's comman to all indices.
type Conf struct {
	//BKD specific items
	T0mCap   int
	LeafCap  int
	IntraCap int
}

//Indexer shall be singleton
type Indexer struct {
	MainDir string //the main directory where stores all indices
	Conf    Conf   //indexer conf

	rwlock   sync.RWMutex                    //concurrent access of docProts, indices
	docProts map[string]*cql.DocumentWithIdx //index meta, need to persist
	indices  map[string]*Index               //index data, need to persist
}

//NewIndexer creates an Indexer.
func NewIndexer(mainDir string, conf *Conf, overwirte bool) (ir *Indexer, err error) {
	ir = &Indexer{
		MainDir: mainDir,
		Conf:    *conf,
	}
	if err = os.MkdirAll(mainDir, 0700); err != nil {
		err = errors.Wrap(err, "")
		return
	}
	if overwirte {
		ir.docProts = make(map[string]*cql.DocumentWithIdx)
		ir.indices = make(map[string]*Index)
		err = ir.removeIndices()
	} else {
		err = ir.Open()
	}
	return
}

//Destroy close and remove index files
func (ir *Indexer) Destroy() (err error) {
	if err = ir.Close(); err != nil {
		return
	}
	if err = ir.removeIndices(); err != nil {
		return
	}
	return
}

//Open opens all indices. Assumes ir.MainDir is already populated.
func (ir *Indexer) Open() (err error) {
	ir.rwlock.Lock()
	defer ir.rwlock.Unlock()
	if ir.indices != nil || ir.docProts != nil {
		panic("indexer already open")
	}
	ir.docProts = make(map[string]*cql.DocumentWithIdx)
	ir.indices = make(map[string]*Index)
	if err = ir.readMeta(); err != nil {
		return
	}
	var ind *Index
	for name, docProt := range ir.docProts {
		if ind, err = NewIndexExt(ir.MainDir, docProt.Index); err != nil {
			return
		}
		ir.indices[name] = ind
	}
	return
}

// Close close indexer
func (ir *Indexer) Close() (err error) {
	ir.rwlock.Lock()
	defer ir.rwlock.Unlock()
	for _, ind := range ir.indices {
		if err = ind.Close(); err != nil {
			return
		}
	}
	ir.indices = nil
	ir.docProts = nil
	return
}

// GetDocProt returns docProt of given index
func (ir *Indexer) GetDocProt(name string) (docProt *cql.DocumentWithIdx) {
	ir.rwlock.RLock()
	docProt, _ = ir.docProts[name]
	ir.rwlock.RUnlock()
	return
}

// CreateIndex creates index
func (ir *Indexer) CreateIndex(docProt *cql.DocumentWithIdx) (err error) {
	ir.rwlock.Lock()
	defer ir.rwlock.Unlock()
	if _, found := ir.docProts[docProt.Index]; found {
		panic("CreateIndex conflict with existing index")
	}
	if err = indexWriteConf(ir.MainDir, docProt); err != nil {
		return
	}
	var ind *Index
	if ind, err = NewIndex(docProt, ir.MainDir, ir.Conf.T0mCap, ir.Conf.LeafCap, ir.Conf.IntraCap); err != nil {
		return
	}
	ir.indices[docProt.Index] = ind
	ir.docProts[docProt.Index] = docProt
	return
}

//DestroyIndex destroy given index
func (ir *Indexer) DestroyIndex(name string) (err error) {
	ir.rwlock.Lock()
	defer ir.rwlock.Unlock()
	delete(ir.indices, name)
	delete(ir.docProts, name)
	err = ir.removeIndex(name)
	return
}

//Insert executes CqlInsert
func (ir *Indexer) Insert(doc *cql.DocumentWithIdx) (err error) {
	var ind *Index
	var found bool
	ir.rwlock.RLock()
	if ind, found = ir.indices[doc.Index]; !found {
		err = errors.Errorf("failed to insert %v to non-existing index %v", doc, doc.Index)
		ir.rwlock.RUnlock()
		return
	}
	ir.rwlock.RUnlock()
	err = ind.Insert(doc)
	return
}

//Del executes CqlDel.
func (ir *Indexer) Del(idxName string, docID uint64) (found bool, err error) {
	var ind *Index
	var fnd bool
	ir.rwlock.RLock()
	if ind, fnd = ir.indices[idxName]; !fnd {
		err = errors.Errorf("failed to delete %v from non-existing index %v", docID, idxName)
		ir.rwlock.RUnlock()
		return
	}
	ir.rwlock.RUnlock()
	found, err = ind.Del(docID)
	return
}

//Select executes CqlSelect.
func (ir *Indexer) Select(q *cql.CqlSelect) (rb *pilosa.Bitmap, err error) {
	var ind *Index
	var found bool
	ir.rwlock.RLock()
	if ind, found = ir.indices[q.Index]; !found {
		err = errors.Errorf("failed to select %v from non-existing index %v", q, q.Index)
		ir.rwlock.RUnlock()
		return
	}
	ir.rwlock.RUnlock()
	rb, err = ind.Select(q)
	return
}

//writeMeta persists Conf and DocProts to files.
func (ir *Indexer) writeMeta() (err error) {
	for _, docProt := range ir.docProts {
		if err = indexWriteConf(ir.MainDir, docProt); err != nil {
			return
		}
	}
	return
}

//readMeta parses Conf and DocProts from files.
func (ir *Indexer) readMeta() (err error) {
	var matches [][]string
	patt := `^index_(?P<name>[^.]+)\.json$`
	if matches, err = bkdtree.FilepathGlob(ir.MainDir, patt); err != nil {
		return
	}
	for _, match := range matches {<commit_msg>Made (*Indexer).Insert creates index  implictily<commit_after>package indexer

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"

	"github.com/deepfabric/bkdtree"
	"github.com/deepfabric/indexer/cql"
	"github.com/deepfabric/pilosa"
	"github.com/pkg/errors"
)

//Conf originate from config file. It's comman to all indices.
type Conf struct {
	//BKD specific items
	T0mCap   int
	LeafCap  int
	IntraCap int
}

//Indexer shall be singleton
type Indexer struct {
	MainDir string //the main directory where stores all indices
	Conf    Conf   //indexer conf

	rwlock   sync.RWMutex                    //concurrent access of docProts, indices
	docProts map[string]*cql.DocumentWithIdx //index meta, need to persist
	indices  map[string]*Index               //index data, need to persist
}

//NewIndexer creates an Indexer.
func NewIndexer(mainDir string, conf *Conf, overwirte bool) (ir *Indexer, err error) {
	ir = &Indexer{
		MainDir: mainDir,
		Conf:    *conf,
	}
	if err = os.MkdirAll(mainDir, 0700); err != nil {
		err = errors.Wrap(err, "")
		return
	}
	if overwirte {
		ir.docProts = make(map[string]*cql.DocumentWithIdx)
		ir.indices = make(map[string]*Index)
		err = ir.removeIndices()
	} else {
		err = ir.Open()
	}
	return
}

//Destroy close and remove index files
func (ir *Indexer) Destroy() (err error) {
	if err = ir.Close(); err != nil {
		return
	}
	if err = ir.removeIndices(); err != nil {
		return
	}
	return
}

//Open opens all indices. Assumes ir.MainDir is already populated.
func (ir *Indexer) Open() (err error) {
	ir.rwlock.Lock()
	defer ir.rwlock.Unlock()
	if ir.indices != nil || ir.docProts != nil {
		panic("indexer already open")
	}
	ir.docProts = make(map[string]*cql.DocumentWithIdx)
	ir.indices = make(map[string]*Index)
	if err = ir.readMeta(); err != nil {
		return
	}
	var ind *Index
	for name, docProt := range ir.docProts {
		if ind, err = NewIndexExt(ir.MainDir, docProt.Index); err != nil {
			return
		}
		ir.indices[name] = ind
	}
	return
}

// Close close indexer
func (ir *Indexer) Close() (err error) {
	ir.rwlock.Lock()
	defer ir.rwlock.Unlock()
	for _, ind := range ir.indices {
		if err = ind.Close(); err != nil {
			return
		}
	}
	ir.indices = nil
	ir.docProts = nil
	return
}

// GetDocProt returns docProt of given index
func (ir *Indexer) GetDocProt(name string) (docProt *cql.DocumentWithIdx) {
	ir.rwlock.RLock()
	docProt, _ = ir.docProts[name]
	ir.rwlock.RUnlock()
	return
}

// CreateIndex creates index
func (ir *Indexer) CreateIndex(docProt *cql.DocumentWithIdx) (err error) {
	ir.rwlock.Lock()
	err = ir.createIndex(docProt)
	ir.rwlock.Unlock()
	return
}

//DestroyIndex destroy given index
func (ir *Indexer) DestroyIndex(name string) (err error) {
	ir.rwlock.Lock()
	defer ir.rwlock.Unlock()
	delete(ir.indices, name)
	delete(ir.docProts, name)
	err = ir.removeIndex(name)
	return
}

//Insert executes CqlInsert. If the given index doesn't exist, create it before insertion.
func (ir *Indexer) Insert(doc *cql.DocumentWithIdx) (err error) {
	var ind *Index
	var found bool
	ir.rwlock.RLock()
	if ind, found = ir.indices[doc.Index]; !found {
		if err = ir.createIndex(doc); err != nil {
			ir.rwlock.RUnlock()
			return
		}
		ind, found = ir.indices[doc.Index]
	}
	ir.rwlock.RUnlock()
	err = ind.Insert(doc)
	return
}

//Del executes CqlDel. It's allowed that the given index doesn't exist.
func (ir *Indexer) Del(idxName string, docID uint64) (found bool, err error) {
	var ind *Index
	var fnd bool
	ir.rwlock.RLock()
	if ind, fnd = ir.indices[idxName]; !fnd {
		ir.rwlock.RUnlock()
		return
	}
	ir.rwlock.RUnlock()
	found, err = ind.Del(docID)
	return
}

//Select executes CqlSelect.
func (ir *Indexer) Select(q *cql.CqlSelect) (rb *pilosa.Bitmap, err error) {
	var ind *Index
	var found bool
	ir.rwlock.RLock()
	if ind, found = ir.indices[q.Index]; !found {
		err = errors.Errorf("failed to select %v from non-existing index %v", q, q.Index)
		ir.rwlock.RUnlock()
		return
	}
	ir.rwlock.RUnlock()
	rb, err = ind.Select(q)
	return
}

// createIndex creates index without holding the lock
func (ir *Indexer) createIndex(docProt *cql.DocumentWithIdx) (err error) {
	if _, found := ir.docProts[docProt.Index]; found {
		err = errors.New("CreateIndex conflict with existing index")
		return
	}
	if err = indexWriteConf(ir.MainDir, docProt); err != nil {
		return
	}
	var ind *Index
	if ind, err = NewIndex(docProt, ir.MainDir, ir.Conf.T0mCap, ir.Conf.LeafCap, ir.Conf.IntraCap); err != nil {
		return
	}
	ir.indices[docProt.Index] = ind
	ir.docProts[docProt.Index] = docProt
	return
}

//writeMeta persists Conf and DocProts to files.
func (ir *Indexer) writeMeta() (err error) {
	for _, docProt := range ir.docProts {
		if err = indexWriteConf(ir.MainDir, docProt); err != nil {
			return
		}
	}
	return
}

//readMeta parses Conf and DocProts from files.
func (ir *Indexer) readMeta() (err error) {
	var matches [][]string
	patt := `^index_(?P<name>[^.]+)\.json$`
	if matches, err = bkdtree.FilepathGlob(ir.MainDir, patt); err != nil {
		return
	}
	for _, match := range matches {
{'fraction_non_alphanumeric': 0.1360765940110002, 'fraction_numerical': 0.0016296598085149725, 'mean_word_length': 3.627238454288407, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1290976', 'n_tokens_mistral': 3807, 'n_tokens_neox': 3649, 'n_words': 1006}