<commit_before>package searcher

import (
	"fmt"
	"hound/config"
	"hound/git"
	"hound/index"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	Repo *config.Repo
}

func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

func expungeOldIndexes(sha, gitDir string) error {
	// TODO(knorton): This is a bandaid for issue #14, but should suffice
	// since people don't usually name their repos with 40 char hashes. In
	// the longer term, I want to remove this naming scheme to support
	// rebuilds of the current hash.
	pat := regexp.MustCompile("-[0-9a-f]{40}$")

	name := fmt.Sprintf("%s-%s", filepath.Base(gitDir), sha)

	dirs, err := filepath.Glob(fmt.Sprintf("%s-*", gitDir))
	if err != nil {
		return err
	}

	for _, dir := range dirs {
		if !pat.MatchString(dir) {
			continue
		}

		if strings.HasSuffix(dir, name) {
			continue
		}

		if err := os.RemoveAll(dir); err != nil {
			return err
		}
	}

	return nil
}

func buildAndOpenIndex(sha, gitDir string) (*index.Index, error) {
	idxDir := fmt.Sprintf("%s-%s", gitDir, sha)
	if _, err := os.Stat(idxDir); err != nil {
		_, err := index.Build(idxDir, gitDir)
		if err != nil {
			return nil, err
		}
	}

	return index.Open(idxDir)
}

func reportOnMemory() {
	var ms runtime.MemStats<commit_msg>Second attempt to bandaid #14. Sorry about that.<commit_after>package searcher

import (
	"fmt"
	"hound/config"
	"hound/git"
	"hound/index"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sync"
	"time"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	Repo *config.Repo
}

func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

func expungeOldIndexes(sha, gitDir string) error {
	// TODO(knorton): This is a bandaid for issue #14, but should suffice
	// since people don't usually name their repos with 40 char hashes. In
	// the longer term, I want to remove this naming scheme to support
	// rebuilds of the current hash.
	pat := regexp.MustCompile("-[0-9a-f]{40}$")

	name := fmt.Sprintf("%s-%s", filepath.Base(gitDir), sha)

	dirs, err := filepath.Glob(fmt.Sprintf("%s-*", gitDir))
	if err != nil {
		return err
	}

	for _, dir := range dirs {
		bn := filepath.Base(dir)
		if !pat.MatchString(bn) || len(bn) != len(name) {
			continue
		}

		if bn == name {
			continue
		}

		if err := os.RemoveAll(dir); err != nil {
			return err
		}
	}

	return nil
}

func buildAndOpenIndex(sha, gitDir string) (*index.Index, error) {
	idxDir := fmt.Sprintf("%s-%s", gitDir, sha)
	if _, err := os.Stat(idxDir); err != nil {
		_, err := index.Build(idxDir, gitDir)
		if err != nil {
			return nil, err
		}
	}

	return index.Open(idxDir)
}

func reportOnMemory() {
	var ms runtime.MemStats
{'fraction_non_alphanumeric': 0.15489989462592202, 'fraction_numerical': 0.004741833508956797, 'mean_word_length': 3.619221411192214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6694811', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1478, 'n_words': 383}