<commit_before>package search

import (
	"bufio"
	"bytes"
	"context"

	"code.cloudfoundry.org/lager"

	"cred-alert/db"
	"cred-alert/gitclient"
	"cred-alert/sniff/matchers"
)

type Result struct {
	Owner      string
	Repository string

	Revision   string
	Path       string
	LineNumber int
	Location   int
	Length     int

	Content []byte
}

//go:generate counterfeiter . Searcher

type Searcher interface {
	SearchCurrent(ctx context.Context, logger lager.Logger, matcher matchers.Matcher) Results
}

type searcher struct {
	repoRepository db.RepositoryRepository
	looper         gitclient.Looper
}

func NewSearcher(repoRepository db.RepositoryRepository, looper gitclient.Looper) Searcher {
	return &searcher{
		repoRepository: repoRepository,
		looper:         looper,
	}
}

func (s *searcher) SearchCurrent(ctx context.Context, logger lager.Logger, matcher matchers.Matcher) Results {
	logger = logger.Session("search-current")

	searchResults := &searchResults{
		resultChan: make(chan Result, 1024),
		err:        nil,
	}

	activeRepos, err := s.repoRepository.Active()
	if err != nil {
		searchResults.fail(err)
		return searchResults
	}

	go func() {
		for _, repo := range activeRepos {
			logger = logger.WithData(lager.Data{
				"owner": repo.Owner,
				"repo": repo.Name,
			})

			select {
			case <-ctx.Done():
				searchResults.fail(ctx.Err())
				return
			default:
				s.looper.ScanCurrentState(repo.Path, func(sha string, path string, content []byte) {
					scanner := bufio.NewScanner(bytes.NewReader(content))

					lineNumber := 1

					for scanner.Scan() {
						line := scanner.Bytes()

						if match, start, end := matcher.Match(line); match {
							searchResults.resultChan <- Result{
								Owner:      repo.Owner,
								Repository: repo.Name,
								Revision:   sha,
								Path:       path,
								LineNumber: lineNumber,
								Location:   start,
								Length:     end - start,
								Content:    line,
							}
						}

						lineNumber++
					}

					if err := scanner.Err(); err != nil {
						if err == bufio.ErrTooLong {
							return
						}

						logger.Error("failed-to-search-file", err, lager.Data{
							"revision": sha,
							"path": path,
							"line-number": lineNumber,
						})
					}
				})
			}
		}

		searchResults.succeed()
	}()

	return searchResults
}

//go:generate counterfeiter . Results

type Results interface {
	C() <-chan Result
	Err() error
}

type searchResults struct {
	err        error
	resultChan chan Result
}

func (r *searchResults) succeed() {
	close(r.resultChan)
}<commit_msg>copy the line contents to avoid mismatches<commit_after>package search

import (
	"bufio"
	"bytes"
	"context"

	"code.cloudfoundry.org/lager"

	"cred-alert/db"
	"cred-alert/gitclient"
	"cred-alert/sniff/matchers"
)

type Result struct {
	Owner      string
	Repository string

	Revision   string
	Path       string
	LineNumber int
	Location   int
	Length     int

	Content []byte
}

//go:generate counterfeiter . Searcher

type Searcher interface {
	SearchCurrent(ctx context.Context, logger lager.Logger, matcher matchers.Matcher) Results
}

type searcher struct {
	repoRepository db.RepositoryRepository
	looper         gitclient.Looper
}

func NewSearcher(repoRepository db.RepositoryRepository, looper gitclient.Looper) Searcher {
	return &searcher{
		repoRepository: repoRepository,
		looper:         looper,
	}
}

func (s *searcher) SearchCurrent(ctx context.Context, logger lager.Logger, matcher matchers.Matcher) Results {
	logger = logger.Session("search-current")

	searchResults := &searchResults{
		resultChan: make(chan Result, 1024),
		err:        nil,
	}

	activeRepos, err := s.repoRepository.Active()
	if err != nil {
		searchResults.fail(err)
		return searchResults
	}

	go func() {
		for _, repo := range activeRepos {
			logger = logger.WithData(lager.Data{
				"owner": repo.Owner,
				"repo":  repo.Name,
			})

			select {
			case <-ctx.Done():
				searchResults.fail(ctx.Err())
				return
			default:
				s.looper.ScanCurrentState(repo.Path, func(sha string, path string, content []byte) {
					scanner := bufio.NewScanner(bytes.NewReader(content))

					lineNumber := 1

					for scanner.Scan() {
						line := make([]byte, len(scanner.Bytes()))
						copy(line, scanner.Bytes())

						if match, start, end := matcher.Match(line); match {
							searchResults.resultChan <- Result{
								Owner:      repo.Owner,
								Repository: repo.Name,
								Revision:   sha,
								Path:       path,
								LineNumber: lineNumber,
								Location:   start,
								Length:     end - start,
								Content:    line,
							}
						}

						lineNumber++
					}

					if err := scanner.Err(); err != nil {
						if err == bufio.ErrTooLong {
							return
						}

						logger.Error("failed-to-search-file", err, lager.Data{
							"revision":    sha,
							"path":        path,
							"line-number": lineNumber,
						})
					}
				})
			}
		}

		searchResults.succeed()
	}()

	return searchResults
}

//go:generate counterfeiter . Results

type Results interface {
	C() <-chan Result
	Err() error
}

type searchResults struct {
	err        error
	resultChan chan Result
}

func (r *searchResults) succeed() {
	close(r.resultChan)
}
{'fraction_non_alphanumeric': 0.11366685834770941, 'fraction_numerical': 0.0019168104274487254, 'mean_word_length': 2.700709219858156, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1536797', 'n_tokens_mistral': 2104, 'n_tokens_neox': 1650, 'n_words': 397}