<commit_before>package file

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
	"sync"

	"github.com/fatih/color"
	"github.com/nomad-software/grope/cli"
)

const MAX_NUMBER_OF_WORKERS = 100

type WorkerQueue struct {
	Group   *sync.WaitGroup
	Input   chan string
	Pattern *regexp.Regexp
	Closed  chan bool
	Output  *cli.Output
}

func (this *WorkerQueue) Start() {

	go this.Output.Start()

	life := make(chan bool)

	for i := 0; i <= MAX_NUMBER_OF_WORKERS; i++ {
		go this.worker(life)
	}

	for i := 0; i <= MAX_NUMBER_OF_WORKERS; i++ {
		<-life
	}

	close(this.Output.Console)
	<-this.Output.Closed

	this.Closed <- true
}

func (this *WorkerQueue) worker(death chan<- bool) {
	for fullPath := range this.Input {

		file, err := os.Open(fullPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, color.RedString(err.Error()))
			this.Group.Done()
			continue
		}

		lines := make([]cli.Line, 0)
		var lineNumber int

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			lineNumber++
			if this.Pattern.MatchString(scanner.Text()) {
				lines = append(lines, cli.Line{
					Number: lineNumber,
					Line:   this.Pattern.ReplaceAllString(scanner.Text(), color.New(color.FgHiRed, color.Bold).SprintFunc()("$0")),
				})
			}
		}

		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, color.RedString(err.Error()))
			this.Group.Done()
			continue
		}

		if len(lines) > 0 {
			this.Output.Console <- cli.Match{<commit_msg>Added better error message for scanner failures.<commit_after>package file

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
	"sync"

	"github.com/fatih/color"
	"github.com/nomad-software/grope/cli"
)

const MAX_NUMBER_OF_WORKERS = 100

type WorkerQueue struct {
	Group   *sync.WaitGroup
	Input   chan string
	Pattern *regexp.Regexp
	Closed  chan bool
	Output  *cli.Output
}

func (this *WorkerQueue) Start() {

	go this.Output.Start()

	life := make(chan bool)

	for i := 0; i <= MAX_NUMBER_OF_WORKERS; i++ {
		go this.worker(life)
	}

	for i := 0; i <= MAX_NUMBER_OF_WORKERS; i++ {
		<-life
	}

	close(this.Output.Console)
	<-this.Output.Closed

	this.Closed <- true
}

func (this *WorkerQueue) worker(death chan<- bool) {
	for fullPath := range this.Input {

		file, err := os.Open(fullPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, color.RedString(err.Error()))
			this.Group.Done()
			continue
		}

		lines := make([]cli.Line, 0)
		var lineNumber int

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			lineNumber++
			if this.Pattern.MatchString(scanner.Text()) {
				lines = append(lines, cli.Line{
					Number: lineNumber,
					Line:   this.Pattern.ReplaceAllString(scanner.Text(), color.New(color.FgHiRed, color.Bold).SprintFunc()("$0")),
				})
			}
		}

		if err := scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, color.RedString(fmt.Sprintf("Error scanning %s - %s", fullPath, err.Error())))
			this.Group.Done()
			continue
		}

		if len(lines) > 0 {
			this.Output.Console <- cli.Match{
{'fraction_non_alphanumeric': 0.15387228260869565, 'fraction_numerical': 0.005434782608695652, 'mean_word_length': 3.3955223880597014, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897641', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1098, 'n_words': 234}