<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	. "github.com/fuzzy/gocolor"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	Filled  int64
	Overlap int64
)

func ProgressIndicator(c int64, t int64) {
	rtime := (time.Now().Unix() - StartTime)
	fmt.Printf("%s %d of %d (%6.02f%%) runtime: %dsec @ %d/sec\r",
		String("Info").Green().Bold(),
		c,
		t,
		(float64(c)/float64(t))*100.00,
		rtime,
		(Filled / rtime))
}

func fillWorker(d chan map[string]string) {
	for {
		msg := <-d
		if msg["SRC"] == "__EXIT__" && msg["DST"] == "__EXIT__" {
			return
		} else {
			if isFile(msg["SRC"]) && isFile(msg["DST"]) {
				if !backfillFile(msg["SRC"], msg["DST"]) {
					Error.Printf("S:%s D:%s - Backfill operation failed.",
						msg["SRC"],
						msg["DST"])
					os.Exit(1)
				} else {
					Filled++
					ProgressIndicator(Filled, Overlap)
				}
			}
		}
	}
}<commit_msg>more informative progress output<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	. "github.com/fuzzy/gocolor"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	Filled  int64
	Overlap int64
)

func ProgressIndicator(c int64, t int64) {
	rtime := (time.Now().Unix() - StartTime)
	count := Filled
	if rtime > 0 {
		count = (Filled / rtime)
	}
	fmt.Printf("%s %d of %d (%6.02f%%) runtime: %dsec @ %d/sec\r",
		String("Info").Green().Bold(),
		c,
		t,
		(float64(c)/float64(t))*100.00,
		rtime,
		count)
}

func fillWorker(d chan map[string]string) {
	for {
		msg := <-d
		if msg["SRC"] == "__EXIT__" && msg["DST"] == "__EXIT__" {
			return
		} else {
			if isFile(msg["SRC"]) && isFile(msg["DST"]) {
				if !backfillFile(msg["SRC"], msg["DST"]) {
					Error.Printf("S:%s D:%s - Backfill operation failed.",
						msg["SRC"],
						msg["DST"])
					os.Exit(1)
				} else {
					Filled++
					ProgressIndicator(Filled, Overlap)
				}
			}
		}
	}
}
{'fraction_non_alphanumeric': 0.20483787956767885, 'fraction_numerical': 0.02213072568193515, 'mean_word_length': 3.127388535031847, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1227060', 'n_tokens_mistral': 962, 'n_tokens_neox': 841, 'n_words': 154}