<commit_before>package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"text/template"

	"github.com/client9/misspell/lib"
)

var (
	defaultWrite *template.Template
	defaultRead  *template.Template
)

const (
	defaultWriteTmpl = `{{ .Filename }}:{{ .Line }} corrected "{{ js .Original }}" to "{{ js .Corrected }}"`
	defaultReadTmpl  = `{{ .Filename }}:{{ .Line }} found "{{ js .Original }}" a misspelling of "{{ js .Corrected }}"`
)

func init() {
	defaultWrite = template.Must(template.New("defaultWrite").Parse(defaultWriteTmpl))
	defaultRead = template.Must(template.New("defaultRead").Parse(defaultReadTmpl))
}

func worker(writeit bool, debug bool, mode string, files <-chan string, results chan<- int) {
	fails := 0
	for filename := range files {
		isGolang := strings.HasSuffix(filename, ".go")

		// ignore directories
		if f, err := os.Stat(filename); f.IsDir() && err == nil {
			continue
		}

		raw, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Printf("Unable to read %q: %s", filename, err)
			continue
		}
		orig := string(raw)
		var updated string

		// GROSS
		if mode == "go" || (mode == "auto" && isGolang) {
			updated = lib.ReplaceGo(orig, debug)
		} else if debug {
			updated = lib.ReplaceDebug(orig)
		} else {
			updated = lib.Replace(orig)
		}

		changes := lib.DiffLines(filename, orig, updated)
		if len(changes) == 0 {
			continue
		}
		for _, diff := range changes {
			// output can be done by doing multiple goroutines
			// and can clobber os.Stdout.
			//
			// the log package can be used simultaneously from multiple goroutines
			var output bytes.Buffer
			if writeit {
				defaultWrite.Execute(&output, diff)
			} else {
				defaultRead.Execute(&output, diff)
			}
			log.Println(output.String())
		}

		if writeit {
			ioutil.WriteFile(filename, []byte(updated), 0)
		}
	}
	results <- fails
}

func main() {
	log.SetFlags(0)

	workers := flag.Int("j", 0, "Number of workers, 0 = number of CPUs")
	writeit := flag.Bool("w", false, "Overwrite file with corrections (default is just to display)")
	format := flag.String("f", "", "Use Golang template for log message")
	ignores := flag.String("i", "", "Ignore the following corrections, comma separated")
	mode := flag.String("source", "auto", "Source mode: auto=guess, go=golang source, text=plain or markdown-like text")
	debug := flag.Bool("debug", false, "Debug matching, very slow")
	flag.Parse()
<commit_msg>make stdout logger that is goroutine safe<commit_after>package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"text/template"

	"github.com/client9/misspell/lib"
)

var (
	defaultWrite *template.Template
	defaultRead  *template.Template
	stdout       *log.Logger // see below in init()
)

const (
	defaultWriteTmpl = `{{ .Filename }}:{{ .Line }} corrected "{{ js .Original }}" to "{{ js .Corrected }}"`
	defaultReadTmpl  = `{{ .Filename }}:{{ .Line }} found "{{ js .Original }}" a misspelling of "{{ js .Corrected }}"`
)

func init() {
	defaultWrite = template.Must(template.New("defaultWrite").Parse(defaultWriteTmpl))
	defaultRead = template.Must(template.New("defaultRead").Parse(defaultReadTmpl))

	// we cant't just write to os.Stdout directly since we have multiple goroutine
	// all writing at the same time causing broken output.  Log is routine safe.
	// we see it so it doesn't use a prefix or include a time stamp.
	stdout = log.New(os.Stdout, "", 0)
}

func worker(writeit bool, debug bool, mode string, files <-chan string, results chan<- int) {
	fails := 0
	for filename := range files {
		isGolang := strings.HasSuffix(filename, ".go")

		// ignore directories
		if f, err := os.Stat(filename); f.IsDir() && err == nil {
			continue
		}

		raw, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Printf("Unable to read %q: %s", filename, err)
			continue
		}
		orig := string(raw)
		var updated string

		// GROSS
		if mode == "go" || (mode == "auto" && isGolang) {
			updated = lib.ReplaceGo(orig, debug)
		} else if debug {
			updated = lib.ReplaceDebug(orig)
		} else {
			updated = lib.Replace(orig)
		}

		changes := lib.DiffLines(filename, orig, updated)
		if len(changes) == 0 {
			continue
		}
		for _, diff := range changes {
			// output can be done by doing multiple goroutines
			// and can clobber os.Stdout.
			//
			// the log package can be used simultaneously from multiple goroutines
			var output bytes.Buffer
			if writeit {
				defaultWrite.Execute(&output, diff)
			} else {
				defaultRead.Execute(&output, diff)
			}

			// goroutine-safe print to os.Stdout
			stdout.Println(output.String())
		}

		if writeit {
			ioutil.WriteFile(filename, []byte(updated), 0)
		}
	}
	results <- fails
}

func main() {
	workers := flag.Int("j", 0, "Number of workers, 0 = number of CPUs")
	writeit := flag.Bool("w", false, "Overwrite file with corrections (default is just to display)")
	format := flag.String("f", "", "Use Golang template for log message")
	ignores := flag.String("i", "", "Ignore the following corrections, comma separated")
	mode := flag.String("source", "auto", "Source mode: auto=guess, go=golang source, text=plain or markdown-like text")
	debug := flag.Bool("debug", false, "Debug matching, very slow")
	flag.Parse()

{'fraction_non_alphanumeric': 0.15047619047619049, 'fraction_numerical': 0.0026666666666666666, 'mean_word_length': 3.614235500878735, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3880171', 'n_tokens_mistral': 1914, 'n_tokens_neox': 1725, 'n_words': 509}