<commit_before>package misspell

import (
	"bufio"
	"bytes"
	"io"
	"regexp"
	"strings"
	"text/scanner"
)

func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func inArray(haystack []string, needle string) bool {
	for _, word := range haystack {
		if needle == word {
			return true
		}
	}
	return false
}

var wordRegexp = regexp.MustCompile(`[a-zA-Z0-9']+`)

// Diff is datastructure showing what changed in a single line
type Diff struct {
	Filename  string
	FullLine  string
	Line      int
	Column    int
	Original  string
	Corrected string
}

// Replacer is the main struct for spelling correction
type Replacer struct {
	Replacements []string
	Debug        bool
	engine       *StringReplacer
	corrected    map[string]string
}

// New creates a new default Replacer using the main rule list
func New() *Replacer {
	r := Replacer{
		Replacements: DictMain,
	}
	r.Compile()
	return &r
}

// RemoveRule deletes existings rules.
// TODO: make inplace to save memory
func (r *Replacer) RemoveRule(ignore []string) {
	newwords := make([]string, 0, len(r.Replacements))
	for i := 0; i < len(r.Replacements); i += 2 {
		if inArray(ignore, r.Replacements[i]) {
			continue
		}
		newwords = append(newwords, r.Replacements[i:i+2]...)
	}
	r.engine = nil
	r.Replacements = newwords
}

// AddRuleList appends new rules.
// Input is in the same form as Strings.Replacer: [ old1, new1, old2, new2, ....]
// Note: does not check for duplictes
func (r *Replacer) AddRuleList(additions []string) {
	r.engine = nil
	r.Replacements = append(r.Replacements, additions...)
}

// Compile compiles the rules.  Required before using the Replace functions
func (r *Replacer) Compile() {

	r.corrected = make(map[string]string, len(r.Replacements)/2)
	for i := 0; i < len(r.Replacements); i += 2 {
		r.corrected[r.Replacements[i]] = r.Replacements[i+1]
	}
	r.engine = NewStringReplacer(r.Replacements...)
}

/*
line1 and line2 are different
extract words from each line1

replace word -> newword
if word == new-word
  continue
if new-word in list of replacements
  continue
new word not original, and not in list of replacements
  some substring got mixed up.  UNdo
*/
func (r *Replacer) recheckLine(s string, lineNum int, buf io.Writer, next func(Diff)) {
	first := 0
	redacted := RemoveNotWords(s)

	idx := wordRegexp.FindAllStringIndex(redacted, -1)
	for _, ab := range idx {
		word := s[ab[0]:ab[1]]
		newword := r.engine.Replace(word)
		if newword == word {
			// no replacement done
			continue
		}
		if StringEqualFold(r.corrected[strings.ToLower(word)], newword) {
			// word got corrected into something we know
			io.WriteString(buf, s[first:ab[0]])
			io.WriteString(buf, newword)
			first = ab[1]
			next(Diff{
				FullLine:  s,
				Line:      lineNum,
				Original:  word,<commit_msg>Potential fix for issue #113<commit_after>package misspell

import (
	"bufio"
	"bytes"
	"io"
	"regexp"
	"strings"
	"text/scanner"
)

func max(x, y int) int {
	if x > y {
		return x
	}
	return y
}

func inArray(haystack []string, needle string) bool {
	for _, word := range haystack {
		if needle == word {
			return true
		}
	}
	return false
}

var wordRegexp = regexp.MustCompile(`[a-zA-Z0-9']+`)

// Diff is datastructure showing what changed in a single line
type Diff struct {
	Filename  string
	FullLine  string
	Line      int
	Column    int
	Original  string
	Corrected string
}

// Replacer is the main struct for spelling correction
type Replacer struct {
	Replacements []string
	Debug        bool
	engine       *StringReplacer
	corrected    map[string]string
}

// New creates a new default Replacer using the main rule list
func New() *Replacer {
	r := Replacer{
		Replacements: DictMain,
	}
	r.Compile()
	return &r
}

// RemoveRule deletes existings rules.
// TODO: make inplace to save memory
func (r *Replacer) RemoveRule(ignore []string) {
	newwords := make([]string, 0, len(r.Replacements))
	for i := 0; i < len(r.Replacements); i += 2 {
		if inArray(ignore, r.Replacements[i]) {
			continue
		}
		newwords = append(newwords, r.Replacements[i:i+2]...)
	}
	r.engine = nil
	r.Replacements = newwords
}

// AddRuleList appends new rules.
// Input is in the same form as Strings.Replacer: [ old1, new1, old2, new2, ....]
// Note: does not check for duplictes
func (r *Replacer) AddRuleList(additions []string) {
	r.engine = nil
	r.Replacements = append(r.Replacements, additions...)
}

// Compile compiles the rules.  Required before using the Replace functions
func (r *Replacer) Compile() {

	r.corrected = make(map[string]string, len(r.Replacements)/2)
	for i := 0; i < len(r.Replacements); i += 2 {
		r.corrected[r.Replacements[i]] = r.Replacements[i+1]
	}
	r.engine = NewStringReplacer(r.Replacements...)
}

/*
line1 and line2 are different
extract words from each line1

replace word -> newword
if word == new-word
  continue
if new-word in list of replacements
  continue
new word not original, and not in list of replacements
  some substring got mixed up.  UNdo
*/
func (r *Replacer) recheckLine(s string, lineNum int, buf io.Writer, next func(Diff)) {
	first := 0
	redacted := RemoveNotWords(s)

	idx := wordRegexp.FindAllStringIndex(redacted, -1)
	for _, ab := range idx {
		word := s[ab[0]:ab[1]]
		newword := r.engine.Replace(word)
		if newword == word {
			// no replacement done
			continue
		}

		// ignore camelCase words
		// https://github.com/client9/misspell/issues/113
		if CaseStyle(word) == CaseUnknown {
			continue
		}

		if StringEqualFold(r.corrected[strings.ToLower(word)], newword) {
			// word got corrected into something we know
			io.WriteString(buf, s[first:ab[0]])
			io.WriteString(buf, newword)
			first = ab[1]
			next(Diff{
				FullLine:  s,
				Line:      lineNum,
				Original:  word,
{'fraction_non_alphanumeric': 0.12109033723571554, 'fraction_numerical': 0.00926087716232745, 'mean_word_length': 3.457943925233645, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1584932', 'n_tokens_mistral': 2183, 'n_tokens_neox': 2034, 'n_words': 609}