<commit_before>package gospell

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// GoSpell is main struct
type GoSpell struct {
	WordChars string            // from AFF file
	ireplacer *strings.Replacer // input conversion
	Compounds []*regexp.Regexp
	Dict      map[string]struct{} // likely will contain some value later
	splitter  *Splitter
}

// InputConversion does any character substitution before checking
//  This is based on the ICONV stanza
func (s *GoSpell) InputConversion(raw []byte) string {
	sraw := string(raw)
	if s.ireplacer == nil {
		return sraw
	}
	return s.ireplacer.Replace(sraw)
}

// Split a text into Words
func (s *GoSpell) Split(text string) []string {
	return s.splitter.Split(text)
}

// Spell checks to see if a given word is in the internal dictionaries
// TODO: add multiple dictionaries
func (s *GoSpell) Spell(word string) bool {
	//log.Printf("Checking %s", word)
	_, ok := s.Dict[word]
	if ok {
		return true
	}
	if isNumber(word) {
		return true
	}
	if isNumberHex(word) {
		return true
	}
	// check compounds
	for _, pat := range s.Compounds {
		if pat.MatchString(word) {
			return true
		}
	}

	// ok maybe a word with units? e.g. 100GB
	units := isNumberUnits(word)
	if units == "" {
		// no, it's something else
		return false
	}

	// dictionary appears to have list of units
	_, ok = s.Dict[units]
	return ok
}

// NewGoSpellReader creates a speller from io.Readers for aff and dic
// Hunspell files
func NewGoSpellReader(aff, dic io.Reader) (*GoSpell, error) {
	affix, err := NewAFF(aff)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(dic)
	// get first line
	if !scanner.Scan() {
		return nil, scanner.Err()
	}
	line := scanner.Text()
	i, err := strconv.ParseInt(line, 10, 64)
	if err != nil {
		return nil, err
	}

	gs := GoSpell{
		WordChars: affix.WordChars,
		Dict:      make(map[string]struct{}, i*5),
		Compounds: make([]*regexp.Regexp, 0, len(affix.CompoundRule)),
		splitter:  NewSplitter(affix.WordChars),
	}

	words := []string{}
	for scanner.Scan() {
		line := scanner.Text()
		words, err = affix.Expand(line, words)
		if err != nil {
			// Need to support Compound rules
			//return nil, fmt.Errorf("Unable to process %q: %s", line, err)
			continue
		}

		if len(words) == 0 {
			//log.Printf("No words for %s", line)
			continue
		}

		style := CaseStyle(words[0])
		for _, word := range words {
			switch style {
			case AllLower:<commit_msg>Turn back on error checking of hunspell file<commit_after>package gospell

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// GoSpell is main struct
type GoSpell struct {
	WordChars string            // from AFF file
	ireplacer *strings.Replacer // input conversion
	Compounds []*regexp.Regexp
	Dict      map[string]struct{} // likely will contain some value later
	splitter  *Splitter
}

// InputConversion does any character substitution before checking
//  This is based on the ICONV stanza
func (s *GoSpell) InputConversion(raw []byte) string {
	sraw := string(raw)
	if s.ireplacer == nil {
		return sraw
	}
	return s.ireplacer.Replace(sraw)
}

// Split a text into Words
func (s *GoSpell) Split(text string) []string {
	return s.splitter.Split(text)
}

// Spell checks to see if a given word is in the internal dictionaries
// TODO: add multiple dictionaries
func (s *GoSpell) Spell(word string) bool {
	//log.Printf("Checking %s", word)
	_, ok := s.Dict[word]
	if ok {
		return true
	}
	if isNumber(word) {
		return true
	}
	if isNumberHex(word) {
		return true
	}
	// check compounds
	for _, pat := range s.Compounds {
		if pat.MatchString(word) {
			return true
		}
	}

	// ok maybe a word with units? e.g. 100GB
	units := isNumberUnits(word)
	if units == "" {
		// no, it's something else
		return false
	}

	// dictionary appears to have list of units
	_, ok = s.Dict[units]
	return ok
}

// NewGoSpellReader creates a speller from io.Readers for aff and dic
// Hunspell files
func NewGoSpellReader(aff, dic io.Reader) (*GoSpell, error) {
	affix, err := NewAFF(aff)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(dic)
	// get first line
	if !scanner.Scan() {
		return nil, scanner.Err()
	}
	line := scanner.Text()
	i, err := strconv.ParseInt(line, 10, 64)
	if err != nil {
		return nil, err
	}

	gs := GoSpell{
		WordChars: affix.WordChars,
		Dict:      make(map[string]struct{}, i*5),
		Compounds: make([]*regexp.Regexp, 0, len(affix.CompoundRule)),
		splitter:  NewSplitter(affix.WordChars),
	}

	words := []string{}
	for scanner.Scan() {
		line := scanner.Text()
		words, err = affix.Expand(line, words)
		if err != nil {
			return nil, fmt.Errorf("Unable to process %q: %s", line, err)
		}

		if len(words) == 0 {
			//log.Printf("No words for %s", line)
			continue
		}

		style := CaseStyle(words[0])
		for _, word := range words {
			switch style {
			case AllLower:
{'fraction_non_alphanumeric': 0.13006134969325153, 'fraction_numerical': 0.004498977505112475, 'mean_word_length': 3.25674499564839, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1908838', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1767, 'n_words': 540}