<commit_before>package keyword

import (
	"strings"
)

type Keyword struct {
	sensitive bool
	keyword   []string
	exclude   []string
}

func New(keyword, exclude string, sensitive bool) *Keyword {
	//lower case all if no sensitive
	if !sensitive {
		keyword = strings.ToLower(keyword)
		exclude = strings.ToLower(exclude)
	}

	//split
	keywordArr := strings.Split(keyword, ",")
	excludeArr := strings.Split(exclude, ",")

	//trim
	keywordArr = trimArr(keywordArr)
	excludeArr = trimArr(excludeArr)

	return &Keyword{
		keyword:   keywordArr,
		exclude:   excludeArr,
		sensitive: sensitive,
	}
}

func (k *Keyword) CheckKeyword(str string) bool {
	if !k.sensitive {
		str = strings.ToLower(str)
	}

	return test(k.keyword, str)
}

func (k *Keyword) CheckExclude(str string) bool {
	if !k.sensitive {
		str = strings.ToLower(str)
	}

	return test(k.exclude, str)
}

func (k *Keyword) Check(str string) bool {
	return k.CheckKeyword(str) && !k.CheckExclude(str)
}

func test(strArr []string, str string) bool {
	for i := 0; i < len(strArr); i++ {
		if strings.Contains(str, strArr[i]) {
			return true
		}
	}

	return false
}

func trimArr(arr []string) (trimmedArr []string) {
	for i := 0; i < len(arr); i++ {
		str := strings.Trim(arr[i], " ")

		//remove empty string
		if str == "" {
			continue
		}

		trimmedArr = append(trimmedArr, str)
	}
	return
}<commit_msg>rename Keyword -> Checker<commit_after>package keyword

import (
	"strings"
)

type Checker struct {
	sensitive bool
	keyword   []string
	exclude   []string
}

func New(keyword, exclude string, sensitive bool) *Checker {
	//lower case all if no sensitive
	if !sensitive {
		keyword = strings.ToLower(keyword)
		exclude = strings.ToLower(exclude)
	}

	//split
	keywordArr := strings.Split(keyword, ",")
	excludeArr := strings.Split(exclude, ",")

	//trim
	keywordArr = trimArr(keywordArr)
	excludeArr = trimArr(excludeArr)

	return &Checker{
		keyword:   keywordArr,
		exclude:   excludeArr,
		sensitive: sensitive,
	}
}

func (k *Checker) CheckKeyword(str string) bool {
	if !k.sensitive {
		str = strings.ToLower(str)
	}

	return test(k.keyword, str)
}

func (k *Checker) CheckExclude(str string) bool {
	if !k.sensitive {
		str = strings.ToLower(str)
	}

	return test(k.exclude, str)
}

func (k *Checker) Check(str string) bool {
	return k.CheckKeyword(str) && !k.CheckExclude(str)
}

func test(strArr []string, str string) bool {
	for i := 0; i < len(strArr); i++ {
		if strings.Contains(str, strArr[i]) {
			return true
		}
	}

	return false
}

func trimArr(arr []string) (trimmedArr []string) {
	for i := 0; i < len(arr); i++ {
		str := strings.Trim(arr[i], " ")

		//remove empty string
		if str == "" {
			continue
		}

		trimmedArr = append(trimmedArr, str)
	}
	return
}
{'fraction_non_alphanumeric': 0.141399416909621, 'fraction_numerical': 0.0014577259475218659, 'mean_word_length': 3.7573656845753898, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3157231', 'n_tokens_mistral': 1058, 'n_tokens_neox': 985, 'n_words': 256}