<commit_before>package str

import (
	"strings"

	"golang.org/x/text/unicode/norm"
)

//RemoveWhiteSpace removes whitespace {'\n' '\t' ' ' '\r`} from a string.
//Note that this converts to runes and back to UTF-8, so RemoveWhiteSpace(s) == s
//for a non-whitespace string does not necessarially hold, since the code points may differ.
func RemoveWhiteSpace(s string) string {
	return RemoveRunes(s, '\n', ' ', '\r', '\t')
}

//NormEqual returns true if the NKFC normalized forms of both strings are equal.
func NormEqual(s, q string) bool {
	return NKFC(s) == NKFC(q)
}

//NormFoldEqual returns true if the casefolded, NKFC normalized forms of both strings are equal.
func NormFoldEqual(s, q string) bool {
	return strings.EqualFold(NKFC(s), NKFC(q))
}

//NKFC normalizes a string to it's NKFC form
func NKFC(s string) string {
	return norm.NFKC.String(s)
}

//NFKD normalizes a string to it's NFKD form
func NFKD(s string) string {
	return norm.NFKD.String(s)
}

//NFD normalizes a string to it's NFD form
func NFD(s string) string {
	return norm.NFD.String(s)
}

//NFC normalizes a string to it's NFC form
func NFC(s string) string {
	return norm.NFC.String(s)
}
<commit_msg>added RemoveDiacriticsNFC<commit_after>package str

import (
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

//RemoveWhiteSpace removes whitespace {'\n' '\t' ' ' '\r`} from a string.
//Note that this converts to runes and back to UTF-8, so RemoveWhiteSpace(s) == s
//for a non-whitespace string does not necessarially hold, since the code points may differ.
func RemoveWhiteSpace(s string) string {
	return RemoveRunes(s, '\n', ' ', '\r', '\t')
}

//NormEqual returns true if the NKFC normalized forms of both strings are equal.
func NormEqual(s, q string) bool {
	return NKFC(s) == NKFC(q)
}

//NormFoldEqual returns true if the casefolded, NKFC normalized forms of both strings are equal.
func NormFoldEqual(s, q string) bool {
	return strings.EqualFold(NKFC(s), NKFC(q))
}

//NKFC normalizes a string to it's NKFC form
func NKFC(s string) string {
	return norm.NFKC.String(s)
}

//NFKD normalizes a string to it's NFKD form
func NFKD(s string) string {
	return norm.NFKD.String(s)
}

//NFD normalizes a string to it's NFD form
func NFD(s string) string {
	return norm.NFD.String(s)
}

//NFC normalizes a string to it's NFC form
func NFC(s string) string {
	return norm.NFC.String(s)
}

func isNonSpacingMark(r rune) bool {
	return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
}

var diacriticRemover = transform.Chain(norm.NFD, transform.RemoveFunc(isNonSpacingMark), norm.NFC)

//RemoveDiacriticsNFC creates a copy of s with the diacritics removed. It also transforms it to NFC.
func RemoveDiacriticsNFC(s string) string {
	s, _, _ = transform.String(diacriticRemover, s)
	return s
}

{'fraction_non_alphanumeric': 0.11517952364024174, 'fraction_numerical': 0.0007109847138286527, 'mean_word_length': 4.172794117647059, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7510655', 'n_tokens_mistral': 1048, 'n_tokens_neox': 978, 'n_words': 341}