<commit_before>package ow_stats

import (
	"regexp"
	"strings"
	"strconv"
	"log"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"golang.org/x/text/runes"
	"unicode"
)

var spaceRegex = regexp.MustCompile(`[-\s]`)
var nonAlphaRegex = regexp.MustCompile(`\W`)
var normalizeRegex = regexp.MustCompile(`_{2,}`)
var hourRegex = regexp.MustCompile(`(?P<Val>[0-9]+) hours?`)
var minuteRegex = regexp.MustCompile(`(?P<Val>[0-9]+) minutes?`)
var secondRegex = regexp.MustCompile(`(?P<Val>[0-9]+(?:\.[0-9]+)?) seconds?`)
var percentRegex = regexp.MustCompile(`(?P<Val>[0-9]{1,3})\s?%`)
var pluralizerRegex = regexp.MustCompile(`(?P<Start>_|[^a-z]|^)(?P<Term>blow|boost|kill|assist|barrier|hit|multikill|elimination)(?P<End>$|[^a-z])`)

var t = transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)

func SanitizeKey(text string) string {
	text = strings.ToLower(text)
	text, _, _ = transform.String(t, text)<commit_msg>Fixed race exception on SanitizeKey<commit_after>package ow_stats

import (
	"regexp"
	"strings"
	"strconv"
	"log"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"golang.org/x/text/runes"
	"unicode"
)

var spaceRegex = regexp.MustCompile(`[-\s]`)
var nonAlphaRegex = regexp.MustCompile(`\W`)
var normalizeRegex = regexp.MustCompile(`_{2,}`)
var hourRegex = regexp.MustCompile(`(?P<Val>[0-9]+) hours?`)
var minuteRegex = regexp.MustCompile(`(?P<Val>[0-9]+) minutes?`)
var secondRegex = regexp.MustCompile(`(?P<Val>[0-9]+(?:\.[0-9]+)?) seconds?`)
var percentRegex = regexp.MustCompile(`(?P<Val>[0-9]{1,3})\s?%`)
var pluralizerRegex = regexp.MustCompile(`(?P<Start>_|[^a-z]|^)(?P<Term>blow|boost|kill|assist|barrier|hit|multikill|elimination)(?P<End>$|[^a-z])`)

func SanitizeKey(text string) string {
	var t = transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)

	text = strings.ToLower(text)
	text, _, _ = transform.String(t, text)
{'fraction_non_alphanumeric': 0.21827676240208876, 'fraction_numerical': 0.013577023498694517, 'mean_word_length': 6.603174603174603, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5570390', 'n_tokens_mistral': 832, 'n_tokens_neox': 766, 'n_words': 107}