<commit_before>package main

import (
	"regexp"
	"strings"
)

var isStrutilInitialized bool
var spacesRe *regexp.Regexp

func maybeInitStrutil() {
	if isStrutilInitialized {
		return
	}

	var err error
	spacesRe, err = regexp.Compile(`\s+`)
	if err != nil {
		panic(err)
	}
	isStrutilInitialized = true
}

func splitSpaces(s string) []string {
	maybeInitStrutil()
	s = strings.TrimSpace(s)
	if s == "" {
		return []string{}
	}
	return spacesRe.Split(s, -1)
}

func matchPattern(pat string, str string) bool {
	patPercentIndex := strings.IndexByte(pat, '%')
	if patPercentIndex < 0 {
		return pat == str
	}

	patPrefix := pat[:patPercentIndex]
	patSuffix := pat[patPercentIndex+1:]
	return strings.HasPrefix(str, patPrefix) && strings.HasSuffix(str, patSuffix)
}

func substPattern(pat string, repl string, str string) string {
	patPercentIndex := strings.IndexByte(pat, '%')
	if patPercentIndex < 0 {
		if str == pat {
			return repl
		} else {
			return str
		}
	}

	patPrefix := pat[:patPercentIndex]
	patSuffix := pat[patPercentIndex+1:]
	replPercentIndex := strings.IndexByte(repl, '%')
	if strings.HasPrefix(str, patPrefix) && strings.HasSuffix(str, patSuffix) {
		if replPercentIndex < 0 {
			return repl
		} else {
			return repl[:replPercentIndex] + str[patPercentIndex:len(str)-len(patSuffix)] + repl[replPercentIndex+1:]
		}
	}
	return str
}
<commit_msg>rewrite strutil<commit_after>package main

import (
	"regexp"
	"strings"
	"sync"
)

var strinit sync.Once
var spacesRe *regexp.Regexp

func initStrutil() {
	var err error
	spacesRe, err = regexp.Compile(`\s+`)
	if err != nil {
		panic(err)
	}
}

func splitSpaces(s string) []string {
	strinit.Do(initStrutil)
	s = strings.TrimSpace(s)
	if s == "" {
		return nil
	}
	return spacesRe.Split(s, -1)
}

func matchPattern(pat string, str string) bool {
	s := strings.SplitN(pat, "%", 2)
	if len(s) != 2 {
		return pat == str
	}
	return strings.HasPrefix(str, s[0]) && strings.HasSuffix(str, s[1])
}

func substPattern(pat string, repl string, str string) string {
	ps := strings.SplitN(pat, "%", 2)
	if len(ps) != 2 {
		if str == pat {
			return repl
		}
		return str
	}
	in := str
	trimed := str
	if ps[0] != "" {
		trimed = strings.TrimPrefix(in, ps[0])
		if trimed == in {
			return str
		}
	}
	in = trimed
	if ps[1] != "" {
		trimed = strings.TrimSuffix(in, ps[1])
		if trimed == in {
			return str
		}
	}

	rs := strings.SplitN(repl, "%", 2)
	if len(rs) != 2 {
		return repl
	}
	return rs[0] + trimed + rs[1]
}

{'fraction_non_alphanumeric': 0.14672594987873888, 'fraction_numerical': 0.00889248181083266, 'mean_word_length': 3.815175097276265, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3157621', 'n_tokens_mistral': 1004, 'n_tokens_neox': 947, 'n_words': 227}