<commit_before>package util

import (
	"crypto/md5"
	crand "crypto/rand"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	mrand "math/rand"
	"time"
)

func GetMD5(data []byte) string {
	return fmt.Sprintf("%x", md5.Sum(data))
}

func GetMD5ByIO(r io.Reader) string {
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		fmt.Println(err)
	}
	return GetMD5(bytes)
}

var (
	lunRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
	lnRunes  = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	random   = mrand.New(mrand.NewSource(time.Now().UnixNano()))
)

func RandomNumber(max int) int {
	return random.Int() % max
}

func RandomLUNStr(n int) string {
	return randomStr(n, lunRunes)
}

func RandomLNStr(n int) string {
	return randomStr(n, lnRunes)
}

func randomStr(n int, s []rune) string {
	buf := make([]byte, 0, n)
	for i := 0; i < n; i++ {
		buf = append(buf, byte(s[random.Int()%len(s)]))
	}
	return string(buf)
}

func SecureRandomStr(n int) string {
	k := make([]byte, n)
	if _, err := io.ReadFull(crand.Reader, k); err != nil {
		return ""
	}
	return base64.StdEncoding.EncodeToString(k)
}
<commit_msg>未使用関数削除<commit_after>package util

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	mrand "math/rand"
	"time"
)

func GetMD5(data []byte) string {
	return fmt.Sprintf("%x", md5.Sum(data))
}

func GetMD5ByIO(r io.Reader) string {
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		fmt.Println(err)
	}
	return GetMD5(bytes)
}

var (
	lunRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
	lnRunes  = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	random   = mrand.New(mrand.NewSource(time.Now().UnixNano()))
)

func RandomNumber(max int) int {
	return random.Int() % max
}

func RandomLUNStr(n int) string {
	return randomStr(n, lunRunes)
}

func randomStr(n int, s []rune) string {
	buf := make([]byte, 0, n)
	for i := 0; i < n; i++ {
		buf = append(buf, byte(s[random.Int()%len(s)]))
	}
	return string(buf)
}

{'fraction_non_alphanumeric': 0.16354430379746834, 'fraction_numerical': 0.029367088607594936, 'mean_word_length': 4.62962962962963, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2725157', 'n_tokens_mistral': 917, 'n_tokens_neox': 865, 'n_words': 170}