<commit_before>package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	_ "github.com/mattn/go-sqlite3"
	"golang.org/x/text/runes"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

const (
	version = "0.1.5"
)

func NormalizeString(text string) string {
	t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
	s, _, _ := transform.String(t, text)
	return s
}

func rightPad2Len(str, pad string, lenght int) string {
	for {
		str += pad
		if utf8.RuneCountInString(str) > lenght {
			return str
		}
	}
}

func rightPad2LenNorm(s string, padStr string, overallLen int) string {
	s = NormalizeString(s)
	var padCountInt int
	padCountInt = 1 + ((overallLen - utf8.RuneCountInString(padStr)) / utf8.RuneCountInString(padStr))
	var retStr = s + strings.Repeat(padStr, padCountInt)
	return retStr[:overallLen]
}

func OutLess(output string) {
	var in io.WriteCloser
	var cmd *exec.Cmd
	os.Setenv("LESSCHARSET", "utf-8")
	os_name := runtime.GOOS
	switch os_name {
	case "windows":
		homedir, err := os.UserHomeDir()
		check(err)
		bin_less := fmt.Sprintf("%s\\scoop\\shims\\less.EXE", homedir)
		cmd = exec.Command(bin_less, "-S", "-R")
	case "darwin", "linux":
		cmd = exec.Command("/usr/bin/less", "-S", "-R")
	}
	in, _ = cmd.StdinPipe()
	cmd.Stdout = os.Stdout
	io.Copy(in, bytes.NewBufferString(output))
	in.Close()
	cmd.Run()
}

func containsStr(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
	for _, s := range slice {
		set[s] = struct{}{}
	}
	_, ok := set[item]
	return ok
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func updateDB() {
	timeout := time.Duration(5 * time.Second)
	client := http.Client{
		Timeout: timeout,<commit_msg>set env var linux/mac<commit_after>package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	_ "github.com/mattn/go-sqlite3"
	"golang.org/x/text/runes"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

const (
	version = "0.1.5"
)

func NormalizeString(text string) string {
	t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
	s, _, _ := transform.String(t, text)
	return s
}

func rightPad2Len(str, pad string, lenght int) string {
	for {
		str += pad
		if utf8.RuneCountInString(str) > lenght {
			return str
		}
	}
}

func rightPad2LenNorm(s string, padStr string, overallLen int) string {
	s = NormalizeString(s)
	var padCountInt int
	padCountInt = 1 + ((overallLen - utf8.RuneCountInString(padStr)) / utf8.RuneCountInString(padStr))
	var retStr = s + strings.Repeat(padStr, padCountInt)
	return retStr[:overallLen]
}

func OutLess(output string) {
	var in io.WriteCloser
	var cmd *exec.Cmd
	os_name := runtime.GOOS
	switch os_name {
	case "windows":
		homedir, err := os.UserHomeDir()
		check(err)
		bin_less := fmt.Sprintf("%s\\scoop\\shims\\less.EXE", homedir)
		cmd = exec.Command(bin_less, "-S", "-R")
	case "darwin", "linux":
		os.Setenv("LESSCHARSET", "utf-8")
		cmd = exec.Command("/usr/bin/less", "-S", "-R")
	}
	in, _ = cmd.StdinPipe()
	cmd.Stdout = os.Stdout
	io.Copy(in, bytes.NewBufferString(output))
	in.Close()
	cmd.Run()
}

func containsStr(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
	for _, s := range slice {
		set[s] = struct{}{}
	}
	_, ok := set[item]
	return ok
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func updateDB() {
	timeout := time.Duration(5 * time.Second)
	client := http.Client{
		Timeout: timeout,
{'fraction_non_alphanumeric': 0.16854550453670608, 'fraction_numerical': 0.0071487489689304375, 'mean_word_length': 3.942934782608696, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5570780', 'n_tokens_mistral': 1550, 'n_tokens_neox': 1491, 'n_words': 308}