<commit_before>package main

import (
	"bytes"
	"errors"
	"strings"
	"syscall"

	"net/url"

	"os"
	"os/exec"
	"os/user"

	"path/filepath"

	"io/ioutil"
)

var netRCBlock bool
var netRCBackup []byte

func EnterNetRCBlock(content string) {
	netRC_read := true
	oldNetRC, err := ioutil.ReadFile(ExpandPath(NETRCPATH))
	if err != nil {
		netRC_read = false
	}

	err = ioutil.WriteFile(ExpandPath(NETRCPATH), []byte(content), 0600)
	if err != nil {
		EXIT_ERROR("Cannot write to "+ExpandPath(NETRCPATH), EXIT_GIT_ERROR)
	}

	if netRC_read && len(oldNetRC) > 0 && !bytes.Equal([]byte(content), oldNetRC) {
		netRCBackup = oldNetRC
		netRCBlock = true
	} else {
		netRCBackup = nil
		netRCBlock = true
	}
}

func ExitNetRCBlock() {
	if !netRCBlock {
		return
	}

	if netRCBackup == nil {
		ioutil.WriteFile(ExpandPath(NETRCPATH), netRCBackup, 0600)
		netRCBlock = false
	} else {
		os.Remove(ExpandPath(NETRCPATH))
		netRCBlock = false
	}
}

func Contains(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
	for _, s := range slice {
		set[s] = struct{}{}
	}

	_, ok := set[item]
	return ok
}

func ParamIsSet(longArg string) bool {
	for _, s := range os.Args[1:] {
		if s[0:2] == "--" {
			if strings.ToLower(s[2:]) == strings.ToLower(longArg) {
				return true
			}
		}
	}

	return false
}

func ParamIsSet2(longArg string, shortArg string) bool {
	for _, s := range os.Args[1:] {
		if s[0:2] == "--" {
			if strings.ToLower(s[2:]) == strings.ToLower(longArg) {
				return true
			}
		} else if s[0:1] == "-" {
			if strings.ToLower(s[1:]) == strings.ToLower(shortArg) {
				return true
			}
		}
	}

	return false
}

func IsValidURL(uri string) bool {
	_, err := url.Parse(uri)
	if err != nil {
		return false
	}

	return true
}

func ExpandPath(path string) string {
	usr, err := user.Current()
	if err != nil {
		EXIT_ERROR("ERROR: Cannot read user home dir", EXIT_ERROR_INTERNAL)
	}

	if path[:2] == "~/" {
		path = filepath.Join(usr.HomeDir, path[2:])
	}

	return path
}

func InRange(needle rune, start rune, end rune) bool {
	return needle >= start && needle <= end
}

func NormalizeStringToFilePath(str string) string {
	var buffer bytes.Buffer

	for _, c := range str {
		if InRange(c, '0', '9') || InRange(c, 'A', 'Z') || InRange(c, 'a', 'z') || c == '-' || c == '.' {
			buffer.WriteRune(c)
		}
	}

	var fragment string
	fragment = buffer.String()
	fragment = strings.ToLower(fragment)
	fragment = strings.TrimSpace(fragment)

	for strings.ContainsRune(fragment, '.') {
		fragment = fragment[:strings.LastIndex(fragment, ".")]
	}

	return fragment
}
<commit_msg>only remove last dot in foldername (for .git extensions)<commit_after>package main

import (
	"bytes"
	"errors"
	"strings"
	"syscall"

	"net/url"

	"os"
	"os/exec"
	"os/user"

	"path/filepath"

	"io/ioutil"
)

var netRCBlock bool
var netRCBackup []byte

func EnterNetRCBlock(content string) {
	netRC_read := true
	oldNetRC, err := ioutil.ReadFile(ExpandPath(NETRCPATH))
	if err != nil {
		netRC_read = false
	}

	err = ioutil.WriteFile(ExpandPath(NETRCPATH), []byte(content), 0600)
	if err != nil {
		EXIT_ERROR("Cannot write to "+ExpandPath(NETRCPATH), EXIT_GIT_ERROR)
	}

	if netRC_read && len(oldNetRC) > 0 && !bytes.Equal([]byte(content), oldNetRC) {
		netRCBackup = oldNetRC
		netRCBlock = true
	} else {
		netRCBackup = nil
		netRCBlock = true
	}
}

func ExitNetRCBlock() {
	if !netRCBlock {
		return
	}

	if netRCBackup == nil {
		ioutil.WriteFile(ExpandPath(NETRCPATH), netRCBackup, 0600)
		netRCBlock = false
	} else {
		os.Remove(ExpandPath(NETRCPATH))
		netRCBlock = false
	}
}

func Contains(slice []string, item string) bool {
	set := make(map[string]struct{}, len(slice))
	for _, s := range slice {
		set[s] = struct{}{}
	}

	_, ok := set[item]
	return ok
}

func ParamIsSet(longArg string) bool {
	for _, s := range os.Args[1:] {
		if s[0:2] == "--" {
			if strings.ToLower(s[2:]) == strings.ToLower(longArg) {
				return true
			}
		}
	}

	return false
}

func ParamIsSet2(longArg string, shortArg string) bool {
	for _, s := range os.Args[1:] {
		if s[0:2] == "--" {
			if strings.ToLower(s[2:]) == strings.ToLower(longArg) {
				return true
			}
		} else if s[0:1] == "-" {
			if strings.ToLower(s[1:]) == strings.ToLower(shortArg) {
				return true
			}
		}
	}

	return false
}

func IsValidURL(uri string) bool {
	_, err := url.Parse(uri)
	if err != nil {
		return false
	}

	return true
}

func ExpandPath(path string) string {
	usr, err := user.Current()
	if err != nil {
		EXIT_ERROR("ERROR: Cannot read user home dir", EXIT_ERROR_INTERNAL)
	}

	if path[:2] == "~/" {
		path = filepath.Join(usr.HomeDir, path[2:])
	}

	return path
}

func InRange(needle rune, start rune, end rune) bool {
	return needle >= start && needle <= end
}

func NormalizeStringToFilePath(str string) string {
	var buffer bytes.Buffer

	for _, c := range str {
		if InRange(c, '0', '9') || InRange(c, 'A', 'Z') || InRange(c, 'a', 'z') || c == '-' || c == '.' {
			buffer.WriteRune(c)
		}
	}

	var fragment string
	fragment = buffer.String()
	fragment = strings.ToLower(fragment)
	fragment = strings.TrimSpace(fragment)

	if strings.ContainsRune(fragment, '.') {
		fragment = fragment[:strings.LastIndex(fragment, ".")]
	}

	return fragment
}

{'fraction_non_alphanumeric': 0.1586483390607102, 'fraction_numerical': 0.009545628102329133, 'mean_word_length': 3.661032028469751, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3412539', 'n_tokens_mistral': 2185, 'n_tokens_neox': 2037, 'n_words': 466}