<commit_before>package core

import (
	"strconv"
	"strings"
)

// SliceToString turns a slice of strings into a single string with space denotation.
func SliceToString(slices []string) string {
	var sl = ""
	for indx, val := range slices {
		if indx != len(slices)-1 {
			sl += val + " "
		} else {
			sl += val
		}
	}
	return sl
}

func SliceToHumanListing(slices []string) string {
	var sl = ""
	for indx, val := range slices {
		if indx == len(slices)-2 {
			sl += val + " and "
		} else if indx < len(slices)-2 {
			sl += val + ", "
		} else {
			sl += val
		}
	}
	return sl
}
<commit_msg>neigh<commit_after>package core

import (
	"strconv"
	"strings"
)

// SliceToString turns a slice of strings into a single string with space denotation.
func SliceToString(slices []string) string {
	var sl = ""
	for indx, val := range slices {
		if indx != len(slices)-1 {
			sl += val + " "
		} else {
			sl += val
		}
	}
	return sl
}

func SliceToHumanListing(slices []string) string {
	var sl = ""
	for indx, val := range slices {
		if indx == len(slices)-2 {
			sl += val + " and "
		} else if indx < len(slices)-2 {
			sl += val + ", "
		} else {
			sl += val
		}
	}
	return sl
}

func SanitizeArgs(slice []string) []string {
	var fslice []string = make([]string, 0)
	var current string
	var inquote bool
	for i := 0; i < len(slice); i++ {
		if strings.HasPrefix(slice[i], "\"") {
			current += slice[i][1:]
			inquote = true
			continue
		}

		if inquote {
			if strings.HasSuffix(slice[i], "\"") {
				current += slice[i][:len(slice[i])-1]
				fslice = append(fslice, slice[i])
				current = ""
				inquote = false
				continue
			}

			current += slice[i]
			continue
		}

		fslice = append(fslice, slice[i])
	}

	if current != "" {
		return slice
	}
	return fslice
}

{'fraction_non_alphanumeric': 0.15093273035613342, 'fraction_numerical': 0.005652911249293386, 'mean_word_length': 2.7579617834394905, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4691411', 'n_tokens_mistral': 757, 'n_tokens_neox': 675, 'n_words': 187}