<commit_before>package djhpkg

func Parse(in []string) (out []string) {
	for k, s := range in {
		if IsQuote(s[0]) {
			if IsQuote(s[len(s)-1]) {
				return append(append(in[:k], s[1:len(s)-1]), Parse(in[k+1:])...)
			}
			return append(in[:k], MergeQuote(s[1:], in[k+1:])...)
		}
	}
	return in
}

func MergeQuote(out string, in []string) []string {
	for k, s := range in {
		out += " " + s
		if IsQuote(out[len(out)-1]) {
			return append([]string{out[:len(out)-1]}, Parse(in[k+1:])...)
		}
	}
	panic("Unclosed Quote!")
}

func IsQuote(in byte) bool {
	return in == '`' || in == '"'
}<commit_msg>re-implemented my version for performance<commit_after>package djhpkg

const (
	QUOTE      = '"'
	SPACE      = " "
	SPACE_BYTE = ' '
)

func Parse(in []string) (out []string) {
	stringJoiner := make(joiner, 255)

	x := len(in)

	var k, j int
	for k, j = 0, 0; k < x; k, j = k+1, j+1 {
		in[j] = in[k]

		if in[k][0] == QUOTE {
			for i := k; k < x; k++ {
				if in[k][len(in[k])-1] == QUOTE {
					// stringJoiner.join is more efficient (in this case), but effectively equiv. to:
					// in[j] = trim(strings.Join(in[k:i+1], SPACE))
					in[j] = stringJoiner.join(in[i : k+1])
					break
				}
			}
		}
	}

	return in[:j]
}

type joiner []byte

// High Performance version strings.Join
func (j *joiner) join(a []string) string {
	if len(a) == 1 {
		return trim(a[0])
	}

	j.grow(sumlen(a))

	bp := copy(*j, a[0])
	for _, s := range a[1:] {
		(*j)[bp] = SPACE_BYTE
		bp += copy((*j)[bp+1:], s) + 1
	}

	return trim(string(*j))
}

func (j *joiner) grow(n int) {
	if cap(*j) < n {
		*j = make([]byte, n)
	}
	*j = (*j)[:n]
}

func sumlen(a []string) int {
	n := (len(a) - 1)
	for i := 0; i < len(a); i++ {
		n += len(a[i])
	}
	return n
}

func trim(in string) string { return in[1 : len(in)-1] }
{'fraction_non_alphanumeric': 0.22595078299776286, 'fraction_numerical': 0.017337807606263984, 'mean_word_length': 2.9844097995545655, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2822430', 'n_tokens_mistral': 871, 'n_tokens_neox': 793, 'n_words': 193}