<commit_before>package multiverse

import "strings"

type nGram struct {
	grams []string
	size  int
}

func newNGram(phrase string, size int) nGram {
	var n = nGram{nil, size}
	words := strings.Split(phrase, " ")

	for _, word := range words {
		for i := size; i <= len(word); i++ {
			n.grams = append(n.grams, word[i-size:i])
		}
	}
	return n
}

func (n nGram) Similarity(phrase string) float32 {
	result := float32(0)

	for i := 0; i < len(phrase)-n.size; i++ {
		for _, myGram := range n.grams {
			if phrase[i:i+n.size] == myGram {
				result += float32(n.size)
			}
		}
	}

	return result
}<commit_msg>Optimized nGram for 2- and 3-grams.<commit_after>package multiverse

import (
	"strings"
	"unsafe"
)

type nGram struct {
	original string
	grams    []string
	size     int
}

func newNGram(phrase string, size int) nGram {
	var n = nGram{phrase, nil, size}
	if size == 2 || size == 3 {
		return n
	}
	words := strings.Split(phrase, " ")

	for _, word := range words {
		for i := size; i <= len(word); i++ {
			n.grams = append(n.grams, word[i-size:i])
		}
	}
	return n
}

func (n nGram) Similarity(phrase string) float32 {
	switch n.size {
	case 2:
		return n.twoSimilarity(phrase)
	case 3:
		return n.threeSimilarity(phrase)
	default:
		return n.nSimilarity(phrase)
	}
}

func (n nGram) twoSimilarity(phrase string) float32 {
	term := *((*[]byte)(unsafe.Pointer(&phrase)))
	original := *((*[]byte)(unsafe.Pointer(&n.original)))

	oLen := len(original)
	tLen := len(term)

	result := 0

	i := 0
	j := 0

	for i < oLen-1 && (original[i] == ' ' || original[i+1] == ' ') {
		i++
	}

	for j < tLen-1 && (term[j] == ' ' || term[j+1] == ' ') {
		j++
	}

	jStart := j

	for ; i < oLen-1; i++ {
		if original[i+1] == ' ' {
			i++
			continue
		}
		for j := jStart; j < tLen-1; j++ {
			if term[j+1] == ' ' {
				j++
				continue
			}

			if original[i] == term[j] && original[i+1] == term[j+1] {
				result++
			}
		}
	}

	return float32(result * 2)
}

func (n nGram) threeSimilarity(phrase string) float32 {
	term := *((*[]byte)(unsafe.Pointer(&phrase)))
	original := *((*[]byte)(unsafe.Pointer(&n.original)))

	oLen := len(original)
	tLen := len(term)

	result := 0

	i := 0
	j := 0

	for i < oLen-2 && (original[i] == ' ' || original[i+1] == ' ' || original[i+2] == ' ') {
		i++
	}

	for j < tLen-2 && (term[j] == ' ' || term[j+1] == ' ' || term[j+2] == ' ') {
		j++
	}

	jStart := j

	for ; i < oLen-2; i++ {
		if original[i+2] == ' ' {
			i += 2
			continue
		}
		for j := jStart; j < tLen-2; j++ {
			if term[j+2] == ' ' {
				j += 2
				continue
			}

			if original[i] == term[j] && original[i+1] == term[j+1] && original[i+2] == term[j+2] {
				result++
			}
		}
	}

	return float32(result * 3)
}

func (n nGram) nSimilarity(phrase string) float32 {
	result := 0

	i := 0

spaceLoop:
	for i < len(phrase)-n.size {
		for j := 0; j < n.size; j++ {
			if phrase[i+j] == ' ' {
				i++
				continue spaceLoop
			}
		}
		break
	}

	for ; i < len(phrase)-n.size; i++ {
		if phrase[i+n.size-1] == ' ' {
			i += n.size - 1
			continue
		}
		for _, myGram := range n.grams {
			if phrase[i:i+n.size] == myGram {
				result++
			}
		}
	}

	return float32(result * n.size)
}
{'fraction_non_alphanumeric': 0.20063091482649842, 'fraction_numerical': 0.02113564668769716, 'mean_word_length': 2.734982332155477, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3666881', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1327, 'n_words': 311}