<commit_before>//package levenshtein
package main

import (
	//"github.com/elazarl/hadoophelpers/go/lib/table"
	"strconv"
)

var (
	DeleteCost = 5
	ReplaceCost = 5
	AddCost    = 1
)

// translated from wikipedia Levenshtein Distance code snippet
func LevenshteinDistance(s, t string) int {
    min := func(x int, xs ...int) int {
	    for _, v := range xs {
		    if x > v {
			    x = v
		    }
	    }
	    return x
    }
    // degenerate cases
    if s == t {
	    return 0
    }
    if len(s) == 0 {
	    return len(t)
    }<commit_msg>build error in previous commit<commit_after>//package levenshtein
package main

/*import (
	"github.com/elazarl/hadoophelpers/go/lib/table"
	"strconv"
)*/

var (
	DeleteCost = 5
	ReplaceCost = 5
	AddCost    = 1
)

// translated from wikipedia Levenshtein Distance code snippet

// LevenshteinDistance calculate how many transformations do we
// need to apply to t, in order to make it an s.
// For example LevenshteinDistance("a", "b") = 1*ReplaceCost, since
// since we need one replace action to get from "b" to "a",
// LevenshteinDistance("a", "ab") is 1*DeleteCost since we need to
// delete one character to get from "ab" to "b".
func LevenshteinDistance(s, t string) int {
    min := func(x int, xs ...int) int {
	    for _, v := range xs {
		    if x > v {
			    x = v
		    }
	    }
	    return x
    }
    // degenerate cases
    if s == t {
	    return 0
    }
    if len(s) == 0 {
	    return len(t)
    }
{'fraction_non_alphanumeric': 0.11667810569663692, 'fraction_numerical': 0.008236101578586136, 'mean_word_length': 2.291196388261851, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1049160', 'n_tokens_mistral': 555, 'n_tokens_neox': 497, 'n_words': 168}