<commit_before>package main

import (
    "fmt"
    "github.com/arbovm/levenshtein"
    "bufio"
    "os"
    "strings"
)


func generateHash(path string) {
  inFile, _ := os.Open(path)
  defer inFile.Close()
  scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines) 
  
  for scanner.Scan() {
    s := strings.Replace(scanner.Text(),"/","",-1)
    partials, num := getPartials(s)
    if num>2 {
        for i := 0; i < num; i ++ {
            addToCache(partials[i],s)
        }
    }
  }
}


func addToCache(spartial string, s string) {
    f, err := os.OpenFile("cache/" + spartial, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
    if err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }

    defer f.Close()

    if _, err = f.WriteString(s+"\n"); err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }
}

func getPartials(s string) ([]string, int) { 
    partials := make([]string,1000)
    num := 0
    s = strings.Replace(s,"/","",-1)
    slen := len(s)
    if slen <= 4 {
       partials[num] = s
       num = num + 1
    } else {
        for i := 0; i <= slen-3; i ++ {
            partials[num] = s[i:i+3]
            num = num + 1
        }
    }
    return partials,num
}

func getMatch(s string) (string) {
    partials, num := getPartials(s)
    matches := make([]string,1000)
    numm := 0
    if num>2 {
        for i := 0; i < num; i ++ {
            // allPossibleMatches
        }
    }  
    match := "No match"
    return match
}


func main() {
    //generateHash("wordlist")
    match := getMatch("aardvark")
    fmt.Println("%v",match)
    s1 := "marcy playground"
    s2 := "mary playground"
    fmt.Printf("The distance between %v and %v is %v\n",
        s1, s2, levenshtein.Distance(s1, s2))
    // -> The distance between kitten and sitting is 3
}
<commit_msg>Workds<commit_after>package main

import (
    "fmt"
    "github.com/arbovm/levenshtein"
    "bufio"
    "os"
    "strings"
)


func generateHash(path string) {
  inFile, _ := os.Open(path)
  defer inFile.Close()
  scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines) 
  
  for scanner.Scan() {
    s := strings.Replace(scanner.Text(),"/","",-1)
    partials, num := getPartials(s)
    if num>2 {
        for i := 0; i < num; i ++ {
            addToCache(partials[i],s)
        }
    }
  }
}


func addToCache(spartial string, s string) {
    f, err := os.OpenFile("cache/" + spartial, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
    if err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }

    defer f.Close()

    if _, err = f.WriteString(s+"\n"); err != nil {
        fmt.Println("%v",spartial)
        panic(err)
    }
}

func stringInSlice(a string, list []string) bool {
    for _, b := range list {
        if b == a {
            return true
        }
    }
    return false
}

func getPartials(s string) ([]string, int) { 
    partials := make([]string,1000)
    num := 0
    s = strings.Replace(s,"/","",-1)
    slen := len(s)
    if slen <= 4 {
       partials[num] = s
       num = num + 1
    } else {
        for i := 0; i <= slen-4; i ++ {
            partials[num] = s[i:i+4]
            num = num + 1
        }
    }
    return partials,num
}

func getMatch(s string) (string) {
    match := "No match"
    partials, num := getPartials(s)
    matches := make([]string,10000)
    numm := 0
    for i := 0; i < num; i ++ {
    
      inFile, _ := os.Open("cache/"+partials[i])
      defer inFile.Close()
      scanner := bufio.NewScanner(inFile)
        scanner.Split(bufio.ScanLines) 
      
      for scanner.Scan() {
        if stringInSlice(scanner.Text(),matches) == false {
            matches[numm] = scanner.Text()
            numm = numm + 1
        }
      }
    
    }
    //fmt.Printf("%v",matches[0:numm])
    fmt.Printf("searching through %v matches\n",numm)
    bestLevenshtein := 1000
    for i := 0; i < numm; i ++ {
      d := levenshtein.Distance(s, matches[i])
      if (d < bestLevenshtein) {
        bestLevenshtein = d
        match = matches[i]
      }
    }
    return match
}


func main() {
    //generateHash("wordlist")
    match := getMatch("madgascar")
    fmt.Printf("Match: %v\n",match)
    s1 := "marcy playground"
    s2 := "mary playground"
    fmt.Printf("The distance between %v and %v is %v\n",
        s1, s2, levenshtein.Distance(s1, s2))
    // -> The distance between kitten and sitting is 3
}

{'fraction_non_alphanumeric': 0.1604882459312839, 'fraction_numerical': 0.016500904159132006, 'mean_word_length': 2.077190542420028, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4271837', 'n_tokens_mistral': 1712, 'n_tokens_neox': 1631, 'n_words': 357}