<commit_before>package chewing

import (
    "errors"
)

type PhraseBKForest struct {
    tree map[int]*PhraseBKTreeNode
}

type PhraseBKTreeNode struct {
    children map[int]*PhraseBKTreeNode
    phraseArrayItem *PhraseArrayItem
}

func newPhraseBKForest() (phraseBKForest *PhraseBKForest) {
    phraseBKForest = new(PhraseBKForest)
    phraseBKForest.tree = make(map[int]*PhraseBKTreeNode)
    return phraseBKForest
}

func newPhraseBKTreeNode(phraseArrayItem *PhraseArrayItem) (phraseBKTreeNode *PhraseBKTreeNode) {
    phraseBKTreeNode = new(PhraseBKTreeNode)
    phraseBKTreeNode.children = make(map[int]*PhraseBKTreeNode)
    phraseBKTreeNode.phraseArrayItem = phraseArrayItem
    return phraseBKTreeNode
}

func (this *PhraseBKForest) insert(phraseArrayItem *PhraseArrayItem) (err error) {
    length := len(phraseArrayItem.phoneSeq)
    if this.tree[length] == nil {
        this.tree[length] = newPhraseBKTreeNode(phraseArrayItem)
        return nil
    } else {
        return this.tree[length].insert(phraseArrayItem)
    }
}

func (this *PhraseBKTreeNode) insert(phraseArrayItem *PhraseArrayItem) (err error) {
    distance, err := calculateHammingDistance(this.phraseArrayItem.phoneSeq, phraseArrayItem.phoneSeq)
    if err != nil {
        return err
    }
    if distance == 0 {
        return errors.New("Duplicate phoneSeq insert")
    }

    if this.children[distance] == nil {
        this.children[distance] = newPhraseBKTreeNode(phraseArrayItem)
        return nil
    } else {
        return this.children[distance].insert(phraseArrayItem)
    }
}

func (this *PhraseBKForest) query(phoneSeq []uint16, threshold int) (phraseArrayItem []*PhraseArrayItem) {
    length := len(phoneSeq)
    if this.tree[length] == nil {
        return make([]*PhraseArrayItem, 0)
    }

    phraseArrayItem = make([]*PhraseArrayItem, 0, 1)
    result := make(chan *PhraseArrayItem)
    count := make(chan int)

    counter := 1
    go this.tree[length].query(phoneSeq, threshold, count, result)

    for counter > 0 {
        select {
        case res := <- result:
            phraseArrayItem = append(phraseArrayItem, res)
        case c := <- count:
            counter += c
        }
    }

    return phraseArrayItem
}

func (this *PhraseBKTreeNode) query(phoneSeq []uint16, threshold int, count chan<- int, result chan<- *PhraseArrayItem) {
    diff, err := calculateHammingDistance(phoneSeq, this.phraseArrayItem.phoneSeq)<commit_msg>Increase channel length<commit_after>package chewing

import (
    "errors"
)

type PhraseBKForest struct {
    tree map[int]*PhraseBKTreeNode
}

type PhraseBKTreeNode struct {
    children map[int]*PhraseBKTreeNode
    phraseArrayItem *PhraseArrayItem
}

func newPhraseBKForest() (phraseBKForest *PhraseBKForest) {
    phraseBKForest = new(PhraseBKForest)
    phraseBKForest.tree = make(map[int]*PhraseBKTreeNode)
    return phraseBKForest
}

func newPhraseBKTreeNode(phraseArrayItem *PhraseArrayItem) (phraseBKTreeNode *PhraseBKTreeNode) {
    phraseBKTreeNode = new(PhraseBKTreeNode)
    phraseBKTreeNode.children = make(map[int]*PhraseBKTreeNode)
    phraseBKTreeNode.phraseArrayItem = phraseArrayItem
    return phraseBKTreeNode
}

func (this *PhraseBKForest) insert(phraseArrayItem *PhraseArrayItem) (err error) {
    length := len(phraseArrayItem.phoneSeq)
    if this.tree[length] == nil {
        this.tree[length] = newPhraseBKTreeNode(phraseArrayItem)
        return nil
    } else {
        return this.tree[length].insert(phraseArrayItem)
    }
}

func (this *PhraseBKTreeNode) insert(phraseArrayItem *PhraseArrayItem) (err error) {
    distance, err := calculateHammingDistance(this.phraseArrayItem.phoneSeq, phraseArrayItem.phoneSeq)
    if err != nil {
        return err
    }
    if distance == 0 {
        return errors.New("Duplicate phoneSeq insert")
    }

    if this.children[distance] == nil {
        this.children[distance] = newPhraseBKTreeNode(phraseArrayItem)
        return nil
    } else {
        return this.children[distance].insert(phraseArrayItem)
    }
}

func (this *PhraseBKForest) query(phoneSeq []uint16, threshold int) (phraseArrayItem []*PhraseArrayItem) {
    length := len(phoneSeq)
    if this.tree[length] == nil {
        return make([]*PhraseArrayItem, 0)
    }

    phraseArrayItem = make([]*PhraseArrayItem, 0, 1)
    result := make(chan *PhraseArrayItem, 1000)
    count := make(chan int, 1000)

    counter := 1
    go this.tree[length].query(phoneSeq, threshold, count, result)

    for counter > 0 || len(result) > 0 {
        select {
        case res := <- result:
            phraseArrayItem = append(phraseArrayItem, res)
        case c := <- count:
            counter += c
        }
    }

    return phraseArrayItem
}

func (this *PhraseBKTreeNode) query(phoneSeq []uint16, threshold int, count chan<- int, result chan<- *PhraseArrayItem) {
    diff, err := calculateHammingDistance(phoneSeq, this.phraseArrayItem.phoneSeq)
{'fraction_non_alphanumeric': 0.1024133035895356, 'fraction_numerical': 0.005881160008111945, 'mean_word_length': 3.6396989651928506, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3493086', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1515, 'n_words': 323}