<commit_before>'''
Created on Oct 24, 2013

@author: <NAME>
'''

from nltk.probability import FreqDist
import sys

costs = {'rem' : 1.0, 'sub' : 1.5, 'add' : 1.0, 'flp' : 1.5, 'nop' : 0.0}

class EditDistance(dict):
    def __init__(self, cost_add = costs['add'], cost_rem = costs['rem'],
            cost_sub = costs['sub'], cost_flp = costs['flp'],
            cost_nop = costs['nop']):
        self.cost_add = cost_add
        self.cost_rem = cost_rem
        self.cost_sub = cost_sub
        self.cost_flp = cost_flp
        self.cost_nop = cost_nop

    def __call__(self, *args):
        result = self[args]
        self.clear()
        return result

    def __missing__(self, key):
        result = self[key] = self._calc_edit_dist(*key)
        return result

    def avg_cost(self):
        costs= (self.cost_add, self.cost_flp, self.cost_nop, self.cost_rem, self.cost_sub)
        return reduce(lambda x, y: x+y, costs) / float(len(values))


    def _calc_edit_dist(self, str_src, str_tar):
        # Get the length of the strings.
        len_src = len(str_src)
        len_tar = len(str_tar)
        if len_src == 0:
            # If the source string is zero, then the transformation is going to be len_tar number of additions.
            return len_tar * self.cost_add

        elif len_tar == 0:
            # If the target string is zero, then the transformation is going to be len_src number of deletions.<commit_msg>Changed the name of EditDistance to EDCalc<commit_after>'''
Created on Oct 24, 2013

@author: <NAME>
'''

from nltk.probability import FreqDist
import sys

costs = {'rem' : 1.0, 'sub' : 1.5, 'add' : 1.0, 'flp' : 1.5, 'nop' : 0.0}

class EDCalc(dict):
    def __init__(self, cost_add = costs['add'], cost_rem = costs['rem'],
            cost_sub = costs['sub'], cost_flp = costs['flp'],
            cost_nop = costs['nop']):
        self.cost_add = cost_add
        self.cost_rem = cost_rem
        self.cost_sub = cost_sub
        self.cost_flp = cost_flp
        self.cost_nop = cost_nop

    def edit_distance(self, *args):
        result = self[args]
        self.clear()
        return result

    def __missing__(self, key):
        result = self[key] = self._calc_edit_dist(*key)
        return result

    def avg_cost(self):
        costs = (self.cost_add, self.cost_flp, self.cost_nop, self.cost_rem, self.cost_sub)
        return reduce(lambda x, y: x+y, costs) / float(len(values))


    def _calc_edit_dist(self, str_src, str_tar):
        # Get the length of the strings.
        len_src = len(str_src)
        len_tar = len(str_tar)

        if len_src == 0:
            # If the source string is zero, then the transformation is going to be len_tar number of additions.
            return len_tar * self.cost_add

        elif len_tar == 0:
            # If the target string is zero, then the transformation is going to be len_src number of deletions.
{'fraction_non_alphanumeric': 0.11294765840220386, 'fraction_numerical': 0.012396694214876033, 'mean_word_length': 2.4707287933094384, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3178172', 'n_tokens_mistral': 1010, 'n_tokens_neox': 954, 'n_words': 279}