<commit_before>import math
import random
from itertools import combinations
import Util
import Individual


class LTGA(object):
    def getMaskValue(self, individual, mask):
        return tuple(individual.genes[g] for g in mask)

    def setMaskValues(self, individual, mask, value):
        for valueIndex, geneIndex in enumerate(mask):
            individual.genes[geneIndex] = value[valueIndex]

    def entropy(self, mask, lookup):
        try:
            return lookup[mask]
        except KeyError:
            occurances = {}
            for individual in self.individuals:
                # extract the gene values for the cluster
                value = self.getMaskValue(individual, mask)
                try:
                    occurances[value] += 1
                except KeyError:
                    occurances[value] = 1
            total = float(len(self.individuals))
            result = -sum(x / total * math.log(x / total, 2)
                          for x in occurances.itervalues())
            lookup[mask] = result
            return result

    def clusterDistance(self, c1, c2, lookup):
        try:
            return lookup[c1, c2]
        except KeyError:
            try:
                result = 2 - ((self.entropy(c1, lookup) +
                               self.entropy(c2, lookup))
                              / self.entropy(c1 + c2, lookup))
            except ZeroDivisionError:
                result = 2  # Zero divison only happens in 0/0
            lookup[c1, c2] = result
            lookup[c2, c1] = result
            return result

    def pairwiseDistance(self, c1, c2, lookup):
        try:
            return lookup[c1, c2]
        # averages the pairwise distance between each cluster
        except KeyError:
            result = sum(self.clusterDistance((a,), (b,), lookup)
                         for a in c1 for b in c2) / float(len(c1) * len(c2))
            lookup[c1, c2] = result
            lookup[c2, c1] = result
            return result

    def buildTree(self, distance):
        clusters = [(i,) for i in xrange(len(self.individuals[0].genes))]
        subtrees = [(i,) for i in xrange(len(self.individuals[0].genes))]
        random.shuffle(clusters)
        random.shuffle(subtrees)
        lookup = {}

        def allLowest():
            minVal = 3  # Max possible distance should be 2
            results = []
            for c1, c2 in combinations(clusters, 2):<commit_msg>Spelling Error<commit_after>import math
import random
from itertools import combinations
import Util
import Individual


class LTGA(object):
    def getMaskValue(self, individual, mask):
        return tuple(individual.genes[g] for g in mask)

    def setMaskValues(self, individual, mask, value):
        for valueIndex, geneIndex in enumerate(mask):
            individual.genes[geneIndex] = value[valueIndex]

    def entropy(self, mask, lookup):
        try:
            return lookup[mask]
        except KeyError:
            occurances = {}
            for individual in self.individuals:
                # extract the gene values for the cluster
                value = self.getMaskValue(individual, mask)
                try:
                    occurances[value] += 1
                except KeyError:
                    occurances[value] = 1
            total = float(len(self.individuals))
            result = -sum(x / total * math.log(x / total, 2)
                          for x in occurances.itervalues())
            lookup[mask] = result
            return result

    def clusterDistance(self, c1, c2, lookup):
        try:
            return lookup[c1, c2]
        except KeyError:
            try:
                result = 2 - ((self.entropy(c1, lookup) +
                               self.entropy(c2, lookup))
                              / self.entropy(c1 + c2, lookup))
            except ZeroDivisionError:
                result = 2  # Zero division only happens in 0/0
            lookup[c1, c2] = result
            lookup[c2, c1] = result
            return result

    def pairwiseDistance(self, c1, c2, lookup):
        try:
            return lookup[c1, c2]
        # averages the pairwise distance between each cluster
        except KeyError:
            result = sum(self.clusterDistance((a,), (b,), lookup)
                         for a in c1 for b in c2) / float(len(c1) * len(c2))
            lookup[c1, c2] = result
            lookup[c2, c1] = result
            return result

    def buildTree(self, distance):
        clusters = [(i,) for i in xrange(len(self.individuals[0].genes))]
        subtrees = [(i,) for i in xrange(len(self.individuals[0].genes))]
        random.shuffle(clusters)
        random.shuffle(subtrees)
        lookup = {}

        def allLowest():
            minVal = 3  # Max possible distance should be 2
            results = []
            for c1, c2 in combinations(clusters, 2):
{'fraction_non_alphanumeric': 0.09168704156479218, 'fraction_numerical': 0.015484922575387123, 'mean_word_length': 1.6592632719393283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1242083', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1309, 'n_words': 394}