<commit_before>
class MyMethod(object):
    def __init__(self, graph, clusters):
        self.graph = graph
        self.clusters = clusters
        self.count_partitions = {}

    def __get_count(self):
        abstraction = []
        for cluster_id, nodes in self.clusters.iteritems():
            if len(nodes) > 1:
                for node_id in nodes:
                    message = self.graph.node[node_id]['preprocessed_event']

                    # get count
                    tokens = message.strip().split()
                    token_count = len(tokens)

                    partition_keys = self.count_partitions.keys()
                    if token_count not in partition_keys:
                        self.count_partitions[token_count] = []
                    self.count_partitions[token_count].append(message)

            elif len(nodes) == 1:
                abstraction[cluster_id] = self.graph.node[nodes[0]]['preprocessed_event']<commit_msg>User OrderedSet to handle ordered abstraction<commit_after>from orderedset import OrderedSet


class MyMethod(object):
    def __init__(self, graph, clusters):
        self.graph = graph
        self.clusters = clusters
        self.count_groups = {}

    def get_abstraction(self):
        abstraction = {}
        abstraction_id = 0
        for cluster_id, nodes in self.clusters.iteritems():
            nodes = list(nodes)
            if len(nodes) > 1:
                # group the preprocessed event with the same word count
                for node_id in nodes:
                    message = self.graph.node[node_id]['preprocessed_event']

                    # get count
                    words_split = message.strip().split()
                    words_count = len(words_split)

                    group_keys = self.count_groups.keys()
                    if words_count not in group_keys:
                        self.count_groups[words_count] = []
                    self.count_groups[words_count].append(OrderedSet(words_split))

                # get common words as abstraction
                for words_count, group in self.count_groups.iteritems():
                    abstraction[abstraction_id] = group[0]
                    for message in group:
                        abstraction[abstraction_id].intersection_update(message)
                    abstraction_id += 1

            elif len(nodes) == 1:
                abstraction[abstraction_id] = self.graph.node[nodes[0]]['preprocessed_event']
                abstraction_id += 1

        return abstraction
{'fraction_non_alphanumeric': 0.08372641509433962, 'fraction_numerical': 0.003930817610062893, 'mean_word_length': 1.7103301384451544, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '906309', 'n_tokens_mistral': 693, 'n_tokens_neox': 635, 'n_words': 141}