<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import itertools

import numpy
import fastcluster
import hcluster

def connected_components(edgelist) :
    root = {}
    component = {}
    indices = {}
    
    for i, edge in enumerate(edgelist['pairs']) :
        (a, b) = edge
        edge = a, b
        root_a = root.get(a)
        root_b = root.get(b)

        if root_a is None and root_b is None :
            component[a] = set([a, b])
            indices[a] = [i]
            root[a] = root[b] = a
        elif root_a is None or root_b is None :
            if root_a is None :
                a, b = b, a
                root_a, root_b = root_b, root_a
            component[root_a].add(b)
            indices[root_a].append(i)
            root[b] = root_a
        elif root_a != root_b :
            component_a = component[root_a]
            component_b = component[root_b]
            if len(component_a) < len(component_b) :
                root_a, root_b = root_b, root_a
                component_a, component_b = component_b, component_a

            component_a |= component_b
            indices[root_a].extend(indices[root_b])

            for node in component_b :
                root[node] = root_a

            del component[root_b]
            del indices[root_b]
        else : 
            indices[root_a].append(i)
    
    for sub_graph in indices.values() :
        yield edgelist[sub_graph]

def condensedDistance(dupes):
    '''
    Convert the pairwise list of distances in dupes to "condensed
    distance matrix" required by the hierarchical clustering
    algorithms. Also return a dictionary that maps the distance matrix
    to the record_ids.
   <commit_msg>debug statement, for understanding memory usage<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import itertools

import numpy
import fastcluster
import hcluster

def connected_components(edgelist) :
    root = {}
    component = {}
    indices = {}
    
    for i, edge in enumerate(edgelist['pairs']) :
        (a, b) = edge
        edge = a, b
        root_a = root.get(a)
        root_b = root.get(b)

        if root_a is None and root_b is None :
            component[a] = set([a, b])
            indices[a] = [i]
            root[a] = root[b] = a
        elif root_a is None or root_b is None :
            if root_a is None :
                a, b = b, a
                root_a, root_b = root_b, root_a
            component[root_a].add(b)
            indices[root_a].append(i)
            root[b] = root_a
        elif root_a != root_b :
            component_a = component[root_a]
            component_b = component[root_b]
            if len(component_a) < len(component_b) :
                root_a, root_b = root_b, root_a
                component_a, component_b = component_b, component_a

            component_a |= component_b
            indices[root_a].extend(indices[root_b])

            for node in component_b :
                root[node] = root_a

            del component[root_b]
            del indices[root_b]
        else : 
            indices[root_a].append(i)

    print "components calculated"
    
    for sub_graph in indices.values() :
        yield edgelist[sub_graph]

def condensedDistance(dupes):
    '''
    Convert the pairwise list of distances in dupes to "condensed
    distance matrix" required by the hierarchical clustering
    algorithms. Also return a dictionary that maps the distance matrix
    to the record_ids.
   
{'fraction_non_alphanumeric': 0.08809182209469153, 'fraction_numerical': 0.0005738880918220947, 'mean_word_length': 1.8527004909983633, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2716467', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1039, 'n_words': 292}