<commit_before>from bigbang.parse import get_date
import pandas as pd
import datetime
import networkx as nx
import numpy as np
import email.utils
import re

import Levenshtein
from functools import partial


def consolidate_senders_activity(activity_df, to_consolidate):
    """
    takes a DataFrame in the format returned by activity
    takes a list of tuples of format ('from 1', 'from 2') to consolidate
    returns the consolidated DataFrame (a copy, not in place)
    """

    df = activity_df.copy(deep=True)
    for consolidate in to_consolidate:
        column_a, column_b = consolidate
        if column_a in df.columns and column_b in df.columns:
            df[column_a] = df[column_a] + df[column_b]
            df.drop(column_b, inplace=True, axis=1)  # delete the second column
    return df


def matricize(series, func):
    """
    create a matrix by applying func to pairwise combos of elements in a Series
    returns a square matrix as a DataFrame
    should return a symmetric matrix if func(a,b) == func(b,a)
    should return the identity matrix if func == '=='
    """
    matrix = pd.DataFrame(columns=series, index=series)
    for index, element in enumerate(series):
        for second_index, second_element in enumerate(series):
            matrix.iloc[index, second_index] = func(element, second_element)

    return matrix


def minimum_but_not_self(column, dataframe):
    minimum = 100
    for index, value in dataframe[column].items():
        if index == column:
            continue
        if value < minimum:
            minimum = value
    return minimum


def sorted_matrix(from_dataframe,limit=None,sort_key=None):
    if limit is None:
        limit = len(from_dataframe.columns)

    distancedf = matricize(from_dataframe.columns[:limit - 1], from_header_distance)

    # specify that the values in the matrix are integers
    df = distancedf.astype(int)
    
    # unless otherwise specified, sort to minimize the integer values with rows other than yourself
    sort_key = sort_key if sort_key is not None else partial(minimum_but_not_self, dataframe=df)
    
    new_columns = sorted(df.columns[:limit - 1], key=sort_key)
    new_df = df.reindex(index=new_columns, columns=new_columns)

    return new_df

def resolve_sender_entities(act, lexical_distance=0):
    """
    Given an Archive's activity matrix, return a dict of lists, each containing
    message senders ('From' fields) that have been groups to be
    probably the same entity.
    """
    
    # senders orders by descending total activity
    senders = act.sum(0).sort_values(ascending=False)
    senders_act = senders.index

    # senders in lexical order
    senders_lex = act.columns.sort_values()
    senders_lex_dict = dict([(p[1],p[0]) for p in enumerate(senders_lex)])

    n = len(senders)
    # binary matrix of similarity between entries
    sim = np.zeros((n,n))

    # find similarity 
    for i in range(n):
        name = senders_act[i]
        i = senders_lex_dict[name]
    
        # checking only lexically close entries and
        # in proportion to total activity
        # is a performance hack.
        for j in range(i - (n - i + 1) // 2, i + (n - i + 1) // 2):
            d = from_header_distance(senders_lex[i],senders_lex[j])
            sim[i,j] = (d <= lexical_distance)

    # An entity is a connected component of the resulting graph
    G = nx.Graph(sim)
    entities_list = [[senders_lex[j] for j in x] for x in nx.connected_components(G)]

    # given each entity a label based on its most active 'member'
    entities_dict = {}
    for e in entities_list:
        # TODO: tighten up this labeling function
        label = sorted(e,key=lambda n:senders[n],reverse=True)[0]
        entities_dict[label] = e

    return entities_dict<commit_msg>abstract out resolve_entities function<commit_after>from bigbang.parse import get_date
import pandas as pd
import datetime
import networkx as nx
import numpy as np
import email.utils
import re

import Levenshtein
from functools import partial


def consolidate_senders_activity(activity_df, to_consolidate):
    """
    takes a DataFrame in the format returned by activity
    takes a list of tuples of format ('from 1', 'from 2') to consolidate
    returns the consolidated DataFrame (a copy, not in place)
    """

    df = activity_df.copy(deep=True)
    for consolidate in to_consolidate:
        column_a, column_b = consolidate
        if column_a in df.columns and column_b in df.columns:
            df[column_a] = df[column_a] + df[column_b]
            df.drop(column_b, inplace=True, axis=1)  # delete the second column
    return df


def matricize(series, func):
    """
    create a matrix by applying func to pairwise combos of elements in a Series
    returns a square matrix as a DataFrame
    should return a symmetric matrix if func(a,b) == func(b,a)
    should return the identity matrix if func == '=='
    """
    matrix = pd.DataFrame(columns=series, index=series)
    for index, element in enumerate(series):
        for second_index, second_element in enumerate(series):
            matrix.iloc[index, second_index] = func(element, second_element)

    return matrix


def minimum_but_not_self(column, dataframe):
    minimum = 100
    for index, value in dataframe[column].items():
        if index == column:
            continue
        if value < minimum:
            minimum = value
    return minimum


def sorted_matrix(from_dataframe,limit=None,sort_key=None):
    """
    Takes a dataframe with 'from' fields for column headers
.
    Returns a sorted distance matrix for the column headers,
    using from_header_distance (see method).
    """
    if limit is None:
        limit = len(from_dataframe.columns)

    distancedf = matricize(from_dataframe.columns[:limit - 1], from_header_distance)

    # specify that the values in the matrix are integers
    df = distancedf.astype(int)
    
    # unless otherwise specified, sort to minimize the integer values with rows other than yourself
    sort_key = sort_key if sort_key is not None else partial(minimum_but_not_self, dataframe=df)
    
    new_columns = sorted(df.columns[:limit - 1], key=sort_key)
    new_df = df.reindex(index=new_columns, columns=new_columns)

    return new_df

def resolve_sender_entities(act, lexical_distance=0):
    """
    Given an Archive's activity matrix, return a dict of lists, each containing
    message senders ('From' fields) that have been groups to be
    probably the same entity.
    """
    
    # senders orders by descending total activity
    senders = act.sum(0).sort_values(ascending=False)

    return resolve_entities(senders, from_header_distance, threshold = lexical_distance)

def resolve_entities(significance, distance_function, threshold = 0):
    """
    Takes a Series mapping entities (index) to significance (values, numerical).

    Resolves the entities based on a lexical distance function.

    Returns a dictionary of labeled (keys) entity lists (values).
    Key is the most significant member of the entity list.
    """
    entities = significance.index

    # entities in lexical order
    entities_lex = entities.sort_values()
    entities_lex_dict = dict([(p[1],p[0]) for p in enumerate(entities_lex)])

    n = len(entities)
    # binary matrix of similarity between entries
    sim = np.zeros((n,n))

    # find similarity 
    for i in range(n):
        name = entities[i]
        i = entities_lex_dict[name]
    
        # checking only lexically close entries and
        # in proportion to total activity
        # is a performance hack.
        for j in range(i - (n - i + 1) // 2, i + (n - i + 1) // 2):
            d = distance_function(entities_lex[i],entities_lex[j])
            sim[i,j] = (d <= threshold)

    # An entity is a connected component of the resulting graph
    G = nx.Graph(sim)
    entities_list = [[entities_lex[j] for j in x] for x in nx.connected_components(G)]

    # given each entity a label based on its most active 'member'
    entities_dict = {}
    for e in entities_list:
        # TODO: tighten up this labeling function
        label = sorted(e,key=lambda n:significance[n],reverse=True)[0]
        entities_dict[label] = e

    return entities_dict
{'fraction_non_alphanumeric': 0.08155339805825243, 'fraction_numerical': 0.00424757281553398, 'mean_word_length': 3.1122754491017965, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3664080', 'n_tokens_mistral': 2474, 'n_tokens_neox': 2395, 'n_words': 834}