<commit_before>#!/usr/bin/env Python
"""
Performs a basic assignee disambiguation
"""
import redis
from collections import Counter
from Levenshtein import jaro_winkler
from alchemy import fetch_session # gives us the `session` variable
from alchemy.schema import *

THRESHOLD = 0.95

# get alchemy.db from the directory above
s = fetch_session(path_to_sqlite='..')

# get redis session
r = redis.StrictRedis(host='localhost')

# delete all previous keys
for i in r.keys():
    r.delete(i)

# get all assignees in database
assignees = s.query(RawAssignee).all()

def get_assignee_id(obj):
    """
    Returns string representing an assignee object. Returns obj.organization if
    it exists, else returns concatenated obj.name_first + '|' + obj.name_last
    """
    if obj.organization: return obj.organization
    try:
        return obj.name_first + '|' + obj.name_last
    except:
        return ''

def create_assignee_blocks(assignees):
    """
    Iterates through all assignees. If the strings match within the THRESHOLD confidence,
    we put them into the same block, else put the current assignee in its own block. Blocks
    are stored as redis lists, named by the first ID we encounter for that block
    """
    for current in assignees:
        for assignee in assignees:
            if current == assignee: continue
            current_id = get_assignee_id(current)
            assignee_id = get_assignee_id(assignee)
            if jaro_winkler(current_id, assignee_id) >= THRESHOLD:
                # block name is the first id we encountered
                r.lpush(current_id, assignee_id, current_id)
                assignees.remove(assignee)
                assignees.remove(current)
            else:
                r.lpush(current_id, current_id)
                #assignees.remove(current)

def disambiguate_by_frequency():
    """
    For block, find the most frequent assignee name and create a hash from each
    assignee organization/name to the most frequent name. Delete the old block
    during this process. This ensures that the only keys left in our database
    are the disambiguations.
    """
    for block in r.keys():
        assignees = r.lrange(block, 0, -1) # get all elements in list [block]
        most_common_id = Counter(assignees).most_common()[0][0]
        r.delete(block)
        for assignee in assignees:
            r.set(assignee, most_common_id)

def create_assignee_table(assignees):
    """
    Given a list of assignees and the redis key-value disambiguation,
    populates the Assignee table in the database
    """
    for assignee in assignees:
        record = {} # dict for insertion
        disambiguated_name = r.get(get_assignee_id(assignee))
        if assignee.organization:
            record['organization'] = disambiguated_name
        else:
            record['name_first'] = disambiguated_name.split('|')[0]
            record['name_last'] = disambiguated_name.split('|')[1]
        for key in ['residence', 'nationality', 'type']:
            record[key] = getattr(assignee, key)
        record['id'] = assignee.uuid
        assignee_obj = Assignee(**record)
        assignee_obj.rawassignees.append(assignee)
        s.merge(assignee_obj)
    try:
        s.commit()
    except Exception, e:
        s.rollback()


def examine():
    assignees = s.query(Assignee).all()
    for a in assignees:
        print a.id, a.rawassignees


if __name__=='__main__':
    create_assignee_blocks(assignees)
    disambiguate_by_frequency()
    create_assignee_table(assignees)<commit_msg>bug fixes with removing from list<commit_after>#!/usr/bin/env Python
"""
Performs a basic assignee disambiguation
"""
import redis
from collections import Counter
from Levenshtein import jaro_winkler
from alchemy import fetch_session # gives us the `session` variable
from alchemy.schema import *

THRESHOLD = 0.95

# get alchemy.db from the directory above
s = fetch_session(path_to_sqlite='..')

# get redis session
r = redis.StrictRedis(host='localhost')

# delete all previous keys
for i in r.keys():
    r.delete(i)

# get all assignees in database
assignees = s.query(RawAssignee).all()

def get_assignee_id(obj):
    """
    Returns string representing an assignee object. Returns obj.organization if
    it exists, else returns concatenated obj.name_first + '|' + obj.name_last
    """
    if obj.organization: return obj.organization
    try:
        return obj.name_first + '|' + obj.name_last
    except:
        return ''

def create_assignee_blocks(list_of_assignees):
    """
    Iterates through all assignees. If the strings match within the THRESHOLD confidence,
    we put them into the same block, else put the current assignee in its own block. Blocks
    are stored as redis lists, named by the first ID we encounter for that block
    """
    assignees = list_of_assignees[:]
    for current in assignees:
        for assignee in assignees:
            if current == assignee: continue
            current_id = get_assignee_id(current)
            assignee_id = get_assignee_id(assignee)
            if jaro_winkler(current_id, assignee_id) >= THRESHOLD:
                # block name is the first id we encountered
                r.lpush(current_id, assignee_id)
                assignees.remove(assignee)
        r.lpush(current_id, current_id)
        #if current in assignees: assignees.remove(current)

def disambiguate_by_frequency():
    """
    For block, find the most frequent assignee name and create a hash from each
    assignee organization/name to the most frequent name. Delete the old block
    during this process. This ensures that the only keys left in our database
    are the disambiguations.
    """
    for block in r.keys():
        assignees = r.lrange(block, 0, -1) # get all elements in list [block]
        most_common_id = Counter(assignees).most_common()[0][0]
        r.delete(block)
        for assignee in assignees:
            r.set(assignee, most_common_id)

def create_assignee_table(assignees):
    """
    Given a list of assignees and the redis key-value disambiguation,
    populates the Assignee table in the database
    """
    for assignee in assignees:
        record = {} # dict for insertion
        disambiguated_name = r.get(get_assignee_id(assignee))
        if assignee.organization:
            record['organization'] = disambiguated_name
        else:
            record['name_first'] = disambiguated_name.split('|')[0]
            record['name_last'] = disambiguated_name.split('|')[1]
        for key in ['residence', 'nationality', 'type']:
            record[key] = getattr(assignee, key)
        record['id'] = assignee.uuid
        assignee_obj = Assignee(**record)
        assignee_obj.rawassignees.append(assignee)
        s.merge(assignee_obj)
        try:
            s.commit()
        except Exception, e:
            s.rollback()


def examine():
    assignees = s.query(Assignee).all()
    print len(assignees)
    print len(s.query(RawAssignee).all())
    for a in assignees:
        print get_assignee_id(a), a.rawassignees


if __name__=='__main__':
    create_assignee_blocks(assignees)
    disambiguate_by_frequency()
    create_assignee_table(assignees)
{'fraction_non_alphanumeric': 0.08790445771420637, 'fraction_numerical': 0.0024996528259963896, 'mean_word_length': 2.905639913232104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3078181', 'n_tokens_mistral': 2281, 'n_tokens_neox': 2108, 'n_words': 635}