<commit_before>from followthemoney import model
import jellyfish
import itertools
from banal import ensure_list

def compare(left, right):
    """Compare two entities and return number between 0 and 1
    
    Returned number indicates probability that two entities are the same.    
    """
    result = 0

    matchable = [s.name for s in model if s.matchable]
    
    if left['schema'] in matchable and right['schema'] in matchable:   
        result = compare_entities(left, right) 
    return result

def compare_entities(left, right):
    name_result = compare_fingerprints(left, right)
    identifiers_result = compare_identifiers(left, right)
    result = (name_result + identifiers_result)/2
    if result > 1:
        result = 1
    return result

def compare_fingerprints(left, right):

    result = 0
    results = []

    left_list = ensure_list(left.get('fingerprints'))
    right_list = ensure_list(right.get('fingerprints'))

    if left_list and right_list:
        input = [left_list, right_list]
        pairs = list(itertools.product(*input))

        for pair in pairs:
            results.append(jellyfish.jaro_distance(pair[0], pair[1]))

        result = max(results)

        for res in results:
            if res == 1:
                result += 0.2
            elif res > 0.8:
                result += 0.1
    return result

def compare_identifiers(left, right):
    result = 0
    
    left_list = ensure_list(left.get('identifiers'))
    right_list = ensure_list(right.get('identifiers'))

    left_list = extract_ids(left_list)
    right_list = extract_ids(right_list)

    if left_list and right_list:
        comp = set.intersection(left_list, right_list)
        result = len(comp)

    return result

def extract_ids(id_strings):
    """For now extracts only numbers, TODO: add other specific identifiers"""
    return set(filter(None, [extract_id(i) for i in id_strings]))
    
def extract_id(id_string):
    """Get one word from string with most numbers and longer than 3 characters"""
    best_id_candidate = None
    numbers_in_id = 0
    id_list = [i for i in id_string.split() if len(i) > 3]
    for word in id_list:
        numbers = sum(symbol.isdigit() for symbol in word)
        if numbers_in_id < numbers:
            numbers_in_id = numbers
            best_id_candidate = word
    return best_id_candidate<commit_msg>A little bit pf pep and clean-up<commit_after>import jellyfish
import itertools
from banal import ensure_list
from followthemoney import model


def compare(left, right):
    """Compare two entities and return number between 0 and 1.
    Returned number indicates probability that two entities are the same.
    """
    left_schema = model.get(left.get('schema'))
    right_schema = model.get(right.get('schema'))
    if right_schema not in list(left_schema.matchable_schemata):
        return 0

    name_result = compare_fingerprints(left, right)
    identifiers_result = compare_identifiers(left, right)
    result = (name_result + identifiers_result)/2
    return max(0.0, min(1.0, result))


def compare_fingerprints(left, right):
    result = 0
    results = []

    left_list = ensure_list(left.get('fingerprints'))
    right_list = ensure_list(right.get('fingerprints'))

    if left_list and right_list:
        for (left, right) in itertools.product(left_list, right_list):
            results.append(jellyfish.jaro_distance(left, right))

        result = max(results)
        for res in results:
            if res == 1:
                result += 0.2
            elif res > 0.8:
                result += 0.1
    return result


def compare_identifiers(left, right):
    result = 0
    left_list = ensure_list(left.get('identifiers'))
    right_list = ensure_list(right.get('identifiers'))

    left_list = extract_ids(left_list)
    right_list = extract_ids(right_list)

    if left_list and right_list:
        comp = set.intersection(left_list, right_list)
        result = len(comp)

    return result


def extract_ids(id_strings):
    """For now extracts only numbers, TODO: add other specific identifiers"""
    return set(filter(None, [extract_id(i) for i in id_strings]))


def extract_id(id_string):
    """Get one word from string with most numbers and longer than
    3 characters"""
    best_id_candidate = None
    numbers_in_id = 0
    id_list = [i for i in id_string.split() if len(i) > 3]
    for word in id_list:
        numbers = sum(symbol.isdigit() for symbol in word)
        if numbers_in_id < numbers:
            numbers_in_id = numbers
            best_id_candidate = word
    return best_id_candidate
{'fraction_non_alphanumeric': 0.08814589665653495, 'fraction_numerical': 0.008684324793747287, 'mean_word_length': 2.937606837606838, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3704683', 'n_tokens_mistral': 1422, 'n_tokens_neox': 1357, 'n_words': 404}