# AUTOGENERATED! DO NOT EDIT! File to edit: 71-relationship-builder.ipynb (unless otherwise specified).

__all__ = ['retrieve_controlled_vocabularies', 'build_reciprocal_relationship', 'alt_assign_relationships',
           'categorize_characteristics', 'assign_characteristics', 'id_unique_individuals', 'find_sus',
           'split_name_col', 'disambiguate', 'determine_principals', 'assign_relationships', 'normalize_date',
           'determine_event_date', 'determine_event_location', 'identify_cleric', 'build_event',
           'drop_obvious_duplicates', 'id_obvious_duplicates', 'assign_unique_ids', 'merge_records', 'merge_duplicates',
           'build_new_person', 'build_entry_metadata']

# Cell
#dependencies

#nlp packages
import spacy
from spacy.util import minibatch, compounding

#manipulation of tables/arrays
import pandas as pd
import numpy as np
import copy

#internal imports
from .collate import *
from .split_data import *
from .modeling import *
from .model_performance_utils import *
from .xml_parser import *
from .unstructured2markup import *
from .utility import *

# Cell
#ideally this function will eventually query the database live, but for now we'll use static snapshots
#of the vocabularies as they currently exist
#should also eventually be enhanced to return equivalents and a requested language

def retrieve_controlled_vocabularies():
    '''
    returns a dictionary containing current version of controlled vocabularies for characteristics
    '''
    age = ["parvola", "parvolo", "parvulo", "parvula", "parv", "adulto", "adulta", "adul", "niño", "niña", "nino", "nina", "dulto", "dulta"]
    occupation = ["religioso", "ingen.o", "sacristan", "sac.", "sachristan", "cura", "vicario", "eclesiastico", "clerigo", "estudiante"]
    phenotype = ["negro", "negra", "preto", "moreno", "morena", "indio", "india", "pardo", "parda", "mestizo", "mestiza", "mulato", "mulata", "blanco", "blanca", "criollo", "criolla", "branco", "branca"]
    titles = ["frai", "sr", "sr.", "rexidor", "regr", "probin sial", "probinsial", "herm.o", "hermano", "dr.", "doc tor", "dna", "difin.dor", "difin.or", "difinidor", "d.a", "aiudante", "doctor", "d.r", "dor", "dr", "d.or", "b.er", "br", "ber", "don", "doña", "da", "padre", "pe", "predicador", "fray", "d.n", "d.", "d,n", "d;n", "p.e", "p", "dn", "fr.", "fr", "f", "regidor", "rex.or", "alg.l m.or", "ldo", "licenciado", "d", "alg.l", "alcalde"]
    ranks = ["my.r", "comandan.te", "com.te", "capt.n", "capp[roto]", "cap[roto]", "capitn", "comend.te63", "comand.te", "alferes", "alfer.s mayor", "capitan", "capitam", "cap.n", "capit.n", "capn", "sarg.to may.r", "sarg.to", "sargento", "sarjento mayor", "sarjento", "sargto mayor", "theniente", "teniente", "thente"]
    ethnicities = ["ganga", "español", "espanol", "caravali", "ingles", "yngles", "angola", "carabalí", "carabali", "carabaly", "congo", "conga", "mandinga", "mina", "temo", "malagas", "arara", "manga"]
    status = ["propiedad", "escrava", "escravos", "esclabo", "esclaba", "escl.a", "escl.o", "clavo", "clava", "escl", "escl.", "escl.s", "clabo", "claba", "esc.va", "esc.ba", "esc.vo", "escvo", "esclavo", "esclava", "escva", "esc.bo", "esclabos", "esclavos", "esc.os", "esc.a", "esc.o", "libre", "esc.s", "esco", "esca"]
    legitimacy = ["lexma", "lexmo", "legitima", "legitimo", "h l", "natural", "nral", "lexitima", "lexitimo", "nat.l"]
    relationships = ["hermano", "hijo", "hija", "esposo", "esposa", "viudo", "viuda", "padrinos", "padrino", "padryno", "soltera", "soltero", "madrina", "padre", "p.p.", "p. ", "p."]
    origin = ["naturales"]

    vocabs = {"origin": origin, "legitimacy": legitimacy, "age": age, "occupation": occupation, "phenotype": phenotype, "titles": titles, "ranks": ranks, "ethnicities": ethnicities, "status": status, "relationships": relationships}

    return vocabs

# Cell

def build_reciprocal_relationship(people, from_person, to_person, relationship_type):
    '''
    helper function that adds a reciprocal relationship of a specified type to the records of two people
        people: list of dictionaries, each of which represents one mention of a person in the entry
        from_person: unique_id of person that relationship "comes from" (i.e. the parent in a "parent"-type relationship)
        to_person: unique_id of person that relationship "goes to" (i.e. the child in a "parent"-type relationship)
        relationship_type: currently accepts "parent", "godparent", "enslaver", and "spouse"

        returns: updated version of people with interpersonal relationship added
    '''

    empty_from = False
    empty_to = False
    no_to = False
    no_from = False

    if from_person == None:
        no_from = True
    if to_person == None:
        no_to = True

    for i in range(len(people)):
        if people[i]['id'] == from_person:
            from_loc = i
            if people[i]['relationships'] == None:
                empty_from = True
        elif people[i]['id'] == to_person:
            to_loc = i
            if people[i]['relationships'] == None:
                empty_to = True

    if relationship_type == "godparent":
        if not no_from:
            if empty_from:
                people[from_loc]['relationships'] = [{"related_person": to_person, "relationship_type": "godchild"}]
            else:
                people[from_loc]['relationships'].append({"related_person": to_person, "relationship_type": "godchild"})

        if not no_to:
            if empty_to:
                people[to_loc]['relationships'] = [{"related_person": from_person, "relationship_type": "godparent"}]
            else:
                people[to_loc]['relationships'].append({"related_person": from_person, "relationship_type": "godparent"})

    elif relationship_type == "parent":
        if not no_from:
            if empty_from:
                people[from_loc]['relationships'] = [{"related_person": to_person, "relationship_type": "child"}]
            else:
                people[from_loc]['relationships'].append({"related_person": to_person, "relationship_type": "child"})

        if not no_to:
            if empty_to:
                people[to_loc]['relationships'] = [{"related_person": from_person, "relationship_type": "parent"}]
            else:
                people[to_loc]['relationships'].append({"related_person": from_person, "relationship_type": "parent"})

    elif relationship_type == "grandparent":
        if not no_from:
            if empty_from:
                people[from_loc]['relationships'] = [{"related_person": to_person, "relationship_type": "grandchild"}]
            else:
                people[from_loc]['relationships'].append({"related_person": to_person, "relationship_type": "grandchild"})

        if not no_to:
            if empty_to:
                people[to_loc]['relationships'] = [{"related_person": from_person, "relationship_type": "grandparent"}]
            else:
                people[to_loc]['relationships'].append({"related_person": from_person, "relationship_type": "grandparent"})

    elif relationship_type == "enslaver":
        if not no_from:
            if empty_from:
                people[from_loc]['relationships'] = [{"related_person": to_person, "relationship_type": "slave"}]
            else:
                people[from_loc]['relationships'].append({"related_person": to_person, "relationship_type": "slave"})

        if not no_to:
            if empty_to:
                people[to_loc]['relationships'] = [{"related_person": from_person, "relationship_type": "enslaver"}]
            else:
                people[to_loc]['relationships'].append({"related_person": from_person, "relationship_type": "enslaver"})

    elif relationship_type == "spouse":
        if not no_from:
            if empty_from:
                people[from_loc]['relationships'] = [{"related_person": to_person, "relationship_type": "spouse"}]
            else:
                people[from_loc]['relationships'].append({"related_person": to_person, "relationship_type": "spouse"})

        if not no_to:
            if empty_to:
                people[to_loc]['relationships'] = [{"related_person": from_person, "relationship_type": "spouse"}]
            else:
                people[to_loc]['relationships'].append({"related_person": from_person, "relationship_type": "spouse"})

    return people

# Cell

def alt_assign_relationships(entry_text, entities, people_df, people, volume_metadata):
    '''
    matches all labeled relationships to the correct individuals and builds triples
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: df containing all entities extracted from that entry by an NER model
        people_df: entities given the label "PER" from a single entry by an NER model with unique ids
        people: list of dictionaries, each of which represents one mention of a person in the entry
        (as produced by assign_characteristics)
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata

        returns: updated version of people with interpersonal relationships added
    '''

    rel_types = retrieve_controlled_vocabularies()["relationships"]
    relationships = copy.deepcopy(entities.loc[entities['pred_label'] == 'REL'])
    relationships.reset_index(inplace=True)
    characteristics = copy.deepcopy(entities.loc[entities['pred_label'] == 'CHAR'])
    characteristics.reset_index(inplace=True)
    cat_char, uncat_char = categorize_characteristics(entities, characteristics)
    entities.reset_index(inplace=True)

    #if determine_principals(entry_text, entities, 1) != None:
    if not (determine_principals(entry_text, entities, 1) == None):
        principal = determine_principals(entry_text, entities, 1)[0]
        for i in range(len(people)):
            if people[i]["name"] == principal:
                principal_id = people[i]['id']
                break
    else:
        principal = "Unknown principal"
        for i in range(len(people)):
            if people[i]["name"] == principal:
                principal_id = people[i]['id']
                break

    found_parents = False
    found_godparents = False
    godparents = []
    found_paternal_grandparents = False
    found_maternal_grandparents = False
    found_enslaver = False
    enslaver_id = None

    #build godparent/godchild relationships
    #future improvement: add logic to look for spousal relationship between godparents
    if (len(entities) != 0) and (len(relationships) != 0):
        for index in range(len(entities)):
            if entities['pred_label'][index] == "REL":
                if ((entities['pred_entity'][index].lower() == "madrina") or (entities['pred_entity'][index].lower() == "padrino") or (entities['pred_entity'][index].lower() == "padryno")) and (found_godparents == False):
                    if (len(entities) > (index + 1)) and (entities['pred_label'][index + 1] == "PER"):
                        for j in range(len(people_df)):
                            if people_df['pred_start'][j] == entities['pred_start'][index + 1]:
                                from_person = people_df['unique_id'][j]
                        people = build_reciprocal_relationship(people, from_person, principal_id, "godparent")
                        found_godparents = True
                        godparent = {"name": people_df["pred_entity"][j], "id": people_df["unique_id"][j]}
                        godparents.append(godparent)
                elif ((entities['pred_entity'][index].lower() == "padrinos") or (entities['pred_entity'][index].lower() == "p.p.")) and (found_godparents == False):
                    if (len(entities) > (index + 1)) and (entities['pred_label'][index + 1] == "PER"):
                        for j in range(len(people_df)):
                            if people_df['pred_start'][j] == entities['pred_start'][index + 1]:
                                from_person = people_df['unique_id'][j]
                        people = build_reciprocal_relationship(people, from_person, principal_id, "godparent")
                        found_godparents = True
                        godparent = {"name": people_df["pred_entity"][j], "id": people_df["unique_id"][j]}
                        godparents.append(godparent)
                    if (len(entities) > (index + 2)) and (entities['pred_label'][index + 2] == "PER"):
                        for j in range(len(people_df)):
                            if people_df['pred_start'][j] == entities['pred_start'][index + 2]:
                                from_person = people_df['unique_id'][j]
                        people = build_reciprocal_relationship(people, from_person, principal_id, "godparent")
                        found_godparents = True
                        godparent = {"name": people_df["pred_entity"][j], "id": people_df["unique_id"][j]}
                        godparents.append(godparent)
                elif ("p." in entities['pred_entity'][index].lower()) and (found_godparents == False):
                    if (len(entities) > (index + 1)) and not ("p." in entities['pred_entity'][index + 1].lower()):
                        if (len(entities) > (index + 1)) and (entities['pred_label'][index + 1] == "PER"):
                            for j in range(len(people_df)):
                                if people_df['pred_start'][j] == entities['pred_start'][index + 1]:
                                    from_person = people_df['unique_id'][j]
                            people = build_reciprocal_relationship(people, from_person, principal_id, "godparent")
                            found_godparents = True
                            godparent = {"name": people_df["pred_entity"][j], "id": people_df["unique_id"][j]}
                            godparents.append(godparent)
                    elif (len(entities) > (index + 1)):
                        if (len(entities) > (index + 2)) and (entities['pred_label'][index + 2] == "PER"):
                            for j in range(len(people_df)):
                                if people_df['pred_start'][j] == entities['pred_start'][index + 2]:
                                    from_person = people_df['unique_id'][j]
                            people = build_reciprocal_relationship(people, from_person, principal_id, "godparent")
                            found_godparents = True
                            godparent = {"name": people_df["pred_entity"][j], "id": people_df["unique_id"][j]}
                            godparents.append(godparent)
                        if (len(entities) > (index + 3)) and (entities['pred_label'][index + 3] == "PER"):
                            for j in range(len(people_df)):
                                if people_df['pred_start'][j] == entities['pred_start'][index + 3]:
                                    from_person = people_df['unique_id'][j]
                            people = build_reciprocal_relationship(people, from_person, principal_id, "godparent")
                            found_godparents = True
                            godparent = {"name": people_df["pred_entity"][j], "id": people_df["unique_id"][j]}
                            godparents.append(godparent)
                #build grandparents
                elif ("abuelos" in entities["pred_entity"][index].lower()):
                    if ("paternos" in entities["pred_entity"][index].lower()) and (found_paternal_grandparents == False):
                        paternal_grandfather = ''
                        paternal_grandmother = ''
                        if entities["pred_label"][index + 1] == "PER":
                            for j in range(len(people_df)):
                                if people_df['pred_start'][j] == entities['pred_start'][index + 1]:
                                    grandparent_id = people_df['unique_id'][j]
                                    break
                            if determine_sex(entities["pred_entity"][index + 1].split(' ')[0], name_list="names.json") == "male":
                                paternal_grandfather = grandparent_id
                                paternal_grandfather_index = j
                            elif determine_sex(entities["pred_entity"][index + 1].split(' ')[0], name_list="names.json") == "female":
                                paternal_grandmother = grandparent_id
                                paternal_grandmother_index = j
                            else:
                                paternal_grandmother = grandparent_id
                                paternal_grandmother_index = j
                            if entities["pred_label"][index + 2] == "PER":
                                for j in range(len(people_df)):
                                    if people_df['pred_start'][j] == entities['pred_start'][index + 2]:
                                        grandparent_id = people_df['unique_id'][j]
                                        break
                                if (determine_sex(entities["pred_entity"][index + 2].split(' ')[0], name_list="names.json") == "male") and (paternal_grandfather == ''):
                                    paternal_grandfather = grandparent_id
                                    paternal_grandfather_index = j
                                elif (determine_sex(entities["pred_entity"][index + 2].split(' ')[0], name_list="names.json") == "female") and (paternal_grandmother == ''):
                                    paternal_grandmother = grandparent_id
                                    paternal_grandmother_index = j
                                elif paternal_grandmother == '':
                                    paternal_grandmother = grandparent_id
                                    paternal_grandmother_index = j
                                else:
                                    paternal_grandfather = grandparent_id
                                    paternal_grandfather_index = j
                        if paternal_grandfather != '':
                            found_paternal_grandparents = True
                            people = build_reciprocal_relationship(people, paternal_grandfather, principal_id, "grandparent")
                        if paternal_grandmother != '':
                            found_paternal_grandparents = True
                            people = build_reciprocal_relationship(people, paternal_grandmother, principal_id, "grandparent")
                        if (paternal_grandfather != '') and (paternal_grandmother != ''):
                            people = build_reciprocal_relationship(people, paternal_grandfather, paternal_grandmother, "spouse")
                    elif ("matern" in entities["pred_entity"][index].lower()) and (found_maternal_grandparents == False):
                        maternal_grandfather = ''
                        maternal_grandmother = ''
                        if entities["pred_label"][index + 1] == "PER":
                            for j in range(len(people_df)):
                                if people_df['pred_start'][j] == entities['pred_start'][index + 1]:
                                    grandparent_id = people_df['unique_id'][j]
                                    break
                            if determine_sex(entities["pred_entity"][index + 1].split(' ')[0], name_list="names.json") == "male":
                                maternal_grandfather = grandparent_id
                                maternal_grandfather_index = j
                            elif determine_sex(entities["pred_entity"][index + 1].split(' ')[0], name_list="names.json") == "female":
                                maternal_grandmother = grandparent_id
                                maternal_grandmother_index = j
                            else:
                                maternal_grandmother = grandparent_id
                                maternal_grandmother_index = j
                            if entities["pred_label"][index + 2] == "PER":
                                for j in range(len(people_df)):
                                    if people_df['pred_start'][j] == entities['pred_start'][index + 2]:
                                        grandparent_id = people_df['unique_id'][j]
                                        break
                                if (determine_sex(entities["pred_entity"][index + 2].split(' ')[0], name_list="names.json") == "male") and (maternal_grandfather == ''):
                                    maternal_grandfather = grandparent_id
                                    maternal_grandfather_index = j
                                elif (determine_sex(entities["pred_entity"][index + 2].split(' ')[0], name_list="names.json") == "female") and (maternal_grandmother == ''):
                                    maternal_grandmother = grandparent_id
                                    maternal_grandmother_index = j
                                elif maternal_grandmother == '':
                                    maternal_grandmother = grandparent_id
                                    maternal_grandmother_index = j
                                else:
                                    maternal_grandfather = grandparent_id
                                    maternal_grandfather_index = j
                        if maternal_grandfather != '':
                            found_maternal_grandparents = True
                            people = build_reciprocal_relationship(people, maternal_grandfather, principal_id, "grandparent")
                        if maternal_grandmother != '':
                            found_maternal_grandparents = True
                            people = build_reciprocal_relationship(people, maternal_grandmother, principal_id, "grandparent")
                        if (maternal_grandfather != '') and (maternal_grandmother != ''):
                            people = build_reciprocal_relationship(people, maternal_grandfather, maternal_grandmother, "spouse")
                elif ("matern" in entities["pred_entity"][index].lower()) and found_paternal_grandparents and (found_maternal_grandparents == False):
                    maternal_grandfather = ''
                    maternal_grandmother = ''
                    if entities["pred_label"][index + 1] == "PER":
                        for j in range(len(people_df)):
                            if people_df['pred_start'][j] == entities['pred_start'][index + 1]:
                                grandparent_id = people_df['unique_id'][j]
                                break
                        if determine_sex(entities["pred_entity"][index + 1].split(' ')[0], name_list="names.json") == "male":
                            maternal_grandfather = grandparent_id
                            maternal_grandfather_index = j
                        elif determine_sex(entities["pred_entity"][index + 1].split(' ')[0], name_list="names.json") == "female":
                            maternal_grandmother = grandparent_id
                            maternal_grandmother_index = j
                        else:
                            maternal_grandmother = grandparent_id
                            maternal_grandmother_index = j
                        if entities["pred_label"][index + 2] == "PER":
                            for j in range(len(people_df)):
                                if people_df['pred_start'][j] == entities['pred_start'][index + 2]:
                                    grandparent_id = people_df['unique_id'][j]
                                    break
                            if (determine_sex(entities["pred_entity"][index + 2].split(' ')[0], name_list="names.json") == "male") and (maternal_grandfather == ''):
                                maternal_grandfather = grandparent_id
                                maternal_grandfather_index = j
                            elif (determine_sex(entities["pred_entity"][index + 2].split(' ')[0], name_list="names.json") == "female") and (maternal_grandmother == ''):
                                maternal_grandmother = grandparent_id
                                maternal_grandmother_index = j
                            elif maternal_grandmother == '':
                                maternal_grandmother = grandparent_id
                                maternal_grandmother_index = j
                            else:
                                maternal_grandfather = grandparent_id
                                maternal_grandfather_index = j
                    if maternal_grandfather != '':
                        found_maternal_grandparents = True
                        people = build_reciprocal_relationship(people, maternal_grandfather, principal_id, "grandparent")
                    if maternal_grandmother != '':
                        found_maternal_grandparents = True
                        people = build_reciprocal_relationship(people, maternal_grandmother, principal_id, "grandparent")
                    if (maternal_grandfather != '') and (maternal_grandmother != ''):
                            people = build_reciprocal_relationship(people, maternal_grandfather, maternal_grandmother, "spouse")

                elif ((found_parents == False) and (found_godparents == False) and (found_paternal_grandparents == False) and (found_maternal_grandparents == False) and (found_enslaver == False)):
                    #ie if after all these checks, there are still no relationships found, then we have a case where we have a relationship but no assignment
                    #Note that this relies on setting ALL to FOUND by default, so I don't have to add to the code above each time
                    #Thus, we only flip it in the case that no relationships are found

                    entities.loc[index, "assigned"] = False
                    #print("Failed to find a category for relationship: " + entities["pred_entity"][index])



    if len(godparents) == 2:
        first_godparent_sex = determine_sex(godparents[0]["name"].split(' ')[0], name_list="names.json")
        second_godparent_sex = determine_sex(godparents[1]["name"].split(' ')[0], name_list="names.json")
        #if (first_godparent_sex != second_godparent_sex) or (first_godparent_sex == "unknown" and second_godparent_sex == "unknown"):
            #print("found possible godparent couple: ")
            #print(godparents[0]["name"])
            #print(godparents[1]["name"])

    for i in range(len(cat_char)):
        #build enslaver/enslaved person relationships
        if cat_char["category"][i] == "status":
            #skip if associated with first mention of principal
            char_start = cat_char['pred_start'][i]
            if char_start <= 25:
                continue

            #match enslaved couple to owner
            if (cat_char["pred_entity"][i].lower()[len(cat_char["pred_entity"][i]) - 1] == 's'):
                close_ep = -1
                far_ep = -1
                ens = -1
                for j in range(len(people_df)):
                    pers_start = people_df["pred_start"][j]
                    poss_diff = char_start - pers_start
                    if (ens == -1) and (poss_diff < 0) and (abs(poss_diff) < 25):
                        ens = j
                    elif (ens != -1) and (poss_diff < 0) and (abs(poss_diff) < abs(char_start - people_df["pred_start"][ens])):
                        ens = j
                    elif (close_ep == -1) and (poss_diff > 0) and (poss_diff < 50):
                        close_ep = j
                    elif (close_ep != -1) and (far_ep == -1) and (poss_diff > 0) and (poss_diff < char_start - people_df["pred_start"][close_ep]):
                        far_ep = close_ep
                        close_ep = j
                    elif (close_ep != -1) and (far_ep == -1) and (poss_diff > 0) and (poss_diff < 50):
                        far_ep = j
                    elif (close_ep != -1) and (far_ep != -1) and (poss_diff > 0) and (poss_diff < char_start - people_df["pred_start"][close_ep]):
                        far_ep = close_ep
                        close_ep = j
                    elif (close_ep != -1) and (far_ep != -1) and (poss_diff > 0) and (poss_diff < char_start - people_df["pred_start"][far_ep]):
                        far_ep = j
                if (ens != -1) and (close_ep != -1) and (far_ep != -1):
                    people = build_reciprocal_relationship(people, people_df["unique_id"][ens], people_df["unique_id"][close_ep], "enslaver")
                    people = build_reciprocal_relationship(people, people_df["unique_id"][ens], people_df["unique_id"][far_ep], "enslaver")
                elif (ens != -1) and (close_ep != -1):
                    people = build_reciprocal_relationship(people, people_df["unique_id"][ens], people_df["unique_id"][close_ep], "enslaver")
            #match enslaved person to owner
            elif "propiedad" in cat_char["pred_entity"][i].lower():
                for j in range(len(entities)):
                    if entities["pred_start"][j] == cat_char["pred_start"][i]:
                        signal_entity_index = j
                        break
                if found_enslaver and (entry_text.rfind("misma", cat_char["pred_start"][i] - 25, cat_char["pred_start"][i]) != -1):
                    if (entities["pred_label"][signal_entity_index - 1] == "PER") and (cat_char["pred_start"][i] - entities["pred_end"][signal_entity_index - 1] <= 20):
                        people = build_reciprocal_relationship(people, enslaver_id, entities["unique_id"][signal_entity_index - 1], "enslaver")
                        if (entities["pred_label"][signal_entity_index - 2] == "PER") and (entities["pred_end"][signal_entity_index - 2] - entities["pred_start"][signal_entity_index - 1] <= 5):
                            people = build_reciprocal_relationship(people, enslaver_id, entities["unique_id"][signal_entity_index - 2], "enslaver")
                elif (entities["pred_label"][signal_entity_index + 1] == "PER") and ((entities["pred_start"][signal_entity_index + 1] - cat_char["pred_start"][i]) <= 25):
                    for j in range(len(people_df)):
                        if people_df['pred_start'][j] == entities['pred_start'][signal_entity_index + 1]:
                            found_enslaver = True
                            enslaver_id = people_df["unique_id"][j]
                            people = build_reciprocal_relationship(people, enslaver_id, principal_id, "enslaver")
                            break
            else:
                ep = -1
                ens = -1
                for k in range(len(people_df)):
                    pers_start = people_df["pred_start"][k]
                    poss_diff = char_start - pers_start
                    if (ep == -1) and (poss_diff > 0) and (poss_diff < 50):
                        ep = k
                    elif (ens == -1) and (poss_diff < 0) and (abs(poss_diff) < 25):
                        ens = k
                    elif (ep != -1) and (poss_diff > 0) and (poss_diff < char_start - people_df["pred_start"][ep]):
                        ep = k
                    elif (ens != -1) and (poss_diff < 0) and (abs(poss_diff) < abs(char_start - people_df["pred_start"][ens])):
                        ens = k
                if (ep != -1) and (ens != -1):
                    people = build_reciprocal_relationship(people, people_df["unique_id"][ens], people_df["unique_id"][ep], "enslaver")
        #build parent/child relationships
        elif (((cat_char["category"][i] == "relationships") and ((cat_char["pred_entity"][i] == "hijo") or (cat_char["pred_entity"][i] == "hija") or (cat_char["pred_entity"][i] == "h") or (cat_char["pred_entity"][i] == "h."))) or (cat_char["category"][i] == "legitimacy")) and (found_parents == False):
            rel_start = cat_char["pred_start"][i]
            close_parent = -1
            far_parent = -1
            for l in range(len(people_df)):
                pers_start = people_df["pred_start"][l]
                poss_diff = rel_start - pers_start
                if (close_parent == -1) and (poss_diff < 0) and (abs(poss_diff) < 25):
                    close_parent = l
                elif (close_parent != -1) and (far_parent == -1) and (poss_diff < 0) and (abs(poss_diff) < abs(rel_start - people_df["pred_start"][close_parent])):
                    far_parent = close_parent
                    close_parent = l
                elif (close_parent != -1) and (far_parent == -1) and (poss_diff < 0) and ((pers_start - people_df["pred_end"][close_parent]) < 10):
                    far_parent = l
                elif (close_parent != -1) and (far_parent != -1) and (poss_diff > 0) and (abs(poss_diff) < abs(rel_start - people_df["pred_start"][close_parent])):
                    far_parent = close_parent
                    close_parent = l
                elif (close_parent != -1) and (far_parent != -1) and (poss_diff > 0) and (abs(poss_diff) < abs(rel_start - people_df["pred_start"][far_parent])):
                    far_parent = l
            if (close_parent != -1) and (far_parent != -1):
                people = build_reciprocal_relationship(people, people_df["unique_id"][close_parent], principal_id, "parent")
                people = build_reciprocal_relationship(people, people_df["unique_id"][far_parent], principal_id, "parent")
                if ((cat_char["category"][i] == "legitimacy") and ('r' not in cat_char["pred_entity"][i])) or ((cat_char["category"][i] == "relationships") and ((cat_char['pred_entity'][i] == 'h') or (cat_char['pred_entity'][i] == 'h.')) and ((entry_text[cat_char["pred_end"][i]] == 'l') or (entry_text[cat_char["pred_end"][i] + 1] == 'l'))):
                    people = build_reciprocal_relationship(people, people_df["unique_id"][close_parent], people_df["unique_id"][far_parent], "spouse")
                #future improvement (after normalization) if both parents enslaved and child not free, make sure child's status is enslaved
                #future improvement (after normalization) if child is enslaved, make sure reciprocal enslaver/enslaved person relationship exists with mother's enslaver
                found_parents = True
            elif (close_parent != -1):
                people = build_reciprocal_relationship(people, people_df["unique_id"][close_parent], principal_id, "parent")
                #future improvement (after normalization) if single parent is mother and she is enslaved and child not free, make sure child's status is enslaved
                #future improvement (after normalization) if child is enslaved, make sure reciprocal enslaver/enslaved person relationship exists with mother's enslaver
                found_parents = True

    #build parent-child relationships between parents and grandparents
    if found_parents and found_paternal_grandparents:
        if (far_parent != -1) and (determine_sex(people_df["pred_entity"][far_parent].split(' ')[0]) == "male"):
            if (paternal_grandmother != '') and (paternal_grandfather != ''):
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandmother_index], people_df["unique_id"][far_parent], "parent")
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandfather_index], people_df["unique_id"][far_parent], "parent")
            elif paternal_grandmother != '':
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandmother_index], people_df["unique_id"][far_parent], "parent")
            else:
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandfather_index], people_df["unique_id"][far_parent], "parent")
        elif (close_parent != -1) and (determine_sex(people_df["pred_entity"][close_parent].split(' ')[0]) == "male"):
            if (paternal_grandmother != '') and (paternal_grandfather != ''):
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandmother_index], people_df["unique_id"][close_parent], "parent")
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandfather_index], people_df["unique_id"][close_parent], "parent")
            elif paternal_grandmother != '':
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandmother_index], people_df["unique_id"][close_parent], "parent")
            else:
                people = build_reciprocal_relationship(people, people_df["unique_id"][paternal_grandfather_index], people_df["unique_id"][close_parent], "parent")
    if found_parents and found_maternal_grandparents:
        if (close_parent != -1)  and (determine_sex(people_df["pred_entity"][close_parent].split(' ')[0]) == "female"):
            if (maternal_grandmother != '') and (maternal_grandfather != ''):
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandmother_index], people_df["unique_id"][close_parent], "parent")
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandfather_index], people_df["unique_id"][close_parent], "parent")
            elif maternal_grandmother != '':
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandmother_index], people_df["unique_id"][close_parent], "parent")
            else:
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandfather_index], people_df["unique_id"][close_parent], "parent")
        elif (far_parent != -1) and (determine_sex(people_df["pred_entity"][far_parent].split(' ')[0]) == "female"):
            if (maternal_grandmother != '') and (maternal_grandfather != ''):
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandmother_index], people_df["unique_id"][far_parent], "parent")
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandfather_index], people_df["unique_id"][far_parent], "parent")
            elif maternal_grandmother != '':
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandmother_index], people_df["unique_id"][far_parent], "parent")
            else:
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandfather_index], people_df["unique_id"][far_parent], "parent")
        elif (close_parent != -1) and (far_parent == -1):
            if (maternal_grandmother != '') and (maternal_grandfather != ''):
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandmother_index], people_df["unique_id"][close_parent], "parent")
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandfather_index], people_df["unique_id"][close_parent], "parent")
            elif maternal_grandmother != '':
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandmother_index], people_df["unique_id"][close_parent], "parent")
            else:
                people = build_reciprocal_relationship(people, people_df["unique_id"][maternal_grandfather_index], people_df["unique_id"][close_parent], "parent")

    return people, entities

# Cell

def categorize_characteristics(entities_df, characteristics_df):
    '''
    determines which category each labeled characteristic belongs to
        characteristics_df: entities given the label "CHAR" from a single entry by an NER model

        returns: the same dataframe with an additional column containing a characteristic category
    '''

    vocabs = retrieve_controlled_vocabularies()
    categories = []
    uncategorized_characteristics = pd.DataFrame({"entry_no": pd.Series([], dtype="str"), "pred_entity": pd.Series([], dtype="str"), "pred_label": pd.Series([], dtype="str"), "pred_start": pd.Series([], dtype="int"), "pred_end": pd.Series([], dtype="int"), "assigned": pd.Series([], dtype="bool")})
    entities_df.reset_index(inplace = True, drop = True)

    for index, characteristic in entities_df.iterrows():
        #development
        #if characteristic["pred_entity"] == "libre":
            #print(characteristic["pred_label"])
        if characteristic["pred_label"] != "CHAR":
            continue
        category = None
        if characteristic["pred_entity"] in ["Natural", "nral", "Nat.l", "N.l", "nat.l", "natural"]:
            loc_indices = []
            for i, entity in entities_df.iterrows():
                if entity["pred_label"] == "LOC":
                    loc_indices.append(i)
            for loc_index in loc_indices:
                if ((loc_index - index) == 1):
                    category = "origin"
            if category == None:
                category = "legitimacy"
        for cat in vocabs:
            if (characteristic['pred_entity'] == 'h') or (characteristic['pred_entity'] == "h."):
                category = "relationships"
            elif characteristic["pred_entity"] == "propiedad":
                category = "status"
            if category != None:
                break
            for term in vocabs[cat]:
                if term in characteristic['pred_entity'].lower():
                        category = cat
                        break

        if category == None:
            uncategorized_characteristics = uncategorized_characteristics.append(entities_df.iloc[index])

        categories.append(category)

    characteristics_df["category"] = categories
    uncategorized_characteristics["category"] = None

    return characteristics_df, uncategorized_characteristics

# Cell
#this is currently configured specifically for baptisms/burials

def assign_characteristics(entry_text, entities_df, characteristics_df, unique_individuals, volume_metadata):
    '''
    matches all labeled characteristics to the correct individual(s) and builds triples
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        characteristics_df: entities given the label "CHAR" from a single entry by an NER model
        unique_individuals: as determined by id_unique_individuals and/or meta-function of disambig pipeline
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata

        returns: structured representation (a list of dictionaries)
    '''
    people = []
    ethnicities = retrieve_controlled_vocabularies()["ethnicities"]
    categorized_characteristics, uncat_char = categorize_characteristics(entities_df, characteristics_df)
    assignments = [None] * len(characteristics_df.index)
    categorized_characteristics.reset_index(inplace=True)
    unique_individuals.reset_index(inplace=True)

    for index in range(len(categorized_characteristics)):
        #development
        #if categorized_characteristics["pred_entity"][index] == "libre":
            #print("libre")
        if ((categorized_characteristics["category"][index] == "age") or (categorized_characteristics["category"][index] == "legitimacy")) and (volume_metadata["type"] == "baptism"):
            principal = determine_principals(entry_text, unique_individuals, 1)
            if principal != None:
                principal = determine_principals(entry_text, unique_individuals, 1)[0]
            else:
                principal = "Unknown principal"
            princ_loc = unique_individuals.index[unique_individuals["pred_entity"] == principal].tolist()
            for loc in princ_loc:
                if assignments[index] == None:
                    assignments[index] = unique_individuals["unique_id"][loc]
                else:
                    assignments[index] += ';' + unique_individuals["unique_id"][loc]
        elif (categorized_characteristics["category"][index] == "occupation") or (categorized_characteristics["category"][index] == "phenotype") or (categorized_characteristics["category"][index] == "ethnicities") or ((categorized_characteristics["category"][index] == "status") and (categorized_characteristics["pred_entity"][index].lower()[-1] != 's')):
            char_start = categorized_characteristics["pred_start"][index]
            lowest_diff = 50
            assign = None
            for i, person in unique_individuals.iterrows():
                person_start = person["pred_start"]
                diff = char_start - person_start
                if (diff > 0) and (diff < lowest_diff):
                    lowest_diff = diff
                    assign = i
            if assign != None:
                assignments[index] = unique_individuals["unique_id"][assign]
        elif categorized_characteristics["category"][index] == "status":
            char_start = categorized_characteristics["pred_start"][index]
            lowest_diff = 30
            second_lowest_diff = 50
            assign = [None, None]
            for i, person in unique_individuals.iterrows():
                person_start = person["pred_start"]
                diff = char_start - person_start
                if (diff > 0) and (diff < lowest_diff):
                    lowest_diff = diff
                    if assign[0] != None:
                        assign[1] = assign[0]
                        second_lowest_diff = lowest_diff
                    assign[0] = i
                elif (diff > 0) and (diff < second_lowest_diff) and (assign[0] != None):
                    second_lowest_diff = diff
                    assign[1] = i
            ids = []
            for a in assign:
                if a != None:
                    ids.append(unique_individuals["unique_id"][a])
            if len(ids) == 2:
                assignments[index] = ids[0] + ';' + ids[1]
            elif len(ids) == 1:
                assignments[index] = ids[0]
        elif categorized_characteristics["category"][index] == "origin":
            for i, entity in entities_df.iterrows():
                if entity["pred_start"] == categorized_characteristics["pred_start"][index]:
                    signal_entity_index = i
                    break
            if (signal_entity_index != 0) and (len(entities_df["pred_label"]) > (signal_entity_index + 1)) and (entities_df["pred_label"][signal_entity_index - 1] == "PER") and (entities_df["pred_label"][signal_entity_index + 1] == "LOC") and (entities_df["pred_start"][signal_entity_index + 1] - entities_df["pred_end"][signal_entity_index - 1] <= 20):
                place = entities_df["pred_entity"][signal_entity_index + 1]
                multiple = False
                if categorized_characteristics["pred_entity"][index] == "naturales":
                    multiple = True
                categorized_characteristics.at[index, "pred_entity"] = place
                for i, person in unique_individuals.iterrows():
                    if person["pred_start"] == entities_df["pred_start"][signal_entity_index - 1]:
                        assignments[index] = person["unique_id"]
                        break
                if multiple and (entities_df["pred_label"][signal_entity_index - 2] == "PER") and (entities_df["pred_start"][signal_entity_index - 1] - entities_df["pred_end"][signal_entity_index - 2] <= 10):
                    for i, person in unique_individuals.iterrows():
                        if person["pred_start"] == entities_df["pred_start"][signal_entity_index - 2]:
                            assignments[index] += ';' + person["unique_id"]
                            break
            elif (len(entities_df["pred_label"]) > (signal_entity_index + 1)) and (entities_df["pred_label"][signal_entity_index + 1] == "LOC"):
                place = entities_df["pred_entity"][signal_entity_index + 1]
                categorized_characteristics.at[index, "pred_entity"] = place
                principal = determine_principals(entry_text, unique_individuals, 1)
                if principal != None:
                    principal = determine_principals(entry_text, unique_individuals, 1)[0]
                else:
                    principal = "Unknown principal"
                princ_loc = unique_individuals.index[unique_individuals["pred_entity"] == principal].tolist()
                for loc in princ_loc:
                    if assignments[index] == None:
                        assignments[index] = unique_individuals["unique_id"][loc]
                    else:
                        assignments[index] += ';' + unique_individuals["unique_id"][loc]


    categorized_characteristics["assignment"] = assignments

    for i in range(len(unique_individuals.index)):

        characteristics = {"origin": None, "ethnicities":[], "age":None, "legitimacy":None,"occupation":[], "phenotype":[], "status":None, "titles":None, "ranks":None, "relationships":None}

        for eth in ethnicities:
            if eth in unique_individuals["pred_entity"][i].lower():
                characteristics["ethnicities"].append(eth[0].upper() + eth[1:])

        for j in range(len(categorized_characteristics.index)):
            if (categorized_characteristics["assignment"][j] == None):
                continue
            if unique_individuals["unique_id"][i] in categorized_characteristics["assignment"][j]:
                if (categorized_characteristics["category"][j] == "origin") or (categorized_characteristics["category"][j] == "age") or (categorized_characteristics["category"][j] == "legitimacy") or (categorized_characteristics["category"][j] == "status"):
                    characteristics[categorized_characteristics["category"][j]] = categorized_characteristics["pred_entity"][j]
                else:
                    characteristics[categorized_characteristics["category"][j]].append(categorized_characteristics["pred_entity"][j])

        person_record = {"id": unique_individuals["unique_id"][i], "name": unique_individuals["pred_entity"][i]}

        for key in characteristics:
            if ((key=="ethnicities") or (key == "occupation") or (key == "phenotype")) and (len(characteristics[key]) > 0):
                person_record[key] = characteristics[key][0]
                if (len(characteristics[key]) > 1):
                    for char in range(1,len(characteristics[key])):
                        person_record[key] += ';' + characteristics[key][char]
            elif (characteristics[key] != None) and (characteristics[key] != []):
                person_record[key] = characteristics[key]
            else:
                person_record[key] = None

        people.append(person_record)

    return people, categorized_characteristics

# Cell

def id_unique_individuals(entry_text, entities, volume_metadata):
    '''
    identifies all unique individuals that appear in an entry (i.e. removing all multiple mentions of the same person)
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: entities of all kinds extracted from that entry by an NER model
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata

        returns: a list of the unique individuals who appear in an entry AND (temporary?) unique IDs for each individual
    '''
    event_id = volume_metadata["id"] + '-' + entities.iloc[0]['entry_no']

    people_df = entities.loc[entities['pred_label'] == 'PER']
    people_df.reset_index(inplace=True)
    people_df = people_df.drop('index',axis=1)

    unique_individuals = people_df['pred_entity'].unique()
    unique_individuals = np.vstack([unique_individuals, [None] * len(unique_individuals)])

    for i in range(len(unique_individuals[0])):
        unique_individuals[1][i] = event_id + '-P' + str(i + 1)

    return unique_individuals

# Cell

def find_sus(entry_text, entities, sus_df, index):
    '''
    identifies corner cases: all entries where there are multiple entities that 1) have the same first name appearing
        multiple times, 2) have compound names and then a segment of that name appearing, and 3) have a full name with
        the first name by itself appearing
    Note that this should not be used in tandem with id_unique_individuals, as that function just drops the duplicate names

    params:
        entry_text: actual text for comparison
        entities: df of entities identified
        sus_df: either the empty df body or the df from previously loop iterations
        i: current row that the loop is on in DEMO_DF

    returns: df of all the entries that may be corner cases, in the same form demo_df, but with two added id columns
    '''
    #Set up
    people_df = entities.loc[entities['pred_label'] == 'PER']
    people_df.reset_index(inplace=True)
    people_df = people_df.drop('index',axis=1)

    my_rows = len(people_df.index)
    hold = my_rows * [0]
    people_df['name_status'] = hold
    first_names = []
    check_against = []
    dups = 0
    sus = 0

    #Get a list of all the first names that appear in the entities/people_df
    #  This is definitely not the most computationally efficient way to do this
    for i in range(my_rows):
        #Separate people based on whether it is a first name or a full/compound name
        if (" " in people_df.iloc[i,1]) or ("-" in people_df.iloc[i,1]):
            check_against.append(people_df.iloc[i,1])
        elif ~(" " in people_df.iloc[i,1]): #No spaces thus we are assuming it is a first name
            first_names.append(people_df.iloc[i,1])
    #Check to see whether they are subsets of full/compound names
    if len(first_names)>0 and len(check_against)>0:
        for j in range(len(first_names)):
            for k in range(len(check_against)):
                if first_names[j] in check_against[k]:
                    #Mark this entire entry as sus
                    sus = 1
    #Generally check to see if there are any duplicate entities (same name) in the entry
    if people_df['pred_entity'].duplicated().any():
        dups = 1;
    #Set the status column
    if sus and dups:
        status = 11 #ie both sus and dups are true
    elif sus:
        status = 10 #ie sus true, dups false
    elif dups:
        status = 0.01 #ie sus false, dups true
    else:
        status = 0
    #ie if the entry is suspect or has duplicates, then add it to sus_df
    if status>0:
        if len(sus_df.index)<1:
            data = [{'vol_titl':demo_df.iloc[index,0], 'vol_id':demo_df.iloc[index,1], 'fol_id':demo_df.iloc[index,2],
                    'text':demo_df.iloc[index,3],'entry_no':entry_no,'suspect':status}]
            sus_df = pd.DataFrame(data)
        else:
            sus_df = sus_df.append({'vol_titl':demo_df.iloc[index,0], 'vol_id':demo_df.iloc[index,1], 'fol_id':demo_df.iloc[index,2],
                    'text':demo_df.iloc[index,3],'entry_no':entry_no,'suspect':status},ignore_index=True)
    return sus_df

# Cell

def split_name_col(people_df):
    '''
    from the fed in entities, strips DF to only include people, then separates based on if it is a first name or a full name


    ### Functionality is not fully realized yet, could probably be generalized further, but this entire task may not be necessary
    '''
    #Set up
    my_rows = len(people_df.index)
    hold = my_rows * [0]
    people_df['name_status'] = hold

    #Separate into two based on first/single and full name status
    for i in range(my_rows):
        if "-" in people_df.iloc[i,1]:
            people_df.iloc[i,5] = 2 #2 therefore represents compound name
        elif " " in people_df.iloc[i,1]:
            people_df.iloc[i,5] = 1 #1 therefore represents a full name
        else: #Must be a single name
            #0 therefore represents a full name
            pass
    first_n = people_df[people_df.name_status == 0]
    full_n = people_df[people_df.name_status == 1]
    cmpd_n = people_df[people_df.name_status == 2]

    print("DF of first names")
    display(first_n.head())
    print("DF of full names")
    display(full_n.head())
    print("DF of compound names")
    display(cmpd_n.head())
    print("---------------------")

    return first_n, full_n, cmpd_n

# Cell

def disambiguate():
    '''
    goes through the problem cases previously identified and then applies split_name_col to break the entities down into
        the ones that may be
    '''
    people_df = entities.loc[entities['pred_label'] == 'PER']
    people_df.reset_index(inplace=True)
    people_df = people_df.drop('index',axis=1)

    first_n, full_n, cmpd_n = split_name_col(people_df)


# Cell

def determine_principals(entry_text, entities, n_principals):
    '''
    determines the principal of a single-principal event
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: entities of all kinds extracted from that entry by an NER model
        n_principals: expected number of principals

        returns: the principal(s) of the event in question, or None if no principal can be identified
    '''

    entry_text = entry_text.lower()
    principals = None

    if n_principals == 1:

        for index, entity in entities.iterrows():
            if (entity['pred_label'] == 'PER') and (entity['pred_start'] <= 20):
                principals = [entity['pred_entity']]

        if principals == None:
            prox = entry_text.find('oleos')
            if prox == -1:
                prox = entry_text.find('óleos')
            if prox != -1:
                for index, entity in entities.iterrows():
                    if (entity['pred_label'] == 'PER') and (abs(entity['pred_start'] - prox) <= 10):
                        principals = [entity['pred_entity']]

        if principals == None:
            prox = entry_text.find('nombre')
            if prox != -1:
                for index, entity in entities.iterrows():
                    if (entity['pred_label'] == 'PER') and (abs(entity['pred_start'] - prox) <= 10):
                        principals = [entity['pred_entity']]

    elif n_principals == 2:
        print("That number of principals is not supported yet.")
        #process marriage principals
    else:
        print("Invalid number of principals.")

    return principals

# Cell

def assign_relationships(entry_text, entities, unique_individuals):
    '''
    Relationship types:
        parent/child --> P. and P.s are parents
        godparents/godchildren --> P.P and p.s are godparents
        slaveholders/enslaved
        spouses
        grandparents
    Note that this function also calls the determine_principals function to help with rel assignment
    Returns: the relationship in forms of triples (subject, REL, relation) i.e. (me, godmother, my_godmother)
    '''
    rel_df = entities.loc[entities['pred_label'] == 'REL']
    rel_df.reset_index(inplace=True)
    rel_df = rel_df.drop('index',axis=1)
    #display(rel_df.head()) #Comment this out in final function, this is just for quick verification

    principal = determine_principals(entry_text, unique_individuals, 1)[0]
    principal_ID = unique_individuals.loc[unique_individuals.pred_entity==principal,"unique_id"].item()
    status = retrieve_controlled_vocabularies()["status"]

    rel = 0 #Variable telling us later whether or not this entry has any identified relationships
    previous = 0 #Variable telling us whether or not the previous REL combined two entities
    #(i.e. P. and P. into P.P. and thus can skip the second P. entity)
    event_id = volume_metadata["id"] + '-' + entities.iloc[0]['entry_no']
    my_relations = []
    m,n = entities.shape
    for i in range(m):
        if (entities.iloc[i,2]=='REL'):
            rel = 1 #Relationship present
            #We must check to make sure the first entity isn't a REL or it breaks the func due to positional index error
            if i==0 or i==(m-1):
                print("First/last entity is a REL, this functionality is not yet supported.")
            elif entities.iloc[i,1]=='P.P.':
                try:
                    #This gathers the first name, probably <NAME>
                    my_ID = unique_individuals.loc[unique_individuals.pred_entity==entities.iloc[i+1,1],"unique_id"].item()
                    my_triple = (principal_ID,'Padrino',my_ID)
                    my_relations.append(my_triple)
                    #This should be the second name, probably the madrina
                    my_ID = unique_individuals.loc[unique_individuals.pred_entity==entities.iloc[i+2,1],"unique_id"].item()
                    my_triple = (principal_ID,'Madrina',my_ID)
                    my_relations.append(my_triple)
                except:
                    print("Exception: had last entity in DF as a REL and thus out of bounds in current form of function")
            #Checking if we have back-to-back entities in the form of 'P.' followed by 'P.'
            elif ( (entities.iloc[i+1,2]=='REL') and ('P' in entities.iloc[i+1,1]) and (entities.iloc[i+2,2]=='PER') ):
                previous = 1
                my_ID = unique_individuals.loc[unique_individuals.pred_entity==entities.iloc[i+2,1],"unique_id"].item()
                my_triple = (principal_ID,'Padrino',my_ID)
                my_relations.append(my_triple)
                my_ID = unique_individuals.loc[unique_individuals.pred_entity==entities.iloc[i+3,1],"unique_id"].item()
                my_triple = (principal_ID,'Madrina',my_ID)
                my_relations.append(my_triple)
            #Skipping the second entity from the above case in the next iteration
            elif previous:
                previous = 0
            elif (entities.iloc[i+1,2]=='PER'):
                try:
                    my_ID = unique_individuals.loc[unique_individuals.pred_entity==entities.iloc[i+1,1],"unique_id"].item()
                    my_triple = (principal_ID,(entities.iloc[i,1]),my_ID)
                    my_relations.append(my_triple)
                except:
                    print("Exception: had last entity in DF as a REL and thus out of bounds in current form of function")
            elif ((entities.iloc[i,1].strip()=='P.') or (entities.iloc[i,1].strip()=='P')) and (entities.iloc[i+1,2]=='PER'):
                try:
                    my_ID = unique_individuals.loc[unique_individuals.pred_entity==entities.iloc[i+1,1],"unique_id"].item()
                    my_triple = (principal_ID,"Padre",my_ID)
                    my_relations.append(my_triple)
                except:
                    print("Exception: had last entity in DF as a REL and thus out of bounds in current form of function")
            else:
                print("Relationship found, but not between adjacent people")
        elif ((entities.iloc[i,1] in status) and (entities.iloc[i+1,2]=='PER')): #Identify the slave owner
            my_ID = unique_individuals.loc[unique_individuals.pred_entity==entities.iloc[i+1,1],"unique_id"].item()
            my_triple = (principal_ID,"Esclavista",my_ID)
            my_relations.append(my_triple)
    if rel:
        print(entry_text) #Uncomment this for verification
        print()
        print(my_relations)
    print("------------------------------------------")
    print()
    return my_relations

# Cell

def normalize_date(date):
    '''
    converts a string date to its numerical equivalent
        date: string identified by model as a date

        returns: date in the format YYYY-MM-DD (ISO 8601)'''

    if date == None:
        return "????", "??", "??"

    date = date.replace('.', '')
    date = date.replace(',', '')
    date = date.replace(';', '')
    date = normalize_text(date.lower(), "synonyms.json", context = "date")

    date = date.replace(',', '')
    day = None
    month = None
    month_pos = None
    year = None
    years = ["un", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve"]
    decades = ["diez", "veinte", "treinta", "cuarenta", "cincuenta", "sesenta", "setenta", "ochenta", "noventa"]
    month_days = ["primero", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve", "diez", "once", "doce", "trece", "catorce", "quince", "diez y seis", "diez y siete", "diez y ocho", "diez y nueve", "veinte", "veinte y uno", "veinte y dos", "veinte y tres", "veinte y cuatro", "veinte y cinco", "veinte y seis", "veinte y siete", "veinte y ocho", "veinte y nueve", "treinta", "treinta y uno"]
    months = ["enero", "febrero", "marzo", "abril", "mayo", "junio", "julio", "agosto", "septiembre", "octubre", "noviembre", "diciembre"]

    for word in (date.split(' ')):
        if word.lower() in months:
            month = '0' * (2 - len(str(months.index(word.lower()) + 1))) + str(months.index(word.lower()) + 1)
            month_pos = date.index(word)

    for n in month_days:
        if (month_pos != None):
            if (n in date) and (date.index(n) < month_pos):
                day = '0' * (2 - len(str(month_days.index(n) + 1))) + str(month_days.index(n) + 1)
        else:
            if (n in date) and (date.index(n) < 15):
                day = '0' * (2 - len(str(month_days.index(n) + 1))) + str(month_days.index(n) + 1)

    if day == None:
        if (month != None):
            month_ind = date.split(' ').index(months[int(month) - 1])
            for i in range(0, month_ind):
                if date.split(' ')[i].isdigit():
                    day = date.split(' ')[i]
        else:
            for i in range(0, 3):
                if len(date.split(' ')) <= i:
                    break
                if date.split(' ')[i].isdigit():
                    day = date.split(' ')[i]

    if (day != None) and (len(day) < 2):
        day = '0' + day

    centuries = ["seiscientos", "setecientos", "ochocientos", "novecientos"]
    century = None
    cent_pos = None
    for cent in centuries:
        if date.find(cent) != -1:
            if cent_pos == None:
                century = cent
                cent_pos = date.find(century)
            elif date.find(cent) < cent_pos:
                century = cent
                cent_pos = date.find(century)
    if century != None:
        year_str = None
        dec_str = None
        for yr in years:
            if date.find(yr, cent_pos + 1) != -1:
                if year_str == None:
                    year_str = yr
                else:
                    if date.find(yr, cent_pos) < date.find(year_str, cent_pos):
                        year_str = yr
        for decade in decades:
            if date.find(decade, cent_pos) != -1:
                dec_str = decade
                break
        if (year_str != None) and (dec_str != None):
            year = 1000 + 100 * (centuries.index(century) + 6) + 10 * (decades.index(dec_str) + 1) + years.index(year_str) + 1
            year = str(year)
        elif (year_str == None) and (dec_str != None):
            year = 1000 + 100 * (centuries.index(century) + 6) + 10 * (decades.index(dec_str) + 1)
            year = str(year)
        elif (year_str != None) and (dec_str == None):
            year = 1000 + 100 * (centuries.index(century) + 6) + years.index(year_str) + 1
            year = str(year)
        else:
            year = str(1000 + 100 * (centuries.index(century) + 6))

    if year == None:
        for token in date.split(' '):
            if token.isdigit() and (int(token) > 1000) and (int(token) < 2000):
                year = token

    if year == None:
        year = "????"
    if month == None:
        month = "??"
    if day == None:
        day = "??"

    return year, month, day

# Cell

def determine_event_date(entry_text, entities, event_type, volume_metadata, event_ref_pos=None):
    '''
    determines the date of a specific event
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: entities of all kinds extracted from that entry by an NER model
        event_type: this could be either a valid record_type OR a secondary event like a birth
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata
        event_ref_pos: optional index for reference to secondary event (to determine most likely date by proximity)

        returns: the date of the event in question, or None if no date can be identified
    '''
    date = None
    date_start = None

    if event_type != volume_metadata["type"]:
        primary_event_date = determine_event_date(entry_text, entities, volume_metadata["type"], volume_metadata)
        for index, entity in entities.iterrows():
            if (entity['pred_label'] == 'DATE') and (entity['pred_entity'] != primary_event_date) and (date == None):
                date = entity['pred_entity']
                date_start = entity['pred_start']
            elif (entity['pred_label'] == 'DATE') and (entity['pred_entity'] != primary_event_date):
                if event_ref_pos == None:
                    date = entity['pred_entity']
                else:
                    if (abs(event_ref_pos - entity['pred_start']) < abs(event_ref_pos - date_start)):
                        date = entity['pred_entity']
                        date_start = entity['pred_start']

    elif volume_metadata["type"] == "baptism":
        entry_length = len(entry_text)

        for index, entity in entities.iterrows():
            if (entity['pred_label'] == 'DATE') and (entity['pred_start'] <= (entry_length / 3)):
                date = entity['pred_entity']

    else:
        print("That event type is not supported yet.")

    year, month, day = normalize_date(date)

    return year + '-' + month + '-' + day

# Cell

def determine_event_location(entry_text, entities, event_type, volume_metadata, event_ref_pos=None):
    '''
    determines the location of a specific event
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: entities of all kinds extracted from that entry by an NER model
        event_type: this could be either a valid record_type OR a secondary event like a birth
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata
        event_ref_pos: optional index for reference to secondary event (to determine most likely date by proximity)

        returns: the location of the event in question, or None if no date can be identified
    '''
    location = None

    if event_type == volume_metadata["type"]:
        location = volume_metadata["institution"]

    return location

# Cell

def identify_cleric(entry_text, entities):
    '''
    identifies the cleric(s) associated with a sacramental entry
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: entities of all kinds extracted from that entry by an NER model

        returns: the associated cleric(s), or None if no date can be identified
    '''
    clerics = None

    for index, entity in entities.iterrows():
            if ((entity['pred_label'] == 'PER') and ((len(entry_text) - entity['pred_end']) <= 10) and (len(entry_text) > 100)):
                clerics = entity['pred_entity']
            #going to keep this condition for now, but it can create false positives when long, incorrect entities are extracted
            #from short and/or garbled entries
            elif (entity['pred_entity'] != None) and (len(entry_text) - entity['pred_end'] <= 2) and (entity['pred_label'] == 'PER'):
                clerics = entity['pred_entity']

    if clerics == None:
        pvs_label = None
        pvs_end = None
        for index, entity in entities.iterrows():
            if entity['pred_label'] == 'PER' and pvs_label == 'DATE' and (entity['pred_start'] - pvs_end) <= 15:
                clerics = entity['pred_entity']
            pvs_label = entity['pred_label']
            pvs_end = entity['pred_end']

    if clerics == None:
        entry_text = entry_text.lower()
        for index, entity in entities.iterrows():
            if entity['pred_label'] == 'PER' and entry_text.find("cura", entity['pred_start'] + len(entity['pred_entity'])) != -1 and ((entry_text.find("cura", entity['pred_start'] + len(entity['pred_entity']))) - entity['pred_end']) <= 15:
                clerics = entity['pred_entity']

    return clerics

# Cell

def build_event(entry_text, entities, event_type, principals, volume_metadata, n_event_within_entry, unique_individuals):
    '''
    builds out relationships related to a baptism or burial event
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: entities of all kinds extracted from that entry by an NER model
        event_type: this could be either a valid record_type OR a secondary event like a birth
        principals: the principal(s) of the event, as indicated by determine_principals
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata
        unique_individuals: as determined by id_unique_individuals and/or meta-function of disambig pipeline

        n_event_within_entry: event number within entry

        returns: structured representation of these relationships, including (but not necessarily limited to)
        the event's principal, the date of the event, the location of the event, and the associated cleric
    '''
    event_id = volume_metadata["id"] + '-' + entities.iloc[0]['entry_no'] + '-E' + str(n_event_within_entry)
    #it's possible that this function should also be returning an event iterator,
    #but for now I'm planning to do that in build_relationships

    if event_type == "baptism":
        if principals != None:
            principal = principals[0]
        else:
            principal = None
        date = determine_event_date(entry_text, entities, event_type, volume_metadata)
        location = determine_event_location(entry_text, entities, event_type, volume_metadata)
        cleric = identify_cleric(entry_text, entities)

        found_principal_id = False
        found_cleric_id = False
        for index, entity in unique_individuals.iterrows():
            if entity['pred_entity'] == principal:
                principal = entity['unique_id']
                found_principal_id = True
                continue
            elif entity['pred_entity'] == cleric:
                cleric = entity['unique_id']
                found_cleric_id = True

        if (principal != None) and (found_principal_id == False):
            principal = None
        if (cleric != None) and (found_cleric_id == False):
            cleric = None

    elif event_type == "birth":
        if principals != None:
            principal = principals[0]
        else:
            principal = None
        date = determine_event_date(entry_text, entities, event_type, volume_metadata)
        location = determine_event_location(entry_text, entities, event_type, volume_metadata)
        cleric = None

        found_principal_id = False
        for index, entity in unique_individuals.iterrows():
            if entity['pred_entity'] == principal:
                principal = entity['unique_id']
                found_principal_id = True
                break

        if (principal != None) and (found_principal_id == False):
            principal = None

    else:
        print("That event type can't be built yet.")
        return

    event_relationships = {"id": event_id, "type": event_type, "principal": principal, "date": date, "location": location, "cleric": cleric}

    return event_relationships

# Cell

def drop_obvious_duplicates(people, principals, cleric):
    '''
    first-pass disambiguation that drops multiple mentions of cleric and principal(s)
        people: df containing all entities labeled as people in the entry
        principals: as indicated by determine_principals

        returns: people df with obvious duplicates dropped
    '''
    found_principal = False
    found_cleric = False

    if len(principals) == 1:
        for index, person in people.iterrows():
            if (person['pred_entity'] == principals[0]) and (found_principal == False):
                found_principal = True
            elif person['pred_entity'] == principals[0]:
                people.drop(index, inplace=True)

            if cleric != None:
                if (person['pred_entity'] == cleric) and (found_cleric == False):
                    found_cleric = True
                elif person['pred_entity'] == cleric:
                    people.drop(index, inplace=True)

    people.reset_index(inplace=True)

    return people

# Cell

def id_obvious_duplicates(people, principals, cleric):
    '''
    first-pass disambiguation that identifies multiple mentions of cleric and principal(s)
        people: df containing all entities labeled as people in the entry with unique ids
        principals: as indicated by determine_principals
        cleric: as identified by identify_cleric

        returns: dictionary with two keys, each containing list of ids corresponding to each mention of individual in question
    '''

    obv_dups = {"principal":[], "cleric":[]}

    #clumsy fix for corner case where principal and cleric share first name
    if (len(principals) > 0) and (cleric != None):
        for principal in principals:
            for p in principal.split(' '):
                if p in cleric:
                    return obv_dups

    if (principals != None) and (len(principals) == 1):

        for index, person in people.iterrows():

            if (person['pred_entity'] == principals[0]) or ((len(person["pred_entity"].split(' ')) == 1) and (person["pred_entity"] in principals[0])):
                obv_dups["principal"].append(person["unique_id"])

            if (person['pred_entity'] == cleric):
                    obv_dups["cleric"].append(person["unique_id"])

    return obv_dups

# Cell

def assign_unique_ids(people, volume_metadata, entry_number=None):
    '''
    assigns unique ids to each person in an entry
        people: df containing all entities labeled as people in the entry that has received first-pass disambiguation
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata
        entry_number: compound id containing folio and folio-specific entry ids

        returns: people df with column containing unique ids appended, next available id
    '''
    size = len(people.index)

    if size == 0:
        return people, volume_metadata["id"] + '-' + entry_number + "-P1"

    unique_ids = []
    entry_id = volume_metadata["id"] + '-' + people.iloc[0]['entry_no']

    for i in range(size):
        unique_ids.append(entry_id + '-P' + str(i+1))

    people['unique_id'] = unique_ids
    next_id = entry_id + '-P' + str(i+2)

    return people, next_id

# Cell

def merge_records(records_to_merge):
    '''
    merge two or more dictionaries with some (but possibly not all) shared keys
        records_to_merge: list containing two or more dictionaries to merge

        returns: single, merged dictionary
    '''
    #why is this necessary?
    if len(records_to_merge) <= 1:
        return records_to_merge

    merged_record = records_to_merge[0]

    for i in range(1, len(records_to_merge)):
        record = records_to_merge[i]
        for key in record:
            if record[key] != None:
                if merged_record[key] == None:
                    merged_record[key] = record[key]
                else:
                    if key == 'relationships':
                        for rel in record[key]:
                            if rel in merged_record[key]:
                                continue
                            else:
                                merged_record[key].append(rel)
                    else:
                        values = record[key].split(';')
                        for value in values:
                            if value.lower() in merged_record[key].lower():
                                continue
                            else:
                                merged_record[key] += ';' + value

    return merged_record

# Cell

def merge_duplicates(people, duplicates):
    '''
    merge two or more dictionaries with some (but possibly not all) shared keys
        people: dataframe in which each row is a person
        duplicates: dictionary containing keys "principal" and "cleric";
        the value of each key is a list containing unique ids for each
        mention of the appropriate individual

        returns: dataframe with duplicate mentions of each individual type merged
    '''

    if (len(duplicates["principal"]) > 1):
        dups = []
        del_indices = []
        for person in people:
            if (person['id'] in duplicates["principal"]):
                dups.append(person)
                del_indices.append(people.index(person))
        minus = 0
        del_indices.sort()
        for index in del_indices:
            del people[index - minus]
            minus += 1
        people.append(merge_records(dups))

    if (len(duplicates["cleric"]) > 1):
        dups = []
        del_indices = []
        for person in people:
            if (person['id'] in duplicates["cleric"]):
                dups.append(person)
                del_indices.append(people.index(person))
        minus = 0
        del_indices.sort()
        for index in del_indices:
            del people[index - minus]
            minus += 1
        people.append(merge_records(dups))

    return people

# Cell

def build_new_person(people_df, next_id, person_type):
    '''
    appends a row representing a new person to an existing people df
        people_df: df containing all entities labeled as people in the entry with unique ids
        volume_metadata: metadata for the volume that the entry comes from, built by retrieve_volume_metadata
        person_type: type of person being added, e.g. "principal" or "cleric"

        returns: updated df with new person added and updated next available id
    '''

    curr_entry = next_id[next_id.find('-') + 1:next_id.find('P') - 1]
    person_name = "Unknown " + person_type
    person_number = next_id[next_id.find('P') + 1:]
    person_number = int(person_number)

    people_df = people_df.append({"entry_no": curr_entry, "pred_entity": person_name, "pred_label": "PER", "unique_id": next_id}, ignore_index=True)

    next_id = next_id[:next_id.find('P') + 1] + str(person_number + 1)

    return people_df, next_id

# Cell

def build_entry_metadata(entry_text, entities, path_to_volume_xml, entry_number=None):
    '''
    applies rules-based engine for relationship linking to the transcription of a single entry
        entry_text: the full text of a single entry, ported directly from spaCy to ensure congruity
        entities: entities of all kinds extracted from that entry by an NER model
        path_to_volume_xml: path to xml file containing full volume transcription and volume-level metadata
        entry_number: entry number, also from spaCy

        returns: three lists containing structured data about the people, places, and events that appear in the entry
    '''

    people = []
    places = []
    events = []

    volume_metadata = retrieve_volume_metadata(path_to_volume_xml)
    people_df = copy.deepcopy(entities.loc[entities['pred_label'] == 'PER'])
    people_df.reset_index(inplace=True)
    people_df, next_id = assign_unique_ids(people_df, volume_metadata, entry_number)
    characteristics_df = copy.deepcopy(entities.loc[entities['pred_label'] == 'CHAR'])
    characteristics_df.reset_index(inplace=True)
    dates_df = copy.deepcopy(entities.loc[entities['pred_label'] == 'DATE'])
    dates_df.reset_index(inplace=True)

    if volume_metadata["type"] == "baptism":
        principal = determine_principals(entry_text, entities, 1)

        if principal == None:
            people_df, next_id = build_new_person(people_df, next_id, "principal")
            principal = ["Unknown principal"]

        cleric = identify_cleric(entry_text, entities)

        events.append(build_event(entry_text, entities, "baptism", principal, volume_metadata, 1, people_df))

        if (len(dates_df.index) > 1):
            events.append(build_event(entry_text, entities, "birth", principal, volume_metadata, 2, people_df))

        characteristics_df, uncategorized_characteristics = categorize_characteristics(entities, characteristics_df)
        people, categorized_characteristics = assign_characteristics(entry_text, entities, characteristics_df, people_df, volume_metadata)
        #############################################################
        ### KAI EDIT: added entities here as output ###
        #############################################################
        people, entities = alt_assign_relationships(entry_text, entities, people_df, people, volume_metadata)
        obvious_duplicates = id_obvious_duplicates(people_df, principal, cleric)
        people = merge_duplicates(people, obvious_duplicates)

        #perform more sophisticated disambiguation

        for event in events:
            if (event["location"] != None) and (not (event["location"] in places)):
                places.append(event["location"])

    elif volume_metadata["type"] == "marriage":
        #process marriage record
        print("That record type is not supported yet.")
        return None
    elif volume_metadata["type"] == "burial":
        #process burial record
        print("That record type is not supported yet.")
        return None
    else:
        print("That record type is not supported yet.")
        return None

    return people, places, events, entities, characteristics_df, categorized_characteristics, uncategorized_characteristics