<commit_before>import os
from collections import defaultdict


from .mention_data import Cmention, Cquotation
from .naf_classes import Cconstituent_information, CquotationNaf


dep_extractor = None
head2deps = defaultdict(list)
dep2heads = defaultdict(list)
stop_words = []


def get_constituent(head):

    global dep_extractor

    mydeps = dep_extractor.get_full_dependents(head, [])
    if not head in mydeps:
        mydeps.append(head)
    return mydeps


def get_relevant_head_ids(nafobj):
    '''
    Returns list of term ids that head potential mention
    :param nafobj: input nafobj
    :return: list of term ids (string)
    '''

    nominal_pos = ['noun','pron','name']
    mention_heads = []
    for term in nafobj.get_terms():
        pos_tag = term.get_pos()
        if pos_tag in nominal_pos:
            mention_heads.append(term.get_id())
        #check if possessive pronoun
        elif pos_tag == 'det' and 'VNW(bez' in term.get_morphofeat():
            mention_heads.append(term.get_id())

    return mention_heads


def get_constituents(nafobj, mention_heads):

    global dep_extractor

    #global head2deps & check dependents?

    dep_extractor = nafobj.get_dependency_extractor()
    constituents = {}
    for head in mention_heads:
        mydeps = get_constituent(head)
        myConstituent = Cconstituent_information(head, mydeps)
        mwe, mods, apps = get_mwe_and_modifiers_and_appositives(head)
        myConstituent.set_multiword(mwe)
        myConstituent.set_modifiers(mods)
        myConstituent.set_appositives(apps)
        add_predicative_information(head, myConstituent)
        constituents[head] = myConstituent

    return constituents

def verify_span_uniqueness(found_spans, span):
    '''
    Function that checks whether entity is not listed twice (bug in cltl-spotlight; does not check whether entity has already been found
    :param found_spans: list of previously found spans
    :param span: current span
    :return: boolean (True if span has not been found before)
    '''

    for fspan in found_spans:
        if len(set(fspan) & set(span)) == len(span):
            return False
    return True


def find_closest_to_head(span):

    if len(span) == 1:
        return span[0]

    head_term_candidates = defaultdict(list)

    for tid in span:
        if tid in head2deps:
            count = 0
            for deprel in head2deps:
                if deprel[0] in span:
                    count += 1
            head_term_candidates[count].append(tid)
    if len(head_term_candidates) > 0:
        max_deps = sorted(head_term_candidates.keys())[-1]
        best_candidates = head_term_candidates.get(max_deps)
        if len(best_candidates) > 0:
            return best_candidates[0]

    return span[0]


def find_head_in_span(span):
    '''
    Function that return the identifier of the head in the span
    :param span: list of term identiers
    :return: term_id
    '''

    head_term = None
    for term in span:
        constituent = get_constituent(term)
        constituent.append(term)
        if set(span).issubset(set(constituent)):
            if head_term is None:
                head_term = term
       #     else:
       #         print('span has more than one head')
    if head_term is None:
        head_term = find_closest_to_head(span)
    return head_term



def get_named_entities(nafobj):
    '''
    Function that runs to entity layer and registers named entities
    :param nafobj: the input nafobject
    :return: dictionary of entities, linked to span and entity type
    '''
    entities = {}
    found_spans = []
    for entity in nafobj.get_entities():
        etype = entity.get_type()
        for ref in entity.get_references():
            espan = ref.get_span().get_span_ids()
            head_term = find_head_in_span(espan)
            full_span = get_constituent(head_term)
            myConstituent = Cconstituent_information(head_term, full_span)
            myConstituent.set_multiword(espan)
            mwe, mods, apps = get_mwe_and_modifiers_and_appositives(head_term)
            myConstituent.set_modifiers(mods)
            myConstituent.set_etype(etype)
            myConstituent.set_appositives(apps)
            add_predicative_information(head_term, myConstituent)
            if verify_span_uniqueness(found_spans, espan):

                if not head_term in entities:
                    entities[head_term] = myConstituent
                else:
                    entities[head_term + 'b'] = myConstituent
                found_spans.append(espan)

    return entities



def get_mention_spans(nafobj):
    '''
    Function explores various layers of nafobj and retrieves all mentions possibly referring to an entity
    :param nafobj: input nafobj
    :return: dictionary of head term with as value constituent object (head id full head, modifiers, complete constituent0
    '''
    mention_heads = get_relevant_head_ids(nafobj)
    mention_constituents = get_constituents(nafobj, mention_heads)
    return mention_constituents


def get_string_of_term(nafobj, tid):

    my_term = nafobj.get_term(tid)
    termstring = ''
    latest_offset = '-1'

    for wid in my_term.get_span().get_span_ids():
        my_tok = nafobj.get_token(wid)
        # add space between tokens
        if len(termstring) > 0 and int(my_tok.get_offset) > latest_offset:
            termstring += ' '
        termstring += my_tok.get_text()
        latest_offset = int(my_tok.get_offset()) + int(my_tok.get_length())
    return termstring


def get_string_of_span(nafobj, span):

    mstring = ''
    latest_offset = -1
    for tid in span:
        my_term = nafobj.get_term(tid)
        for wid in my_term.get_span().get_span_ids():
            my_tok = nafobj.get_token(wid)
            #add space between tokens
            if len(mstring) > 0 and int(my_tok.get_offset()) > latest_offset:
                mstring += ' '
            mstring += my_tok.get_text()
            latest_offset = int(my_tok.get_offset()) + int(my_tok.get_length())
    return mstring


def get_pos_of_term(nafobj, tid):

    term = nafobj.get_term(tid)
    return term.get_pos()

def get_pos_of_span(nafobj, span):

    pos_seq = []
    for tid in span:
        tpos = get_pos_of_term(nafobj, tid)
        pos_seq.append(tpos)

    return pos_seq

def identify_and_set_person(morphofeat, mention):

    if '1' in morphofeat:
        mention.set_person('1')
    elif '2' in morphofeat:
        mention.set_person('2')
    elif '3' in morphofeat:
        mention.set_person('3')


def identify_and_set_number(morphofeat, myterm, mention):

    if 'ev' in morphofeat:
        mention.set_number('ev')
    elif 'mv' in morphofeat:
        mention.set_number('mv')
    elif 'getal' in morphofeat:
        lemma = myterm.get_lemma()
        if lemma in ['haar','zijn','mijn','jouw','je']:
            mention.set_number('ev')
        elif lemma in ['ons','jullie','hun']:
            mention.set_number('mv')


def identify_and_set_gender(morphofeat, mention):

    if 'fem' in morphofeat:
        mention.set_number('fem')
    elif 'masc' in morphofeat:
        mention.set_number('masc')
    elif 'onz,' in morphofeat:
        mention.set_number('neut')

def set_is_relative_pronoun(morphofeat, mention):

    if 'betr,' in morphofeat:
        mention.set_relative_pronoun(True)
    if 'refl,' in morphofeat:
        mention.set_reflective_pronoun(True)



def analyze_nominal_information(nafobj, term_id, mention):

    myterm = nafobj.get_term(term_id)
    morphofeat = myterm.get_morphofeat()
    identify_and_set_person(morphofeat, mention)
    identify_and_set_gender(morphofeat, mention)
    identify_and_set_number(morphofeat, myterm, mention)
    set_is_relative_pronoun(morphofeat, mention)


def add_non_stopwords(nafobj, span, mention):
    '''
    Function that verifies which terms in span are not stopwords and adds these to non-stop-word list
    :param nafobj: input naf (for linguistic information)
    :param span: list of term ids
    :param mention: mention object
    :return:
    '''
    global stop_words
    non_stop_terms = []

    for tid in span:
        my_term = nafobj.get_term(tid)
        if not my_term.get_type() == 'closed' and not my_term.get_lemma().lower() in stop_words:
            non_stop_terms.append(tid)

    non_stop_span = get_span_in_offsets(nafobj, non_stop_terms)
    mention.set_no_stop_words(non_stop_span)

def add_main_modifiers(nafobj, span, mention):
    '''
    Function that creates list of all modifiers that are noun or adjective (possibly including head itself)
    :param nafobj: input naf
    :param span: list of term ids
    :param mention: mention object
    :return:
    '''

    main_mods = []
    for tid in span:
        term = nafobj.get_term(tid)
        if term.get_pos() in ['adj','noun']:
            main_mods.append(tid)

    main_mods_offset = get_span_in_offsets(nafobj, main_mods)
    mention.set_main_modifiers(main_mods_offset)


def get_sentence_number(nafobj, head):

    myterm = nafobj.get_term(head)
    tokid = myterm.get_span().get_span_ids()[0]
    mytoken = nafobj.get_token(tokid)
    sent_nr = int(mytoken.get_sent())

    return sent_nr




def create_mention(nafobj, constituentInfo, head, mid):
    '''
    Function that creates mention object from naf information
    :param nafobj: the input naffile
    :param span: the mention span
    :param span: the id of the constituent's head
    :param idnr: the idnr (for creating a unique mention id
    :return:
    '''

    if head is None:
        head_id = head
    else:
        head_id = get_offset(nafobj, head)

    span = constituentInfo.get_span()
    offset_ids_span = get_span_in_offsets(nafobj, span)
    mention = Cmention(mid, span=offset_ids_span, head_id=head_id)
    sentence_number = get_sentence_number(nafobj, head)
    mention.set_sentence_number(sentence_number)
    #add no stop words and main modifiers
    add_non_stopwords(nafobj, span, mention)
    add_main_modifiers(nafobj, span, mention)
    #mwe info
    full_head_tids = constituentInfo.get_multiword()
    full_head_span = get_span_in_offsets(nafobj, full_head_tids)
    mention.set_full_head(full_head_span)
    #modifers and appositives:
    relaxed_span = offset_ids_span
    for mod_in_tids in constituentInfo.get_modifiers():
        mod_span = get_span_in_offsets(nafobj, mod_in_tids)
        mention.add_modifier(mod_span)
        for mid in mod_span:
            if mid > head_id and mid in relaxed_span:
                relaxed_span.remove(mid)
    for app_in_tids in constituentInfo.get_appositives():
        app_span = get_span_in_offsets(nafobj, app_in_tids)
        mention.add_appositive(app_span)
        for mid in app_span:
            if mid > head_id and mid in relaxed_span:
                relaxed_span.remove(mid)
    mention.set_relaxed_span(relaxed_span)

    for pred_in_tids in constituentInfo.get_predicatives():
        pred_span = get_span_in_offsets(nafobj, pred_in_tids)
        mention.add_predicative(pred_span)

    #set sequence of pos FIXME: if not needed till end; remove
    #os_seq = get_pos_of_span(nafobj, span)
    #mention.set_pos_seq(pos_seq)
    #set pos of head
    if head != None:
        head_pos = get_pos_of_term(nafobj, head)
        mention.set_head_pos(head_pos)
        if head_pos in ['pron','noun','name']:
            analyze_nominal_information(nafobj, head, mention)

    begin_offset, end_offset = get_offsets_from_span(nafobj, span)
    mention.set_begin_offset(begin_offset)
    mention.set_end_offset(end_offset)

    return mention




def merge_two_mentions(mention1, mention2):
    '''
    Merge information from mention 1 into mention 2
    :param mention1:
    :param mention2:
    :return: updated mention
    '''

    if mention1.head_id == mention2.head_id:
        if set(mention1.span) == set(mention2.span):
            #if mention1 does not have entity type, take the one from entity 2
            if mention2.get_entity_type() is None:
                mention2.set_entity_type(mention1.get_entity_type())
        else:
            #if mention2 has no entity type, it's span is syntax based (rather than from the NERC module)
            if mention1.get_entity_type() is None:
                mention2.set_span(mention1.get_span())
    else:
        if mention1.get_entity_type() is None:
            mention2.set_head_id(mention1.get_head_id())
        else:
            mention2.set_entity_type(mention1.get_entity_type())

    return mention2



def merge_mentions(mentions, heads):
    '''
    Function that merges information from entity mentions
    :param mentions: dictionary mapping mention number to specific mention
    :param heads: dictionary mapping head id to mention number
    :return: list of mentions where identical spans are merged
    '''

    final_mentions = {}

    #TODO: create merge function and merge identical candidates

    for m, val in mentions.items():
        found = False
        for prevm, preval in final_mentions.items():
            if val.head_id == preval.head_id:
                updated_mention = merge_two_mentions(val, preval)
                final_mentions[prevm] = updated_mention
                found = True
            elif set(val.span) == set(preval.span):
               # print('same_set')
                updated_mention = merge_two_mentions(val, preval)
                final_mentions[prevm] = updated_mention
                found = True
        if not found:
            final_mentions[m] = val

    return final_mentions


def get_offsets_from_span(nafobj, span):
    '''
    Function that identifies begin and end offset for a span of terms
    :param nafobj: input naf
    :param span: list of term identifiers
    :return:
    '''

    offsets = []
    end_offsets = []
    for termid in span:
        offset = get_offset(nafobj, termid)
        length = get_term_length(nafobj, termid)
        offsets.append(offset)
        end_offsets.append(offset+length)

    begin_offset = 0
    end_offset = 0
    if len(offsets) > 0:
        begin_offset = sorted(offsets)[0]
        end_offset = sorted(end_offsets)[-1]

    return begin_offset, end_offset


def get_term_length(nafobj, term_id):
    '''
    Function that returns the length of a term
    :param nafobj: input naf
    :param term_id: id of term in question
    :return:
    '''

    my_term = nafobj.get_term(term_id)
    length = 0
    expected_offset = 0
    for wid in my_term.get_span().get_span_ids():
        my_token = nafobj.get_token(wid)
        offset = int(my_token.get_offset())
        token_length = int(my_token.get_length())
        length += token_length
        if expected_offset != 0 and expected_offset != offset:
            length += offset - expected_offset
        expected_offset = offset + token_length

    return length


def get_offset(nafobj, term_id):
    '''
    Function that returns beginning offset of term
    :param nafobj: input naf
    :param term_id: id of term in question
    :return:
    '''

    my_term = nafobj.get_term(term_id)
    offsets = []
    for wid in my_term.get_span().get_span_ids():
        my_token = nafobj.get_token(wid)
        offsets.append(int(my_token.get_offset()))

    term_offset = sorted(offsets)[0]
    return term_offset


def get_mwe_and_modifiers_and_appositives(head_id):
    '''
    Function that identifies full mwe head and posthead modifiers
    :param head_id: head_id
    :return: list of full head terms, list of posthead modifiers
    '''

    global head2deps, dep2heads

    mwe = [head_id]
    mods = []
    apps = []

    if head_id in head2deps:
        for deprel in head2deps.get(head_id):
            if deprel[1] == 'mwp/mwp':
                mwe.append(head_id)
            elif deprel[1] == 'hd/mod':
                dep_constituent = get_constituent(deprel[0])
                mods.append(dep_constituent)
            elif deprel[1] == 'hd/app':
                dep_constituent = get_constituent(deprel[0])
                apps.append(dep_constituent)

    return mwe, mods, apps


def add_predicative_information(head_id, myConstituent):
    '''<commit_msg>Simplify `get_offset`<commit_after>import os
from collections import defaultdict


from .mention_data import Cmention, Cquotation
from .naf_classes import Cconstituent_information, CquotationNaf


dep_extractor = None
head2deps = defaultdict(list)
dep2heads = defaultdict(list)
stop_words = []


def get_constituent(head):

    global dep_extractor

    mydeps = dep_extractor.get_full_dependents(head, [])
    if not head in mydeps:
        mydeps.append(head)
    return mydeps


def get_relevant_head_ids(nafobj):
    '''
    Returns list of term ids that head potential mention
    :param nafobj: input nafobj
    :return: list of term ids (string)
    '''

    nominal_pos = ['noun','pron','name']
    mention_heads = []
    for term in nafobj.get_terms():
        pos_tag = term.get_pos()
        if pos_tag in nominal_pos:
            mention_heads.append(term.get_id())
        #check if possessive pronoun
        elif pos_tag == 'det' and 'VNW(bez' in term.get_morphofeat():
            mention_heads.append(term.get_id())

    return mention_heads


def get_constituents(nafobj, mention_heads):

    global dep_extractor

    #global head2deps & check dependents?

    dep_extractor = nafobj.get_dependency_extractor()
    constituents = {}
    for head in mention_heads:
        mydeps = get_constituent(head)
        myConstituent = Cconstituent_information(head, mydeps)
        mwe, mods, apps = get_mwe_and_modifiers_and_appositives(head)
        myConstituent.set_multiword(mwe)
        myConstituent.set_modifiers(mods)
        myConstituent.set_appositives(apps)
        add_predicative_information(head, myConstituent)
        constituents[head] = myConstituent

    return constituents

def verify_span_uniqueness(found_spans, span):
    '''
    Function that checks whether entity is not listed twice (bug in cltl-spotlight; does not check whether entity has already been found
    :param found_spans: list of previously found spans
    :param span: current span
    :return: boolean (True if span has not been found before)
    '''

    for fspan in found_spans:
        if len(set(fspan) & set(span)) == len(span):
            return False
    return True


def find_closest_to_head(span):

    if len(span) == 1:
        return span[0]

    head_term_candidates = defaultdict(list)

    for tid in span:
        if tid in head2deps:
            count = 0
            for deprel in head2deps:
                if deprel[0] in span:
                    count += 1
            head_term_candidates[count].append(tid)
    if len(head_term_candidates) > 0:
        max_deps = sorted(head_term_candidates.keys())[-1]
        best_candidates = head_term_candidates.get(max_deps)
        if len(best_candidates) > 0:
            return best_candidates[0]

    return span[0]


def find_head_in_span(span):
    '''
    Function that return the identifier of the head in the span
    :param span: list of term identiers
    :return: term_id
    '''

    head_term = None
    for term in span:
        constituent = get_constituent(term)
        constituent.append(term)
        if set(span).issubset(set(constituent)):
            if head_term is None:
                head_term = term
       #     else:
       #         print('span has more than one head')
    if head_term is None:
        head_term = find_closest_to_head(span)
    return head_term



def get_named_entities(nafobj):
    '''
    Function that runs to entity layer and registers named entities
    :param nafobj: the input nafobject
    :return: dictionary of entities, linked to span and entity type
    '''
    entities = {}
    found_spans = []
    for entity in nafobj.get_entities():
        etype = entity.get_type()
        for ref in entity.get_references():
            espan = ref.get_span().get_span_ids()
            head_term = find_head_in_span(espan)
            full_span = get_constituent(head_term)
            myConstituent = Cconstituent_information(head_term, full_span)
            myConstituent.set_multiword(espan)
            mwe, mods, apps = get_mwe_and_modifiers_and_appositives(head_term)
            myConstituent.set_modifiers(mods)
            myConstituent.set_etype(etype)
            myConstituent.set_appositives(apps)
            add_predicative_information(head_term, myConstituent)
            if verify_span_uniqueness(found_spans, espan):

                if not head_term in entities:
                    entities[head_term] = myConstituent
                else:
                    entities[head_term + 'b'] = myConstituent
                found_spans.append(espan)

    return entities



def get_mention_spans(nafobj):
    '''
    Function explores various layers of nafobj and retrieves all mentions possibly referring to an entity
    :param nafobj: input nafobj
    :return: dictionary of head term with as value constituent object (head id full head, modifiers, complete constituent0
    '''
    mention_heads = get_relevant_head_ids(nafobj)
    mention_constituents = get_constituents(nafobj, mention_heads)
    return mention_constituents


def get_string_of_term(nafobj, tid):

    my_term = nafobj.get_term(tid)
    termstring = ''
    latest_offset = '-1'

    for wid in my_term.get_span().get_span_ids():
        my_tok = nafobj.get_token(wid)
        # add space between tokens
        if len(termstring) > 0 and int(my_tok.get_offset) > latest_offset:
            termstring += ' '
        termstring += my_tok.get_text()
        latest_offset = int(my_tok.get_offset()) + int(my_tok.get_length())
    return termstring


def get_string_of_span(nafobj, span):

    mstring = ''
    latest_offset = -1
    for tid in span:
        my_term = nafobj.get_term(tid)
        for wid in my_term.get_span().get_span_ids():
            my_tok = nafobj.get_token(wid)
            #add space between tokens
            if len(mstring) > 0 and int(my_tok.get_offset()) > latest_offset:
                mstring += ' '
            mstring += my_tok.get_text()
            latest_offset = int(my_tok.get_offset()) + int(my_tok.get_length())
    return mstring


def get_pos_of_term(nafobj, tid):

    term = nafobj.get_term(tid)
    return term.get_pos()

def get_pos_of_span(nafobj, span):

    pos_seq = []
    for tid in span:
        tpos = get_pos_of_term(nafobj, tid)
        pos_seq.append(tpos)

    return pos_seq

def identify_and_set_person(morphofeat, mention):

    if '1' in morphofeat:
        mention.set_person('1')
    elif '2' in morphofeat:
        mention.set_person('2')
    elif '3' in morphofeat:
        mention.set_person('3')


def identify_and_set_number(morphofeat, myterm, mention):

    if 'ev' in morphofeat:
        mention.set_number('ev')
    elif 'mv' in morphofeat:
        mention.set_number('mv')
    elif 'getal' in morphofeat:
        lemma = myterm.get_lemma()
        if lemma in ['haar','zijn','mijn','jouw','je']:
            mention.set_number('ev')
        elif lemma in ['ons','jullie','hun']:
            mention.set_number('mv')


def identify_and_set_gender(morphofeat, mention):

    if 'fem' in morphofeat:
        mention.set_number('fem')
    elif 'masc' in morphofeat:
        mention.set_number('masc')
    elif 'onz,' in morphofeat:
        mention.set_number('neut')

def set_is_relative_pronoun(morphofeat, mention):

    if 'betr,' in morphofeat:
        mention.set_relative_pronoun(True)
    if 'refl,' in morphofeat:
        mention.set_reflective_pronoun(True)



def analyze_nominal_information(nafobj, term_id, mention):

    myterm = nafobj.get_term(term_id)
    morphofeat = myterm.get_morphofeat()
    identify_and_set_person(morphofeat, mention)
    identify_and_set_gender(morphofeat, mention)
    identify_and_set_number(morphofeat, myterm, mention)
    set_is_relative_pronoun(morphofeat, mention)


def add_non_stopwords(nafobj, span, mention):
    '''
    Function that verifies which terms in span are not stopwords and adds these to non-stop-word list
    :param nafobj: input naf (for linguistic information)
    :param span: list of term ids
    :param mention: mention object
    :return:
    '''
    global stop_words
    non_stop_terms = []

    for tid in span:
        my_term = nafobj.get_term(tid)
        if not my_term.get_type() == 'closed' and not my_term.get_lemma().lower() in stop_words:
            non_stop_terms.append(tid)

    non_stop_span = get_span_in_offsets(nafobj, non_stop_terms)
    mention.set_no_stop_words(non_stop_span)

def add_main_modifiers(nafobj, span, mention):
    '''
    Function that creates list of all modifiers that are noun or adjective (possibly including head itself)
    :param nafobj: input naf
    :param span: list of term ids
    :param mention: mention object
    :return:
    '''

    main_mods = []
    for tid in span:
        term = nafobj.get_term(tid)
        if term.get_pos() in ['adj','noun']:
            main_mods.append(tid)

    main_mods_offset = get_span_in_offsets(nafobj, main_mods)
    mention.set_main_modifiers(main_mods_offset)


def get_sentence_number(nafobj, head):

    myterm = nafobj.get_term(head)
    tokid = myterm.get_span().get_span_ids()[0]
    mytoken = nafobj.get_token(tokid)
    sent_nr = int(mytoken.get_sent())

    return sent_nr




def create_mention(nafobj, constituentInfo, head, mid):
    '''
    Function that creates mention object from naf information
    :param nafobj: the input naffile
    :param span: the mention span
    :param span: the id of the constituent's head
    :param idnr: the idnr (for creating a unique mention id
    :return:
    '''

    if head is None:
        head_id = head
    else:
        head_id = get_offset(nafobj, head)

    span = constituentInfo.get_span()
    offset_ids_span = get_span_in_offsets(nafobj, span)
    mention = Cmention(mid, span=offset_ids_span, head_id=head_id)
    sentence_number = get_sentence_number(nafobj, head)
    mention.set_sentence_number(sentence_number)
    #add no stop words and main modifiers
    add_non_stopwords(nafobj, span, mention)
    add_main_modifiers(nafobj, span, mention)
    #mwe info
    full_head_tids = constituentInfo.get_multiword()
    full_head_span = get_span_in_offsets(nafobj, full_head_tids)
    mention.set_full_head(full_head_span)
    #modifers and appositives:
    relaxed_span = offset_ids_span
    for mod_in_tids in constituentInfo.get_modifiers():
        mod_span = get_span_in_offsets(nafobj, mod_in_tids)
        mention.add_modifier(mod_span)
        for mid in mod_span:
            if mid > head_id and mid in relaxed_span:
                relaxed_span.remove(mid)
    for app_in_tids in constituentInfo.get_appositives():
        app_span = get_span_in_offsets(nafobj, app_in_tids)
        mention.add_appositive(app_span)
        for mid in app_span:
            if mid > head_id and mid in relaxed_span:
                relaxed_span.remove(mid)
    mention.set_relaxed_span(relaxed_span)

    for pred_in_tids in constituentInfo.get_predicatives():
        pred_span = get_span_in_offsets(nafobj, pred_in_tids)
        mention.add_predicative(pred_span)

    #set sequence of pos FIXME: if not needed till end; remove
    #os_seq = get_pos_of_span(nafobj, span)
    #mention.set_pos_seq(pos_seq)
    #set pos of head
    if head != None:
        head_pos = get_pos_of_term(nafobj, head)
        mention.set_head_pos(head_pos)
        if head_pos in ['pron','noun','name']:
            analyze_nominal_information(nafobj, head, mention)

    begin_offset, end_offset = get_offsets_from_span(nafobj, span)
    mention.set_begin_offset(begin_offset)
    mention.set_end_offset(end_offset)

    return mention




def merge_two_mentions(mention1, mention2):
    '''
    Merge information from mention 1 into mention 2
    :param mention1:
    :param mention2:
    :return: updated mention
    '''

    if mention1.head_id == mention2.head_id:
        if set(mention1.span) == set(mention2.span):
            #if mention1 does not have entity type, take the one from entity 2
            if mention2.get_entity_type() is None:
                mention2.set_entity_type(mention1.get_entity_type())
        else:
            #if mention2 has no entity type, it's span is syntax based (rather than from the NERC module)
            if mention1.get_entity_type() is None:
                mention2.set_span(mention1.get_span())
    else:
        if mention1.get_entity_type() is None:
            mention2.set_head_id(mention1.get_head_id())
        else:
            mention2.set_entity_type(mention1.get_entity_type())

    return mention2



def merge_mentions(mentions, heads):
    '''
    Function that merges information from entity mentions
    :param mentions: dictionary mapping mention number to specific mention
    :param heads: dictionary mapping head id to mention number
    :return: list of mentions where identical spans are merged
    '''

    final_mentions = {}

    #TODO: create merge function and merge identical candidates

    for m, val in mentions.items():
        found = False
        for prevm, preval in final_mentions.items():
            if val.head_id == preval.head_id:
                updated_mention = merge_two_mentions(val, preval)
                final_mentions[prevm] = updated_mention
                found = True
            elif set(val.span) == set(preval.span):
               # print('same_set')
                updated_mention = merge_two_mentions(val, preval)
                final_mentions[prevm] = updated_mention
                found = True
        if not found:
            final_mentions[m] = val

    return final_mentions


def get_offsets_from_span(nafobj, span):
    '''
    Function that identifies begin and end offset for a span of terms
    :param nafobj: input naf
    :param span: list of term identifiers
    :return:
    '''

    offsets = []
    end_offsets = []
    for termid in span:
        offset = get_offset(nafobj, termid)
        length = get_term_length(nafobj, termid)
        offsets.append(offset)
        end_offsets.append(offset+length)

    begin_offset = 0
    end_offset = 0
    if len(offsets) > 0:
        begin_offset = sorted(offsets)[0]
        end_offset = sorted(end_offsets)[-1]

    return begin_offset, end_offset


def get_term_length(nafobj, term_id):
    '''
    Function that returns the length of a term
    :param nafobj: input naf
    :param term_id: id of term in question
    :return:
    '''

    my_term = nafobj.get_term(term_id)
    length = 0
    expected_offset = 0
    for wid in my_term.get_span().get_span_ids():
        my_token = nafobj.get_token(wid)
        offset = int(my_token.get_offset())
        token_length = int(my_token.get_length())
        length += token_length
        if expected_offset != 0 and expected_offset != offset:
            length += offset - expected_offset
        expected_offset = offset + token_length

    return length


def get_offset(nafobj, term_id):
    '''
    Function that returns beginning offset of term
    :param nafobj: input naf
    :param term_id: id of term in question
    :return:
    '''

    return min(
        int(nafobj.get_token(wid).get_offset())
        for wid in nafobj.get_term(term_id).get_span_ids()
    )


def get_mwe_and_modifiers_and_appositives(head_id):
    '''
    Function that identifies full mwe head and posthead modifiers
    :param head_id: head_id
    :return: list of full head terms, list of posthead modifiers
    '''

    global head2deps, dep2heads

    mwe = [head_id]
    mods = []
    apps = []

    if head_id in head2deps:
        for deprel in head2deps.get(head_id):
            if deprel[1] == 'mwp/mwp':
                mwe.append(head_id)
            elif deprel[1] == 'hd/mod':
                dep_constituent = get_constituent(deprel[0])
                mods.append(dep_constituent)
            elif deprel[1] == 'hd/app':
                dep_constituent = get_constituent(deprel[0])
                apps.append(dep_constituent)

    return mwe, mods, apps


def add_predicative_information(head_id, myConstituent):
    '''
{'fraction_non_alphanumeric': 0.08485996398186674, 'fraction_numerical': 0.0043159659690740855, 'mean_word_length': 2.7899505766062602, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1989470', 'n_tokens_mistral': 10735, 'n_tokens_neox': 10393, 'n_words': 2500}