<jupyter_start><jupyter_text>TechNet 2.0 Import Libraries<jupyter_code>import pickle
import spacy
from spacy.tokenizer import Tokenizer
from spacy.util import compile_prefix_regex, compile_infix_regex, compile_suffix_regex
import re
import numpy as np
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import wordnet as wn
    
nlp = spacy.load('en_core_web_sm',disable = ['ner','textcat'])
with open('patents_ta_20191231_abstracts_fixed.pkl', 'rb') as f:
    patents = pickle.load(f)
with open('phrases_serhad_incl_nums.pkl','rb') as f:
    vocab = pickle.load(f)
def custom_tokenizer(nlp):
    '''
        Custom tokenizer for not to split words which include "-" and "/"
    '''
    infix_re = re.compile(r'''[.\,\?\:\;\...\‘\’\`\“\”\"\'~]''')
    prefix_re = compile_prefix_regex(nlp.Defaults.prefixes)
    suffix_re = compile_suffix_regex(nlp.Defaults.suffixes)

    return Tokenizer(nlp.vocab, prefix_search=prefix_re.search,
                                suffix_search=suffix_re.search,
                                infix_finditer=infix_re.finditer,
                                token_match=None)

nlp.tokenizer = custom_tokenizer(nlp)

lemmatizer = WordNetLemmatizer()<jupyter_output><empty_output><jupyter_text>To clean the text<jupyter_code>#the functions below helps in extracting relevant information from the brackets and fixing the numbers, leading to proper splitting of sentences
def text_remove(text):
    
    new_text = ''
    ls = []
    flag = 0
    double_flag = False
    brac_text = ''
    for n in range(len(text)):
        i = text[n]
        if i == '(':
            new_text = new_text[:-1]
            flag += 1
            if flag >1:
                brac_text += i
                double_flag = True
        elif i == ')':
            if flag > 1:
                flag -=1
                brac_text += i
                continue
            elif flag == 0:
                continue
                new_text += i
            if brac_text.count(' ') == 0:
                pass
            else:
                new_text += ' ' + brac_text + ' '
            flag -= 1
            brac_text = ''
        elif flag == 0:
            new_text += i
        else:
            brac_text += i
    text = convert_num(new_text)
    return text

def convert_num(text):
    flag1 = False
    flag2 = 0
    flag3 = False
    flag4 = False
    num = ''
    new_text = ''
    for i in text:
        if i.isdigit():
            num+=i
            flag1 = True
            flag3 = False
            flag4 = False
        elif i == '.' and flag1:
            num+=i
            flag2 +=1
            flag3 = True
        elif i == ',' and flag1:
            flag4 = True
        elif flag4:
            flag1 = False
            flag4 = False
            if flag2 == 1:
                num = float(num)
                n=0
                while num%1 != 0:
                    num*=10
                    n+=1
                if n==0:
                    pass
                else:
                    new_text += str(int(num)) + 'x10^-' + str(n) + ' '
            flag2 = 0
            num = ''
        elif flag1 and flag2 and (not flag3):
            flag1 = False
            if flag2 == 1:
                num = float(num)
                n=0
                while num%1 != 0:
                    num*=10
                    n+=1
                if n==0:
                    pass
                else:
                    new_text += str(int(num)) + 'x10^-' + str(n) + ' '
            flag2 = 0
            num = ''
        elif flag1 and flag2 and flag3:
            if flag2 == 1:
                new_text += num
            if flag2 == 2:
                num = float(num[:-1])
                n=0
                while num%1 != 0:
                    num*=10
                    n+=1
                if n==0:
                    pass
                else:
                    new_text += str(int(num)) + 'x10^-' + str(n) + '. '
            flag2 = 0
            num = ''
        else:
            flag1 = False
            new_text += num + i
            num = ''
    if flag1 and flag2 and (not flag3):
        if flag2 == 1:
            num = float(num)
            n=0
            while num%1 != 0:
                num*=10
                n+=1
            if n==0:
                pass
            else:
                new_text += str(int(num)) + 'x10^-' + str(n) + ' '
    elif flag1 and flag2 and flag3:
        if flag2 == 1:
            new_text += num
        if flag2 == 2:
            num = float(num[:-1])
            n=0
            while num%1 != 0:
                num*=10
                n+=1
            if n==0:
                pass
            else:
                new_text += str(int(num)) + 'x10^-' + str(n) + '. '
    return new_text<jupyter_output><empty_output><jupyter_text>Functions listed below use SpaCy and the clean text to find relations among terms by using verbs/qualitative relations<jupyter_code>all_pos = {'NOUN':True,'VERB':True,'ADJ':True,'ADP':True,'PUNCT':True,'CCONJ':True,'DET':True,'PRON':True,'PROPN':True,'PART':True,'INJ':True,'SYM':True}
fromby = {'from':True, 'by':True}
dcfindVerb1 = {'xcomp':True,'advcl':True,'pcomp':True,'conj':True,'acl':True,'ccomp':True,'relcl':True}
dcfindVerb2 = {'obj':True, 'pobj':True, 'dobj':True}
dcfindVerb3 = {'NOUN':True, 'PROPN':True, 'PRON':True}
dcfindVerb4 = {'subj':True, 'nsubj':True, 'nsubjpass':True, 'csubj':True, 'csubjpass':True,'obj':True, 'pobj':True, 'dobj':True}
dcprep = {'subj':True, 'nsubj':True, 'nsubjpass':True, 'csubj':True, 'csubjpass':True,'obj':True, 'pobj':True, 'dobj':True,'ROOT':True}
dcprep1 = {'to':True, 'in':True, 'with':True}
dcprep2 = {'of':'has'}
dcprep3 = {'NOUN':True,'PRON':True,'PROPN':True}
dcprep4 = {'prep':True,'agent':True}
dcxbyy = {'subj':True, 'nsubj':True, 'csubj':True}
dcxbyy1 = {'nsubjpass':True, 'csubjpass':True}
dcname = {'subj':True, 'nsubj':True, 'nsubjpass':True, 'csubj':True, 'csubjpass':True,'obj':True, 'pobj':True, 'dobj':True}
dcextra = {'obj':True,'dobj':True,'pobj':True,'conj':True,'ROOT':True}


def depSearch(dep, pos, parent, cand, current_level = 0, level_limit = -1):
    
#     parents = [(c, c.text, c.pos_, c.dep_, c.children) for c in parent.children]
    parents = [(parent, parent.text,parent.pos_,parent.dep_,parent.children)]
    init_cand =[x for x in cand]
    current_level += 1
    while level_limit==-1 or current_level<level_limit:
        children = []
        for par in parents:
            item = [(c, c.text, c.pos_, c.dep_,  c.children) for c in par[4]]
            if item != []:
                children += item
        if children == []:
            return cand
        else:
            for child in children:
                if dep.get(child[3],False) and pos.get(child[2],False):
                    cand.append(child[0])
        if init_cand != cand:
            return cand
        else:
            parents = children
        current_level += 1
        
def depSearch_rev(dep, pos, token, cand, current_level = 0, level_limit = -1):
    
    init_cand =[x for x in cand]
    current_level += 1    
    parent = token
    while level_limit==-1 or current_level<level_limit:    
        if parent.head == parent:
            return current_level
        else:
            parent = parent.head
            if dep.get(parent.dep_,False) and pos.get(parent.pos_,False):
                cand.append(parent)
        if init_cand != cand:
            return current_level
        current_level += 1
        
def depSearch1(dep, pos, parent, cand,  current_level = 0, level_limit = -1):

    if level_limit==-1 or current_level<level_limit:
        init_cand =[x for x in cand]
        if [x for x in parent.children]!=[]:
            children = [(c, c.text, c.pos_, c.dep_) for c in parent.children]
        else:
            return cand
    else:
        return cand
    current_level += 1
    if level_limit==-1 or current_level<level_limit:
        for child in children:
            if dep.get(child[3],False) and pos.get(child[2],False):
                cand.append(child[0])
        if cand!=init_cand:
            return cand
        else:
            for child in children:
                cand = depSearch1(dep, pos, child[0], cand, current_level, level_limit)
            return cand
    else:
        return cand        

def depSearch2(dep, pos, parent, cand, current_level = 0, level_limit = -1):
    
#     parents = [(c, c.text, c.pos_, c.dep_, c.children) for c in parent.children]
    parents = [(parent, parent.text,parent.pos_,parent.dep_,parent.children)]
    init_cand =[x for x in cand]
    current_level += 1
    while level_limit==-1 or current_level<level_limit:
        children = []
        for par in parents:
            item = [(c, c.text, c.pos_, c.dep_,  c.children) for c in par[4]]
            if item != []:
                children += item
        if children == []:
            return cand
        else:
            flag = False
            for child in children:
                if child[2] == 'VERB':
                    flag = True
                elif dep.get(child[3],False) and pos.get(child[2],False):
                    cand.append(child[0])
            if flag:
                return cand
        if init_cand != cand:
            return cand
        else:
            parents = children
        current_level += 1

def findInSubtree(dep, pos, parent, cand, current_level = 0, level_limit = -1):
    
    current_level += 1
    if level_limit == -1 or current_level<=level_limit:
        
        for child in parent.children:
            if pos.get(child.pos_,False) and dep.get(child.dep_,False):
                cand.append(child)
            findInSubtree(dep, pos, child, cand, current_level, level_limit)
        current_level += -1
        
    return cand
                                                        

def findInSubtreeChain(dep, pos, parent, cand, current_level = 0, level_limit = -1):
    current_level += 1
    if level_limit == -1 or current_level<=level_limit:
        cand_new = []
        for child in parent.children:
            if pos.get(child.pos_,False) and dep.get(child.dep_,False):
                cand_new.append(child)
                cand.append(child)
        if cand_new:
            for p in cand_new:
                findInSubtreeChain(dep, pos, p, cand)
                current_level -= 1

    return cand
dcfindROOT = {'obj':True, 'pobj':True, 'dobj':True}     
def findROOT(tokens,triplets,RT_triplets): #ROOT is the enabling verb
    for token in tokens:
        if token.dep_ == 'ROOT': #ROOT is the enabling verb
            n = token.i
            subj = []
            subjpass = []
            obj = []
            for i in token.children:
                if dcxbyy.get(i.dep_,False):
                    subj.append(i)
                if dcxbyy1.get(i.dep_,False):
                    subjpass.append(i)
            depSearch(dcfindROOT, all_pos, token, obj, 0, -1)
            if subj and obj:
                for i in subj:
                    for j in obj:
                        try:
                            if fromby.get(tokens[n+1].text,False) or fromby.get(tokens[n+2].text,False):
                                triplets.append([j, token, i])
                            else:
                                triplets.append([i, token, j])
                        except:
                            triplets.append([i, token, j])
            if subjpass and obj:
                for i in subj:
                    for j in obj:
                        triplets.append([i, token, j])
            if subjpass:
                for i in subjpass:
                    triplets.append(['Entity', token, i])
                    for j in obj:
                        RT_triplets.append([j, 'relatedTo', i])
            if obj!=[]:
                cand_list = []
                for k in token.children:
                    if dcprep4.get(k.dep_,False) and k.pos_ == 'ADP':
                        depSearch(dcfindVerb2, dcfindVerb3, k, cand_list, 0, 2)
                        parent = k
                        flag = True
                        while flag == True:
                            flag = False
                            for w in parent.children:
                                if w.dep_ == 'conj' and w.pos_ == 'ADP':
                                    depSearch(dcfindVerb2, dcfindVerb3, w, cand_list, 0, 2)
                                    parent = w
                                    flag = True
                for k in obj:
                    for y in cand_list:
                        RT_triplets.append([k,'relatedTo',y])  
            if subj!=[]:
                for k in token.children:
                    if k.dep_ == 'attr' and dcprep3.get(k.pos_,False):
                        for y in subj:
                            RT_triplets.append([y,'relatedTo',k])  
            flag = True
            parent = token
            while flag:
                flag = False
                for z in parent.children:
                    if z.dep_ == 'conj' and z.pos_ == 'VERB':
                        flag = True
                        parent = z
                        n = z.i
                        subj = []
                        subjpass = []
                        obj = []
                        for i in z.children:
                            if dcxbyy.get(i.dep_,False):
                                subj.append(i)
                            if dcxbyy1.get(i.dep_,False):
                                subjpass.append(i)
                        depSearch(dcfindROOT, all_pos, z, obj, 0, -1)
                        if subj and obj:
                            for i in subj:
                                for j in obj:
                                    try:
                                        if fromby.get(tokens[n+1].text,False) or fromby.get(tokens[n+2].text,False):
                                            triplets.append([j, z, i])
                                        else:
                                            triplets.append([i, z, j])
                                    except:
                                        triplets.append([i, z, j])
                        if subjpass and obj:
                            for i in subj:
                                for j in obj:
                                    triplets.append([i, z, j])
                        if subjpass:
                            for i in subjpass:
                                triplets.append(['Entity', z, i])
                                for j in obj:
                                    RT_triplets.append([j, 'relatedTo', i])
                        if obj!=[]:
                            cand_list = []
                            for k in z.children:
                                if dcprep4.get(k.dep_) and k.pos_ == 'ADP':
                                    depSearch(dcfindVerb2, dcfindVerb3, k, cand_list, 0, 2)
                            for k in obj:
                                for y in cand_list:
                                    RT_triplets.append([k,'relatedTo',y])  
                        if subj!=[]:
                            for k in z.children:
                                if k.dep_ == 'attr' and dcprep3.get(k.pos_,False):
                                    for y in subj:
                                        RT_triplets.append([y,'relatedTo',k])
                        
                        
def findVerb(tokens,triplets,RT_triplets):
    for token in tokens:
        if dcfindVerb1.get(token.dep_) and token.pos_ == 'VERB':
            tri = []
            n = token.i
            children = [x for x in token.children]
            dir_obj_cands = []
            dir_subj_cands = []
            dir_subjpass_cands = []
            for child in children:
                if dcfindVerb3.get(child.pos_,False) and dcxbyy.get(child.dep_,False):
                    dir_subj_cands.append(child)
                elif dcfindVerb3.get(child.pos_,False) and dcxbyy1.get(child.dep_,False):
                    dir_subjpass_cands.append(child)
                elif dcfindVerb2.get(child.dep_,False):
                    if dcfindVerb3.get(child.pos_,False):
                        dir_obj_cands.append(child)
                    else:
                        cld = [x for x in child.children]
                        for c in cld:
                            if c.dep_ == 'conj' and c.pos_ == 'NOUN':
                                dir_obj_cands.append(c)
            if not dir_obj_cands:
                indir_objs = []
                findInSubtree(dcfindVerb2, dcfindVerb3, token, indir_objs, level_limit = 3)
                if indir_objs:
                    indir_objs = sorted([(x, np.abs(x.i-token.i)) for x in indir_objs], key = lambda x:x[1])
                    indir_objs = [indir_objs[0][0]]
            if dir_obj_cands and dir_subj_cands:
                for subj in dir_subj_cands:
                    for obj in dir_obj_cands:
                        try:
                            if fromby.get(tokens[n+1].text,False) or fromby.get(tokens[n+2].text,False):
                                tri.append([obj, token, subj])
                                triplets.append([obj, token, subj])
                            else:
                                tri.append([subj, token, obj])
                                triplets.append([subj, token, obj])
                        except:
                            tri.append([subj, token, obj])
                            triplets.append([subj, token, obj])
            elif dir_obj_cands and dir_subjpass_cands:
                for subj in dir_subjpass_cands:
                    for obj in dir_obj_cands:
                        tri.append([obj, token, subj])
                        triplets.append([obj, token, subj])
            elif (dir_subjpass_cands or dir_subj_cands) and indir_objs:
                if dir_subjpass_cands:
                    for subj in dir_subjpass_cands:
                        for obj in indir_objs:
                            tri.append([obj, token, subj])
                            triplets.append([obj, token, subj])
                elif dir_subj_cands:
                    for subj in dir_subj_cands:
                        for obj in indir_objs:
                            try:
                                if fromby.get(tokens[n+1].text,False)or fromby.get(tokens[n+2].text,False) :
                                    tri.append([obj, token, subj])
                                    triplets.append([obj, token, subj])
                                else:
                                    tri.append([subj, token, obj])
                                    triplets.append([subj, token, obj])
                            except:
                                tri.append([subj, token, obj])
                                triplets.append([subj, token, obj])
            else:
                head = token.head
                count_ROOT = 0
                pre_cands = []
                count_rev = depSearch_rev(dcfindVerb4, dcfindVerb3, tokens[n], pre_cands)
                post_cands = []
                depSearch(dcfindVerb4, dcfindVerb3, tokens[n], post_cands)
                while head.head != head:
                    if head.dep_ == 'ROOT':
                        break
                    count_ROOT+=1
                    head = head.head
                if head.dep_ == 'ROOT' and count_ROOT<count_rev:
                    n = token.i
                    subj = []
                    subjpass = []
                    obj = []
                    flag = True
                    for i in triplets:
                        if type(i[1]) != str:
                            if i[1] == head:
                                subj.append(i[0])
                                flag = False
                    if flag:
                        for i in head.children:
                            if dcxbyy.get(i.dep_,False):
                                subj.append(i)
                            elif dcxbyy1.get(i.dep_,False):
                                subjpass.append(i)
                    for i in token.children:
                        if dcfindVerb2.get(i.dep_,False):
                            obj.append(i)
                    if obj == []:
                        depSearch(dcfindVerb2, dcfindVerb3, tokens[n], obj)
                    if subj and obj:
                        for i in subj:
                            for j in obj:
                                tri.append([i, tokens[n], j])
                                triplets.append([i, tokens[n], j])
                    elif subjpass and obj:
                        for i in subjpass:
                            for j in obj:
                                tri.append([i, tokens[n], j])
                                triplets.append([i, tokens[n], j])
                else:
                    if pre_cands and post_cands:
                        for i in range(len(post_cands)):
                            for j in range(len(pre_cands)):
                                try:
                                    if fromby.get(tokens[n+1].text,False) or fromby.get(tokens[n+2].text,False):  
                                        tri.append([post_cands[i], tokens[n], pre_cands[j]])
                                        triplets.append([post_cands[i], tokens[n], pre_cands[j]])                                    
                                    else:
                                        tri.append([pre_cands[j], tokens[n], post_cands[i]])
                                        triplets.append([pre_cands[j], tokens[n], post_cands[i]])
                                except:
                                    tri.append([pre_cands[j], tokens[n], post_cands[i]])
                                    triplets.append([pre_cands[j], tokens[n], post_cands[i]])
            if tri!=[]:
                cand_list = []
                for k in token.children:
                    if dcprep4.get(k.dep_,False) and k.pos_ == 'ADP':
                        depSearch(dcfindVerb2, dcfindVerb3, k, cand_list, 0, 2)
                        parent = k
                        flag = True
                        while flag == True:
                            flag = False
                            for w in parent.children:
                                if w.dep_ == 'conj' and w.pos_ == 'ADP':
                                    depSearch(dcfindVerb2, dcfindVerb3, w, cand_list, 0, 2)
                                    parent = w
                                    flag = True
                            
#                     elif k.dep_ == 'attr' and dcprep3.get(k.pos_,False):
#                         print(k)
                for k in tri:
                    for y in cand_list:
                        RT_triplets.append([k[2],'relatedTo',y])
def findPrep(tokens, triplets,RT_triplets):
    
    ntriplets = []
    nRT_triplets = []
    for n in range(len(tokens)):
        ntri = []
        nRT_tri = []
        if dcprep4.get(tokens[n].dep_,False) and tokens[n].pos_ == 'ADP':
            tre = []
            if tokens[n].text == 'of':
                tre = X_of_Y(tokens[n])
            elif tokens[n].text == 'for':
                tre = X_for_Y(tokens[n])
            elif tokens[n].text == 'by':
                tre = X_by_Y(tokens[n])
            elif tokens[n].text == 'from':
                tre = X_from_Y(tokens[n])
            elif dcprep1.get(tokens[n].text,False) :
                tre = X_with_Y(n, tokens)
            if tre!=[]:
                if type(tre[1]) == str:
                    RT_triplets.append(tre)
                else:
                    triplets.append(tre)
            else:
                if dcprep3.get(tokens[n].head.pos_,False) and dcprep.get(tokens[n].dep_,False): 
                    cands = []
                    depSearch(dcprep, dcprep3, tokens[n], cands)
                    if tokens[n].head.pos_ == 'SCONJ':
                        for k in tokens[n].head.children:
                            first = k
                    else:
                        first = tokens[n].head
                    if cands:
                        if cands[0].pos_ == 'VERB':
                            flag = 0
                            for j in triplets:
                                try:
                                    if first in [j[0],j[2]] and cands[0] == j[1]:
                                        flag = 1
                                        break
                                except:
                                    pass
                            if not flag:
                                nRT_tri = [first,'relatedTo',cands[0]]
                        else:
                            flag = 0
                            for j in triplets:
                                try:
                                    if first in [j[0],j[2]] and cands[0] in [j[0],j[2]]:
                                        flag = 1
                                        break
                                except:
                                    continue
                            if not flag:
                                ntri = [first,tokens[n],cands[0]]
                if not (ntri+nRT_tri):
                    if dcprep1.get(tokens[n].text,False):
                        refer1 = n-1
                        while refer1>=0 and dcprep3.get(tokens[refer1].pos_,False)==None or dcprep.get(tokens[refer1].dep_,False)==None:
                            refer1-=1
                        refer2 = n+1
                        while refer2<len(tokens) and dcprep3.get(tokens[refer2].pos_,False)==None or dcprep.get(tokens[refer2].dep_,False)==None:
                            refer2+=1
                        if refer1>=0 and refer2<len(tokens):
                            nRT_triplets.append([tokens[refer2],'relatedTo',tokens[refer1]])
                if ntri:
                    ntriplets.append(ntri)
                elif nRT_tri:
                    nRT_triplets.append(nRT_tri)
    conjFinder(ntriplets,tokens)
    conjFinder(ntriplets,tokens)
    for i in ntriplets:
        triplets.append(i)
    for i in nRT_triplets:
        RT_triplets.append(i)


def X_of_Y (token):
    
    triplet = []
    temp = []
    if token.pos_ == 'ADP' and token.dep_ == 'prep':
        children = [child for child in token.children]
        for child in children:
            if dcfindVerb2.get(child.dep_,False) and dcfindVerb3.get(child.pos_,False):
                temp = [child, 'has']
                break
        if temp and token.head.pos_=='NOUN':
            temp.append(token.head)
            triplet = temp
            return triplet
    return triplet


def X_for_Y (token):
    
    triplet = []
    temp = []
    if token.pos_ == 'ADP' and token.dep_ == 'prep':
        temp.append(token.head)
        children = [child for child in token.children]
        flag = 0
        for child in children:
            if dcfindVerb2.get(child.dep_,False) and dcfindVerb3.get(child.pos_,False):
                temp += ['relatedTo', child]
                flag = 1
                break
        if flag == 0:
            for child in children:
                if child.dep_ == 'pcomp' and child.pos_ == 'VERB':
                    obj_cands = []
                    depSearch(dcfindVerb2, dcfindVerb3, child, obj_cands, level_limit = 3)
                    if obj_cands:
                        flag = 1
                        temp += [child, obj_cands[0]]
                        break
        if flag:
            triplet = temp
    return triplet

def X_by_Y (token):
    
    triplet = []
    temp = []
    flag = 0
    if token.pos_ == 'ADP' and token.dep_ == 'prep':
        temp.append(token.head)
        children = [child for child in token.children]
        for child in children:
            if dcfindVerb2.get(child.dep_,False) and dcfindVerb3.get(child.pos_,False):
                temp += ['relatedTo', child]
                flag = 1
                break
    elif token.pos_ == 'ADP' and token.dep_ == 'agent' and token.head.pos_ == 'VERB':
        
        
        subj_cands = []
        depSearch(dcxbyy,dcfindVerb3, token.head, subj_cands)
        subjpass_cands = []
        depSearch(dcxbyy1, dcfindVerb3, token.head, subjpass_cands)
        if subj_cands:
            temp = [subj_cands[0], token.head]
        elif subjpass_cands:
            temp = [token.head, subjpass_cands[0]]
        
        else:
            x = token.head
            while x.pos_ != 'NOUN':
                x = x.head
            temp = [x, token.head]
        children = [child for child in token.children]   
        for child in children:
            if dcfindVerb2.get(child.dep_,False) and dcfindVerb3.get(child.pos_,False):
                flag = 1
                if subjpass_cands:
                    temp.insert(0, child)
                else:
                    temp.append(child)
                break
    if flag:
        triplet = temp
        triplet.reverse()
    return triplet



def X_from_Y (token):
    
    triplet = []
    temp = []
    flag = 0
    if token.pos_ == 'ADP' and token.dep_ == 'prep' and token.head.pos_ == 'VERB':
        objs_from = []
        findInSubtree(dcfindVerb2, {'NOUN':True}, token, objs_from, current_level = 0, level_limit = 1)
        objs_verb = []
        findInSubtree(dcfindVerb2, {'NOUN':True}, token.head, objs_verb, current_level = 0, level_limit = 1)
        if objs_from and objs_verb:
            temp = [objs_verb[0], 'has', objs_from[0]]
            flag = 1
        
    else:
        temp.append(token.head)
        children = [child for child in token.children]
        for child in children:
            if dcfindVerb2.get(child.dep_,False) and dcfindVerb3.get(child.pos_,False):
                temp += ['relatedTo', child]
                flag = 1
                break
    if flag:
        triplet = temp
    return triplet

def X_with_Y (n, tokens):
    
    triplet = []
#     high = min(n+3,len(tokens))
#     low = max(n-3,0)
    first = False
    last = False
    for i in range(n-1,-1):
        if dcfindVerb3.get(tokens[i].pos_,False):
            first = tokens[i]
            break
    for i in range(n+1,len(tokens)):
        if dcfindVerb3.get(tokens[i].pos_,False):
            last = tokens[i]
            break
    if first and last:
        triplet = [first,'relatedTo',last]
    return triplet


def X_PART_Y (tokens,triplets):
    
    temp = []
    for token in tokens:
        if token.pos_ == 'PART':
            triplets.append([token.head, 'has', token.head.head])
        
def nameCheck(triplets):
    
    dc = {}
    items = ['set','number','groups','who','that','therewith'] #and more!
    for i in items:
        dc[i] = True
    n = 0
    while n < len(triplets):
        i = triplets[n]
        try:
            if i[0] == i[2]:
                triplets.pop(n)
                continue
        except:
            pass
        flag = False
        if type(i[0]) != str:
            if dc.get(i[0].text.lower(),False):
                change = False
                for j in i[0].children:
                    if j.dep_ == 'prep':
                        for k in j.children:
                            if dcfindVerb4.get(k.dep_,False):
                                change = k
                                break
                    elif dcfindVerb4.get(j.dep_,False):
                        change = j
                        break
                if change:
                    try:
                        if i[2] != change:
                            i[0] = change
                            change = False 
                    except:
                        i[0] = change
                        change = False
#                 check the head or child, you need to check for amod
        if type(i[2]) != str:
            if dc.get(i[2].text.lower(),False):
                change = False
                for j in i[2].children:
                    if j.dep_ == 'prep':
                        for k in j.children:
                            if dcfindVerb4.get(k.dep_,False):
                                change = k
                                break
                    elif dcfindVerb4.get(j.dep_,False):
                        change = j
                        break
                if change:
                    try:
                        if i[0] != change:
                            i[2] = change
                            change = False 
                    except:
                        i[2] = change
                        change = False 
        n+=1
    
    
def conjFinder (triplets, doc):
    
    new_triplets = []
    for triplet in triplets:
        conj0 = []
        conj1 = []
        conj2 = []
        if type(triplet[0]) != str:
            depSearch1({'conj':True}, dcfindVerb3, triplet[0], conj0, 0, 2)        
            if triplet[0].dep_ == 'conj' and triplet[0].head.pos_ == 'NOUN':
                conj0.append(triplet[0].head)
        conj2 = []
        if type(triplet[2]) != str:
            depSearch1({'conj':True}, dcfindVerb3, triplet[2], conj2, 0, 2)        
            if triplet[2].dep_ == 'conj' and triplet[2].head.pos_ == 'NOUN':
                conj2.append(triplet[2].head)
        if conj0:
            new_triplets += [[x]+triplet[1:] for x in conj0]
        if conj2:
            new_triplets += [triplet[:2]+[x] for x in conj2]
        if type(triplet[1]) != str:
            depSearch1({'conj':True}, {'VERB':True}, triplet[1], conj1, 0, 2)        
            for i in conj1:
                obj = []
                depSearch({'conj':True}, {'VERB':True}, i, obj, 0, 2)
                for j in obj:
                    triplets.append([triplet[0],i,j])
    for i in new_triplets:
        flag = True
        for j in triplets:
            try:
                if i==j:
                    flag = False
            except:
                continue
        if flag:
            triplets.append(i)
        
def findNeg(tokens,triplets):
    
    for token in tokens:
        if 'neg' == token.dep_:
            for i in triplets:
                if type(i[1])!=str:
                    if i[1] == token.head:
                        #triplets
                        i.append('!')
 

def uniq(triplets,RT_triplets):
    
    dc = {}
    for i in triplets:
        if len(i) == 3:
            dc[i[0]+'~'+i[1]+'~'+i[2]] = True
        else:
            dc[i[0]+'~!'+i[1]+'~'+i[2]] = True
    n = []
    for i in list(dc.keys()):
        n.append([j for j in i.split('~')])
            
    dc_RT = {}
    dc_ext = {}
    for i in RT_triplets:
        if i[1] == 'relatedTo':
            if i[0]<i[2]:
                dc_RT[i[0]+'~relatedTo~'+i[2]] = True
            else:
                dc_RT[i[2]+'~relatedTo~'+i[0]] = True
        else:
            dc_ext[i[0]+'~'+i[1]+'~'+i[2]] = True
            
    for i in list(dc_ext.keys()):
        n.append([j for j in i.split('~')])
        
    dc = {}
    for i in n:
        if i[0]<i[2]:
            dc[i[0]+'~'+i[2]] = True
        else:
            dc[i[2]+'~'+i[0]] = True
        
    for i in list(dc_RT.keys()):
        a = [j for j in i.split('~')]
        if dc.get(a[0]+'~'+a[2],False):
            pass
        else:
            n.append(a)
    return n
                
def findExtra(tokens, triplets):
    for token in tokens:
        if dcextra.get(token.dep_) and dcfindVerb3.get(token.pos_):
            obj = []
            depSearch2(dcextra,dcfindVerb3,token,obj)
            for i in obj:
                triplets.append([token,'relatedTo',i])
def refine(triplets,RT_triplets,sent,vocab):
    new_tri = []
    trips = []
    RT_trips = []
    for n in range(len(triplets)):
        i = triplets[n]
        try:
            if not dcfindVerb3.get(i[0].pos_,False):
                continue
        except:
            pass
        try:
            if not dcfindVerb3.get(i[2].pos_,False):
                continue
        except:
            pass
        try:
            i[0] = phraseCompleter(i[0],sent,vocab,new_tri)
        except:
            pass
        try:
            i[1] = lemmatizer.lemmatize(i[1].text, 'v')
        except:
            i[1] = lemmatizer.lemmatize(i[1], 'v')
        try:
            i[2] = phraseCompleter(i[2],sent,vocab,new_tri)
        except:
            pass  
        trips.append(i)
    for n in range(len(RT_triplets)):
        i = RT_triplets[n]
        try:
            if not dcfindVerb3.get(i[0].pos_,False):
                continue
        except:
            pass
        try:
            if not dcfindVerb3.get(i[2].pos_,False):
                continue
        except:
            pass
        try:
            i[0] = phraseCompleter(i[0],sent,vocab,new_tri)
        except:
            pass
        try:
            i[2] = phraseCompleter(i[2],sent,vocab,new_tri)
        except:
            pass
        RT_trips.append(i)
    return new_tri + trips, RT_trips
    

def phraseCompleter(token, tokens, vocab, new_tri):

#     tokens = [x for x in doc]
    ind_token = token.i       
    i = ind_token-1
    temp = []
    while ind_token-4<i and i>-1:
        word = tokens[i]
        if (word.dep_ == 'compound' or word.dep_[-3:] == 'mod') and word.head.i<=ind_token:
            temp.append(word)
            i-=1
        else:
            break
    temp.reverse()
#     temp = [i for i in tokens[max(ind_token-3, 0):ind_token] if (i.dep_ == 'compound' or i.dep_[-3:] == 'mod')]
    attr = []
    flag = False
    for i in range(len(temp)):
        tempstr = ' '.join([x.text.lower() for x in temp[i:]])
        if token.pos_ == 'NOUN':
            tempstr = ' '.join([tempstr]+[lemmatizer.lemmatize(token.text, 'n')])
        else:
            tempstr = ' '.join([tempstr]+[token.text])
        tempstr = tempstr.lower()
        if vocab.get(tempstr,False):
            flag = True 
            break
    if flag:
        return tempstr
    else:
        if token.pos_ == 'NOUN':
            return token.lemma_.lower()
        else:
            return token.text.lower()<jupyter_output><empty_output><jupyter_text>A few basic examples to check if the functions work on short yet confusing (for the computer) sentences.<jupyter_code>sent = nlp('Lysozymes destroy bacteria to protect animals.')
triplets = []
findROOT(sent,triplets,triplets)
findVerb(sent,triplets,triplets)
print(triplets)
sent = nlp('Bacteria are destroyed to protect animals.')
triplets = []
findROOT(sent,triplets,triplets)
findVerb(sent,triplets,triplets)
print(triplets)
sent = nlp('Lysozymes destroy bacteria, protecting animals.')
triplets = []
findROOT(sent,triplets,triplets)
findVerb(sent,triplets,triplets)
print(triplets)
sent = nlp('By destroying invading bacteria, lysozyme protects animals.')
triplets = []
findROOT(sent,triplets,triplets)
findVerb(sent,triplets,triplets)
print(triplets)
sent = nlp('To protect animals, lysozymes destroy bacteria.')
triplets = []
findROOT(sent,triplets,triplets)
findVerb(sent,triplets,triplets)
print(triplets)
sent = nlp('Lysozymes destroy bacteria and protect animals')
triplets = []
findROOT(sent,triplets,triplets)
findVerb(sent,triplets,triplets)
print(triplets)
sent = nlp("In order to refine gold, thermosetting purifies gold by evaporating the impurities. the the")
triplets = []
findROOT(sent,triplets,triplets)
findVerb(sent,triplets,triplets)
print(triplets)<jupyter_output>[[order, refine, gold], [order, thermosetting, gold], [order, evaporating, impurities]]<jupyter_text>Patent Dataset to check the performance of the functions created aboce and will be used later to create the knowledge graph<jupyter_code>import numpy as np
pat_no = ['10402966','3960278','4060573','4143024','5310498','5728779','6279073','7296552','8660118','9437756']
tre4 = []
for i in pat_no:
    triplets = []
    RT_triplets = []
    texts = []
    print(i)
    if patents[i]['title']:
        title = patents[i]['title']
    else:
        title = ""
    if patents[i]['abstract']:
        abstract = patents[i]['abstract']
    else:
        abstract = ""
    text = title + '. ' + abstract
    if title or abstract:
        text = text_remove(text)
        text = text.replace('.sub.', '')
        text = text.replace('  ',' ')
        text = text.replace('.sup.','')
        text = text.replace('.dergree.','')
        text = text.replace('`',"'")
        text = text.replace('~','')
        text = text.replace('FIG. ','figure ')
        text = text.replace('e.g.','example')
        text = text.replace('i.e.','that is')
        for l in text.split('. '):
            for j in l.split('; '):
                k = j.replace('.','')
                k = k.replace(';','')
                texts.append(j)
    print(text)
    for j in texts:
        sent = nlp(j)
        findROOT(sent,triplets,RT_triplets)
        findVerb(sent,triplets,RT_triplets)
        findPrep(sent,triplets,RT_triplets)
#         for i in triplets + RT_triplets:
#             print(i)
        nameCheck(triplets)
        nameCheck(RT_triplets)
        conjFinder(triplets,sent)
        conjFinder(triplets,sent)
        conjFinder(triplets,sent)
        findNeg(sent,triplets)
        X_PART_Y(sent,triplets)
        findExtra(sent,RT_triplets)
        triplets,RT_triplets = refine(triplets,RT_triplets,sent,vocab)
    triplets = uniq(triplets,RT_triplets)
    for j in triplets:
        tre4.append(j)
        print(j)<jupyter_output>10402966
Systems and methods for identifying optimized ablation targets for treating and preventing arrhythmias sustained by reentrant circuits. Methods and systems for identifying optimized ablation targets for treating and preventing arrhythmias sustained by reentrant circuits are described. The methods comprise receiving at least one mesh generated from one or more images of a patient's heart, receiving activation data generated from one or more simulations of electrical-signal propagation over the at least one mesh, generating at least one flow graph based on the activation data and the at least one mesh, and applying a max-flow min-cut algorithm to the at least one flow graph to determine at least one of a number, one or more dimensions, and one or more locations of one or more ablation targets. Non-transitory computer-readable media storing a set of instructions for treating and preventing arrhythmias sustained by reentrant circuits are also described.
['ablation target', 'treat'[...]<jupyter_text>To create relations for the knowledge graph<jupyter_code>#this code block helps in getting clean sentences with the help of text_remove, convert_num, removing noise and splitting the sentences for us
texts = []
for a in list(patents.keys()):
    if patents[a]['title']:
        title = patents[a]['title']
    else:
        title = ""
    if patents[a]['abstract']:
        abstract = patents[a]['abstract']
    else:
        abstract = ""
    text = title + '. ' + abstract
    if title or abstract:
        text = text_remove(text)
        text = text.replace('.sub.', '')
        text = text.replace('  ',' ')
        text = text.replace('.sup.','')
        text = text.replace('.dergree.','')
        text = text.replace('`',"'")
        text = text.replace('~','')
        text = text.replace('FIG. ','figure ')
        text = text.replace('e.g.','example')
        text = text.replace('i.e.','that is')
        for i in text.split('. '):
            for j in i.split('; '):
                k = j.replace('.','')
                k = k.replace(';','')
                texts.append(j)
triplets = []
for sent in nlp.pipe(texts[:1000], n_threads=20, batch_size=100):
    tri = []
    findROOT(sent,tri)
    findVerb(sent,tri)
    findPrep(sent,tri)
    nameCheck(tri)
    conjFinder(tri,sent)
    conjFinder(tri,sent)
    conjFinder(tri,sent)
    findNeg(sent,tri)
    X_PART_Y(sent,tri)
    findExtra(sent,tri)
    refine(tri,sent,vocab)
    for i in tri:
        triplets.append(i)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11167891464104014, 'fraction_numerical': 0.012232899943470887, 'mean_word_length': 1.308969405868226, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 63, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '218145', 'license': 'MIT', 'n_tokens_mistral': 13485, 'n_tokens_neox': 12831, 'n_words': 3045, 'path': 'RelationFinder.ipynb', 'repo_name': 'mihirchhiber/TechNet', 'size': '44225'}