<jupyter_start><jupyter_text>Rule-based Relation Extraction Here is a rule-based method which performs heuristic search of subject-attribute entity pairs from the NER result.<jupyter_code>import pandas as pd
import ast
from nltk import pos_tag
from nltk.chunk import conlltags2tree
from nltk.tree import Tree<jupyter_output><empty_output><jupyter_text>Utility functions<jupyter_code># BIO tags
def tag(ner_result):
    tags = []
    for word in ner_result:
        # print('word, ', word)
        if 'SEP' not in word['tag'] and 'CLS' not in word['tag'] :    #bert large may return 'CLS' tag
            tags.append((word['word'], word['tag']))
    return tags
# Create tree
def stanford_tree(bio_tagged):
    if len(bio_tagged) != 0:
        tokens, ne_tags = zip(*bio_tagged)
        pos_tags = [pos for token, pos in pos_tag(tokens)]

        conlltags = [(token, pos, ne) for token, pos, ne in zip(tokens, pos_tags, ne_tags)]
        ne_tree = conlltags2tree(conlltags)
        return ne_tree
    else:
        return None
# Parse named entities from tree
def structure_ne(ne_tree):
    if ne_tree is not None:
        ne = []
        for subtree in ne_tree:
            if type(subtree) == Tree:
                ne_label = subtree.label()
                ne_string = " ".join([token for token, pos in subtree.leaves()])
                ne.append((ne_string, ne_label))
        return ne
    else:
        return None
def find_index(mentions, s):
    index=0
    indexes=[]
    for mention in mentions:
        if mention in s:
            c = mention[0]
            # for ch in s:
            # Iterate over index
            for i in range(index, len(s)):            
                if s[i]==c:
                    if s[i:i+len(mention)] == mention:
                        indexes.append((mention,i,i+len(mention)))
                        index = i+len(mention)
                        break
    return indexes
# Apply relation extraction on the NER result
def call_RE_neighborpairs(df_ner):
    
    df_ner = df_ner[['#nct_id','eligibility_type','criterion','NER']]    
    df_ner = df_ner.drop_duplicates()
    
    result_ner = df_ner['NER']
    
    doc_index = df_ner['#nct_id']
    result_re = []
    for ner in result_ner:
        if isinstance(ner, str):
            ner = ast.literal_eval(ner)
        words = tag(ner)
        
        text=""
        for word in words:
            token = word[0]
            text+=token+" "
        
        tags_formatted = structure_ne(stanford_tree(words))
        mentions=[]
        tags=[]
        for one in tags_formatted:
            mentions.append(one[0])
            tags.append(one[1])
        
        indexes = find_index(mentions,text)
        # print(text)
        # print(indexes)
                
        entitylist=[]
        # 3:6:age,7:14:upper_bound	< age @NUMBER
        string=""
        for i in range(0,len(indexes)):
            mention=indexes[i][0]
            start=indexes[i][1]
            end=indexes[i][2]
            label=tags[i]
            string+= str(start)+":"+str(end)+":"+label +","
                        
            #entitylist.append((mention,label,start,end))
            entitylist.append((mention,label))

        string+="\t"+text
        # print(string)
                
        # generate relation statements
        relations=[]
        for i in range(len(entitylist)):
            if 'bound' in entitylist[i][1]:
                # print("index: ", i)
                left=entitylist[:i]

                # Find the neareast entity type on the left (of _bound)
                # find first element in a list with condition
                a = next((x for x in reversed(left) if x[1] in ["clinical_variable",'bmi','age']), None)        
                if a != None:
                    # print("nearby left: has value ", a, entitylist[i])
                    relations.append(('hasValue',a,entitylist[i]))
                b = next((x for x in reversed(left) if x[1] in ["allergy_name","cancer","chronic_disease","pregnancy",'treatment']), None)
                if a==None and b!=None:
                    # print("nearby left: has temp ", b, entitylist[i])
                    relations.append(('hasTemp',b,entitylist[i]))

                # Find the neareast entity type on the right (of _bound); if nothing on the left
                if(a==None and b==None):
                    right=entitylist[i:]
                    c = next((x for x in right if x[1] in ["clinical_variable",'bmi','age']), None)        
                    if c != None:
                        # print("nearby right: has value ", entitylist[i], c)
                        relations.append(('hasValue',entitylist[i],c))
                    d = next((x for x in right if x[1] in ["allergy_name","cancer","chronic_disease","pregnancy",'treatment']), None)
                    if c==None and d!=None:
                        print("nearby right: has temp ", entitylist[i], d)
                        relations.append(('hasTemp',entitylist[i],d))
            
        result_re.append(relations)
    
    print(len(df_ner))
    print(len(result_re))
    df_ner['Relation'] = result_re
    
    #reformat the relation statements
    df_re = df_ner[['#nct_id','eligibility_type','criterion','Relation']]
    df_re = df_re.explode('Relation')
    
    df_re[['RelationType', 'Entity1', 'Entity2']] = pd.DataFrame(df_re['Relation'].tolist(), index=df_re.index)
    
    df_re = df_re[['#nct_id','eligibility_type','criterion','RelationType','Entity1','Entity2']]
    
    return df_re<jupyter_output><empty_output><jupyter_text>Do relation extraction using sample data<jupyter_code># Load sample NER data
df_ner = pd.read_excel('data_ner/sample_trial_NER.xlsx')
df_ner
# Input is the NER result
df_relations = call_RE_neighborpairs(df_ner)
df_relations
df_relations.to_excel("data_re/sample_trial_relations_Rule-prediction.xlsx", index=False, encoding='utf8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12187761944677285, 'fraction_numerical': 0.0045264040234702435, 'mean_word_length': 2.1235602094240837, 'pattern_counts': {'":': 2, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '506478', 'license': 'Apache-2.0', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1810, 'n_words': 432, 'path': 'relation_extraction_Rule-based.ipynb', 'repo_name': 'aipharma/ctbert', 'size': '5965'}