<jupyter_start><jupyter_text>ML-based Relation Extraction While the rule-based method performs heuristic search of subject-attribute entity pairs, the machine learning method classifies the candidate entity pairs into predefined relation classes. We created annotated datasets of relations between entities in clinical trial eligibility criteria, including the full data (data_re/allrealtions/train.tsv) and a subset of 2000 random samples (data_re/2000relations/train.tsv). The data can be used to train many types of machine learning models, including support vector machine, condition random field, and deep learning-based classification. We trained BERT-based models using a recent package (https://github.com/uf-hobi-informatics-lab/ClinicalTransformerRelationExtraction) with our relation training data.Here we show how to use a trained BERT-classification model to classify candidate relations using a sample data set.<jupyter_code>import pandas as pd
import ast
import csv
from nltk import pos_tag
from nltk.chunk import conlltags2tree
from nltk.tree import Tree
from itertools import combinations<jupyter_output><empty_output><jupyter_text>Test Data Preparation  Utility functions<jupyter_code># BIO tags
def tag(ner_result):
    tags = []
    for word in ner_result:
        # print('word, ', word)
        if 'SEP' not in word['tag'] and 'CLS' not in word['tag'] :
            tags.append((word['word'], word['tag']))
    return tags
# Create tree
def stanford_tree(bio_tagged):
    if len(bio_tagged) != 0:
        tokens, ne_tags = zip(*bio_tagged)
        pos_tags = [pos for token, pos in pos_tag(tokens)]

        conlltags = [(token, pos, ne) for token, pos, ne in zip(tokens, pos_tags, ne_tags)]
        ne_tree = conlltags2tree(conlltags)
        return ne_tree
    else:
        return None
# Parse named entities from tree
def structure_ne(ne_tree):
    if ne_tree is not None:
        ne = []
        for subtree in ne_tree:
            if type(subtree) == Tree:
                ne_label = subtree.label()
                ne_string = " ".join([token for token, pos in subtree.leaves()])
                ne.append((ne_string, ne_label))
        return ne
    else:
        return None<jupyter_output><empty_output><jupyter_text>Functions to generate test data with the right format<jupyter_code>def find_index(mentions, s):
    index=0
    indexes=[]
    for mention in mentions:
        if mention in s:
            c = mention[0]
            # Iterate over index
            for i in range(index, len(s)):            
                if s[i]==c:
                    if s[i:i+len(mention)] == mention:
                        indexes.append((mention,i,i+len(mention)))
                        index = i+len(mention)
                        break
    return indexes
def nerTokens_to_sentences(df_ner):

    df_ner = df_ner[['#nct_id','eligibility_type','criterion','NER']]
    df_ner = df_ner.drop_duplicates()
    
    result_ner = df_ner['NER']
    
    doc_index = df_ner['#nct_id']
    result_re = []
    stringlist = []
    for ner in result_ner:
        if isinstance(ner, str):
            ner = ast.literal_eval(ner)
        words = tag(ner)
        
        text=""
        for word in words:
            token = word[0]
            text+=token+" "
        
        tags_formatted = structure_ne(stanford_tree(words))
        mentions=[]
        tags=[]
        for one in tags_formatted:
            mentions.append(one[0])
            tags.append(one[1])
        
        indexes = find_index(mentions,text)
        # print(text)
        # print(indexes)        
        
        entitylist=[]
        # 3:6:age,7:14:upper_bound	< age @NUMBER
        string=""
        for i in range(0,len(indexes)):
            mention=indexes[i][0]
            start=indexes[i][1]
            end=indexes[i][2]
            label=tags[i]
            string+= str(start)+":"+str(end)+":"+label +","
                        
            entitylist.append((mention,label))

        string+="\t"+text
        # print(string)
        
        if string!="\t"+text: #don't process sentences without entities
            stringlist.append(string)
        
    return stringlist
# Annotate mentions with start [s] and end [e] positions in the sentence
def annotate_mentions(sent, doc_index):
    result = sent.split('\t')
    tags = result[0]
    # remove the last "," in tags
    tags = tags.rstrip(',')
    tagList = tags.split(',')
    text = result[1]

    # print('tagList: ', tagList)
    
    labels = []
    entity = 0

    for tagStr in tagList:
        tup = tagStr.split(":")
        #start = int(tup[0])-1
        #end = int(tup[1])-1
        start = int(tup[0])
        end = int(tup[1])
        text_new = (text[:start] + "[s1] " + text[start:end] + " [e1]" + text[end:], tup[2], str(doc_index)+"_"+str(entity), text[start:end])

        labels.append(text_new)
        entity = entity + 1

    return labels
def generate_candidate_relations(stringlist):
    
    df_re = pd.DataFrame(stringlist,columns=['Annotated String'])
    
    doc=0
    result_re = []
    for string in stringlist:
        
        # NOTE: prepare test data based on predicted NERs
        entitylist = annotate_mentions(string, doc)       
        
        relations=[]
        
        # do combo
        for combo in combinations(entitylist, 2):  # 2 for pairs, 3 for triplets, etc
            relations.append(('NonRel',combo[0], combo[1],doc))
            
        result_re.append(relations)
        doc=doc+1
    
    df_re['Relation'] = result_re
    df_re = df_re.explode('Relation')    
    df_re[['RelationType', 'Entity1', 'Entity2', 'DoC']] = pd.DataFrame(df_re['Relation'].tolist(), index=df_re.index)    
    df_re = df_re[['RelationType','Entity1','Entity2','DoC']]
    
    # Continue to finalize the format
    df_re[['Sent1', 'EntityType1', 'EntityIndex1','Mention1']] = pd.DataFrame(df_re['Entity1'].tolist(), index=df_re.index) 
    df_re[['Sent2', 'EntityType2', 'EntityIndex2','Mention2']] = pd.DataFrame(df_re['Entity2'].tolist(), index=df_re.index) 
    
    test_df = df_re[['RelationType','Sent1','Sent2','EntityType1','EntityType2','EntityIndex1','EntityIndex2','DoC']]

    test_df = test_df.copy()
    test_df['Sent1'] = test_df['Sent1'].str.replace("\n", "")
    test_df['Sent2'] = test_df['Sent2'].str.replace("s1", "s2").str.replace("e1", "e2").str.replace("\n", "")

    # Filter rows without relations
    test_df = test_df[~test_df['RelationType'].isnull()]
    
    test_df.columns = ['1','2','3','4','5','6','7','8']
    
    return test_df<jupyter_output><empty_output><jupyter_text>Generate candidate relations (test data) using sample data<jupyter_code># Load sample NER data
df_ner = pd.read_excel('data_ner/sample_trial_NER.xlsx')
df_ner
sentencelist = nerTokens_to_sentences(df_ner)
candidate_relations = generate_candidate_relations(sentencelist)
candidate_relations = candidate_relations.reset_index(drop=True)
candidate_relations
# Save candidate relations (test data), which will be used to make predictions
candidate_relations.to_csv("data_re/2000relations/test.tsv", sep="\t", index=False, quoting=csv.QUOTE_NONE)<jupyter_output><empty_output><jupyter_text>ML-based Relation Extraction You may train a BERT-based relation extraction model using this package (https://github.com/uf-hobi-informatics-lab/ClinicalTransformerRelationExtraction) with our training data. Here we show sample commands to train a model. And then use the test data to make predictions.For details, refer to the instruction at that repo.  Train a model 1) set up environment variablesexport CUDA_VISIBLE_DEVICES=1\data_dir=./data_re/2000relations\pof=./data_re/2000relations/predictions.txt\log=./data_re/2000relations/log.txt\nmd=./2000relations_model2) train model  python ./src/relation_extraction.py \\\    --model_type bert \\\    --data_format_mode 0 \\\    --classification_scheme 1 \\\    --pretrained_model bert-base-uncased \\\    --data_dir \\$data_dir \\\    --new_model_dir \\$nmd \\\    --predict_output_file \\$pof \\\    --overwrite_model_dir \\\    --seed 13 \\\    --max_seq_length 256 \\\    --cache_data \\\    --do_train \\\    --do_lower_case \\\    --train_batch_size 4 \\\    --eval_batch_size 4 \\\    --learning_rate 1e-5 \\\    --num_train_epochs 3 \\\    --gradient_accumulation_steps 1 \\\    --do_warmup \\\    --warmup_ratio 0.1 \\\    --weight_decay 0 \\\    --max_num_checkpoints 0 \\\    --log_file \\$log \\    Predict relations in the test data python ./src/relation_extraction.py \\\  --model_type bert \\\  --data_format_mode 0 \\\  --classification_scheme 1 \\\  --pretrained_model bert-base-uncased \\\  --data_dir \\$data_dir \\\  --new_model_dir \\$nmd \\\  --predict_output_file \\$pof \\\  --overwrite_model_dir \\\  --seed 13 \\\  --max_seq_length 256 \\\  --cache_data \\\  --do_predict \\\  --do_lower_case \\\  --eval_batch_size 4 \\\  --log_file \\$log \\  Generate prediction output<jupyter_code># Read ML predicted relation labels
prediction = pd.read_csv('data_re/2000relations/predictions.txt', sep='\t',header=None)
prediction
# Merge with input data
predicted_relations = candidate_relations
predicted_relations['prediction'] = prediction
predicted_relations = predicted_relations[['1','prediction','2','3','4','5','6','7','8']]
predicted_relations
predicted_relations.to_excel("data_re/sample_trial_relations_ML-prediction.xlsx", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13080749285638693, 'fraction_numerical': 0.014922213990898507, 'mean_word_length': 2.8904899135446684, 'pattern_counts': {'":': 3, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '559914', 'license': 'Apache-2.0', 'n_tokens_mistral': 3132, 'n_tokens_neox': 2978, 'n_words': 693, 'path': 'relation_extraction_ML-based.ipynb', 'repo_name': 'aipharma/ctbert', 'size': '9449'}