<jupyter_start><jupyter_text>NLP System<jupyter_code># Main NLP Class

from pipeUtils import Document
from pipeUtils import Annotation
import re

class PadClassificationSystem:
    def __init__(self):
        #initiate necessary components        
        self.target_rules=self.getTargetRegexes()        
        self.negation_rules = self.getNegRegexes()
                
    def process(self, document):
        document_id = document.document_id
        ann_index=0
        for reg in self.target_rules:
            for match in reg.finditer(document.text):
                ann_id = 'NLP_'+ str(document_id) + '_' + str(ann_index)
                ann_index=ann_index+1
                new_annotation = Annotation(start_index=int(match.start()), 
                                    end_index=int(match.end()), 
                                    type='pe_ann',
                                    ann_id = ann_id
                                    )
                new_annotation.spanned_text = document.text[new_annotation.start_index:new_annotation.end_index]

                # Check negation right before the found target up to 30 charachers before, 
                # making sure that the pre-text does not cross the text boundary and is valid

                if new_annotation.start_index - 30 > 0:
                    pre_text_start = new_annotation.start_index - 30
                else:
                    pre_text_start = 0

                # ending index of the pre_text is the beginning of the found target    
                pre_text_end = new_annotation.start_index    

                # substring the document text to identify the pre_text string
                pre_text = doc.text[pre_text_start: pre_text_end]

                # We do not need to know the exact location of the negation keyword, so re.search is acceptable
                for neg_regex in self.negation_rules:
                    if re.search(neg_regex, pre_text):
                        new_annotation.attributes["Negation"] ="Negated"

                document.annotations.append(new_annotation)
        
        return document 
    
    def getTargetRegexes(self):
        target_regexes = []
        regexes = [            
            r'\bpe\b',
            r'\b(pulmonary|septic)?(\s{0,10})?(artery)?(\s{0,10})?(embolism|emboli|embolus)\b'
        ]
        for reg in regexes:
            target_regexes.append(re.compile(reg, re.IGNORECASE))
        return target_regexes

    def getNegRegexes(self):
        target_regexes = []
        regexes = [            
            r'evidence of'  ,
            r'does not have',
            r'denies',
            r'neg for',
            r'negative for',
            r'no central',
            r'no filling defect',
            r'\bno\s{1,10}(?=pe)',
            r'\bno\s{1,10}(?=pulmonary embolism)',
            r'bno\s{1,10}(?=septic embolism)',
            r'low likelihood',
            r'less likely',
            r'low suspicion for'
            
        ]
        for reg in regexes:
            target_regexes.append(re.compile(reg, re.IGNORECASE))
        return target_regexes<jupyter_output><empty_output><jupyter_text>Classification<jupyter_code># pe notes
import os
unid = 'u0496358'
path = "/home/"+str(unid)+"/BRAT/"+str(unid)+"/Project_pe_test" 
files = os.listdir(path)
len(files)
nlp_system = PadClassificationSystem()

import chardet
docs=dict()  

for i in files[:]:
    if ".txt" in i:                   
         
        with open(os.path.join(path,i), 'rb') as f:
            doc_txt = chardet.detect(f.read())        
        
        with open(os.path.join(path,i),encoding=doc_txt["encoding"]) as f:
            doc_text = f.read()           
            
            doc=Document(text=doc_text)            
            out_doc=nlp_system.process(doc)
            
            docs[i] = out_doc

doc_class=dict()
for k, v in docs.items():
    doc_class[k] = "pe_doc"
    for anns in docs[k].annotations:
        #print(anns.attributes)
        if anns.attributes != {}:
            doc_class[k] = "no_pe"
            break
            
for k in doc_class:
    print(k, '---', doc_class[k])<jupyter_output>90688_292.txt --- no_pe
65675_64.txt --- no_pe
48640_63.txt --- no_pe
86087_123.txt --- pe_doc
83838_106.txt --- pe_doc
72554_306.txt --- no_pe
15899_182.txt --- pe_doc
13867_266.txt --- pe_doc
61180_73.txt --- pe_doc
32113_141.txt --- no_pe
59381_293.txt --- pe_doc
58515_159.txt --- pe_doc
6878_279.txt --- no_pe
820_14.txt --- pe_doc
32113_109.txt --- pe_doc
49079_68.txt --- pe_doc
10568_20.txt --- no_pe
25764_268.txt --- no_pe
1498_225.txt --- no_pe
82326_55.txt --- pe_doc<jupyter_text>Document Level Validation<jupyter_code>path = path
posLab = "pe_doc"   #positive_DOC
negLab = "no_pe"    #negative_DOC
from pynlp_valid import Validnote
validnote = Validnote()
std_doc = validnote.readstd_ann(path, posLab, negLab)
precision, recall, f1 = validnote.validation(doc_class, std_doc, posLab, negLab)
print("*"*20)
print("Precision: ", precision)
print("Recall: ", recall)
print("F1: ", f1)
%%time
#  test case
nlp_system = PadClassificationSystem()
doc_text = '''
Patient has peripheral artery disease. ---------- \nPatient also has PVD or peripheral vascular\ndisease or pvd . 
\n The patient does not have any peripheral artery disease 
but has peripheral arterial disease . The patient denies having peripheral vascular disease . \n 
The patient has a femoral and illiac occlusion which is suggestive of peripheral arterial disease.
'''
doc=Document(text=doc_text, document_id='Doc1')
 
out_doc=nlp_system.process(doc)
print(out_doc.toString())<jupyter_output>Doc1
-------

Patient has peripheral artery disease. ---------- 
Patient also has PVD or peripheral vascular
disease or pvd . 

 The patient does not have any peripheral artery disease 
but has peripheral arterial disease . The patient denies having peripheral vascular disease . 
 
The patient has a femoral and illiac occlusion which is suggestive of peripheral arterial disease.

-------

CPU times: user 0 ns, sys: 0 ns, total: 0 ns
Wall time: 418 µs<jupyter_text>Mention Level Validation<jupyter_code>from pipeUtils import Annotation
from pipeUtils import Document
 
import os
import glob 
# Read all test documents
unid='u0496358'
project_1 = "Project_pe_train"
project_2 = "Project_pe_test"
path_1 = "/home/"+str(unid)+"/BRAT/"+str(unid)+"/"+project_1
path_2 = "/home/"+str(unid)+"/BRAT/"+str(unid)+"/"+project_2
test_docs=dict()
test_doc_paths = glob.glob(str(path_2+'/*.txt')) 
for d in test_doc_paths:
    doc = Document()
    #print(d)
    doc.load_document_from_file(d)
    #print(str(d[:-3])+'ann')
    doc.load_annotations_from_brat(str(d[:-3])+'ann')
    #print(os.path.basename(d))
    test_docs[os.path.basename(d)]=doc


#test_docs    
# Processing the all notes
nlp_system = PadClassificationSystem()

for doc_id in  test_docs.keys():

    test_docs[doc_id] = nlp_system.process(test_docs.get(doc_id))

print(test_docs.get("10568_20.txt").annotations[1].attributes)
test_docs.get("10568_20.txt").annotations[1].attributes=={'Negation': 'Negated'}<jupyter_output>{'Negation': 'Negated'}<jupyter_text><jupyter_code>tp_total = 0
fp_total = 0
fn_total = 0
tp_list_total = []
fp_list_total= []
fn_list_total = []
for doc_id in test_docs.keys():
    tp, fp, fn, tp_list, fp_list, fn_list = (test_docs.get(doc_id)).compare_types_by_span('PE','pe_ann', False)
    tp_total = tp_total + tp
    fp_total = fp_total + fp
    fn_total = fn_total + fn
    tp_list_total.extend(tp_list)
    fp_list_total.extend(fp_list)
    fn_list_total.extend(fn_list)

print('TP =',tp_total, 'FP =',fp_total, 'FN =',fn_total)

if tp_total > 0 :
    precision = tp_total / (tp_total + fp_total)
    print('Precision=',round(precision,3))

if tp_total > 0 :
    recall = tp_total / (tp_total + fn_total)
    print('Recall=',round(recall,3))

for a in tp_list_total:
    print(a[0].toString(),'||', a[1].toString())
for a in fp_list_total:
    print(a.toString())
for a in fn_list_total:
    print(a.toString())
tp_total = 0
fp_total = 0
fn_total = 0
tp_list_total = []
fp_list_total= []
fn_list_total = []
attributes_to_compare=[]
# To compare attributes, create a list of tuples for each pair to compare:
# attributes_to_compare.append[(A1_type, A1_att_name, A1_att_value),(A2_type, A2_att_name, A2_att_value)]
attributes_to_compare.append([('PE', 'Negation', 'Negated'),('pe_ann', 'Negation', 'Negated')])

for doc_id in test_docs.keys():
    tp, fp, fn, tp_list, fp_list, fn_list = (test_docs.get(doc_id)).compare_types_by_span_and_attributes('PE','pe_ann', False)
    #compare_types_by_span_and_attributes('PE','pe_ann', attributes_to_compare , False)
    tp_total = tp_total + tp
    fp_total = fp_total + fp
    fn_total = fn_total + fn
    tp_list_total.extend(tp_list)
    fp_list_total.extend(fp_list)
    fn_list_total.extend(fn_list)

print('TP =',tp_total, 'FP =',fp_total, 'FN =',fn_total)

if tp_total > 0 :
    precision = tp_total / (tp_total + fp_total)
    print('Precision=',round(precision,3))

if tp_total > 0 :
    recall = tp_total / (tp_total + fn_total)
    print('Recall=',round(recall,3))

#for a in tp_list_total:
#    print(a[0].toString(),'||', a[1].toString())
for a in fp_list_total:
    print(a.toString())
for a in fn_list_total:
    print(a.toString())<jupyter_output>TP = 89 FP = 37 FN = 32
Precision= 0.706
Recall= 0.736
NLP_90688_292.txt_3 pe_ann 688 690 PE 
NLP_90688_292.txt_9 pe_ann 3968 3985 pulmonary embolus 
NLP_65675_64.txt_0 pe_ann 1343 1345 PE 
NLP_65675_64.txt_2 pe_ann 2025 2043 pulmonary embolism 
NLP_65675_64.txt_3 pe_ann 2492 2510 pulmonary embolism 
NLP_65675_64.txt_4 pe_ann 10094 10112 pulmonary embolism 
NLP_48640_63.txt_0 pe_ann 12822 12824 PE 
NLP_48640_63.txt_1 pe_ann 8302 8320 pulmonary embolism 
NLP_86087_123.txt_0 pe_ann 7114 7135 pulmonary
   embolism 
NLP_86087_123.txt_2 pe_ann 7915 7933 pulmonary embolism 
NLP_83838_106.txt_2 pe_ann 6244 6246 PE 
NLP_83838_106.txt_5 pe_ann 7286 7288 PE 
NLP_83838_106.txt_6 pe_ann 5548 5566 pulmonary embolism 
NLP_72554_306.txt_0 pe_ann 139 141 PE 
NLP_72554_306.txt_1 pe_ann 413 415 PE 
NLP_72554_306.txt_2 pe_ann 617 619 pe 
NLP_72554_306.txt_3 pe_ann 3334 3336 PE 
NLP_72554_306.txt_5 pe_ann 4136 4154 pulmonary embolism 
NLP_13867_266.txt_4 pe_ann 1125 1143 pulmonary embolism 
NLP_61180_73.t[...]<jupyter_text>System deployment<jupyter_code># imports
import pymysql
import pandas as pd
import getpass
conn = pymysql.connect(host="mysql.chpc.utah.edu",
                       port=3306,user="mimicclass",
                       getpass.getpass("Enter MySQL passwd for jovyan"),                       
                       db='mimic3')
cursor = conn.cursor()
# identify patients with PAD for reference standard
# pad_data 
docs_text = pd.read_sql("SELECT subject_id, text from NOTEEVENTS where text like '%d-dimer% CT %pulmonary emboli%'",conn)
try:
    conn.close()
except:
    print("Connection is already closed!")
print(docs_text.columns)
print("Number of records = ", len(docs_text))

docs_text.head(1)
final_nlp_system = PadClassificationSystem()
output = []
counter = 0
for index , row in docs_text.iterrows():    
    doc = Document(document_id=str(row.subject_id) + '_' + str(index), text=row.text)
    final_nlp_system.process(doc)
    if(len(doc.annotations) > 0):
        i = 1
        for a in doc.annotations:
            if( a.type == 'pe_ann'):
                neg_flag = 0
                # Switch the flag to 1 when the mention is negated
                # if('Negated' in a.attributes): # definite_negated_existence Negated
                if a.attributes == {'Negation': 'Negated'}:
                    neg_flag=1
                ### Each row in the dictionary
                record_id  = str(row.subject_id) + '_' + str(index)+'_'+str(i)
                subject_id =  row.subject_id
                note_id = str(row.subject_id) + '_' + str(index)
                annotation_type = a.type
                snippet = doc.text[int(a.start_index): int(a.end_index)]
                out_list = [record_id, subject_id, note_id, annotation_type, \
                            a.start_index, a.end_index, \
                            snippet, neg_flag]
                output.append(out_list)
                i=i+1
                counter=counter+1
                # Print . after 10 identified records
                if counter%10 == 0:
                    print('.', end='')
        else:
            continue
        break<jupyter_output>.............................................................................................................................................................................................<jupyter_text>Output to CSV file<jupyter_code>columns=['record_id','subject_id', 'note_id', 'annotation_type', 'span_start', 'span_end', 'PAD_snippet', 'neg_flag']
result_data_frame = (pd.DataFrame(output, columns=columns))

result_data_frame.describe()

result_data_frame
result_data_frame.to_csv('out_table.csv', index=False)
print('Done')<jupyter_output>Done
{'fraction_non_alphanumeric': 0.12477991273061319, 'fraction_numerical': 0.0461609125009569, 'mean_word_length': 2.620842572062084, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '600156', 'license': 'Apache-2.0', 'n_tokens_mistral': 4871, 'n_tokens_neox': 4337, 'n_words': 965, 'path': 'pe_Simple_Classification_System.ipynb', 'repo_name': 'asy1113/NLP_PE', 'size': '13063'}