<jupyter_start><jupyter_text>LOADING ALL LIBRARIES<jupyter_code>import os
import tensorflow as tf
import xml.etree.ElementTree
import numpy as np
import pandas as pd 
import re
import matplotlib.pyplot as plt
from IPython.display import Image
from PIL import Image
from numpy import array
import string
import os
from PIL import Image
import glob
import pickle
from pickle import dump, load
from time import time
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import LSTM, Embedding, TimeDistributed, Dense, RepeatVector,\
                         Activation, Flatten, Reshape, concatenate, Dropout, BatchNormalization
from keras.optimizers import Adam, RMSprop
from keras.layers.wrappers import Bidirectional
from keras.layers.merge import add
from keras.applications.inception_v3 import InceptionV3
from keras.preprocessing import image
from keras.models import Model
from keras import Input, layers
from keras import optimizers
from keras.applications.inception_v3 import preprocess_input
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from sklearn.model_selection import train_test_split
from tqdm import tqdm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm
import xml.etree.ElementTree as ET
import os
import cv2
import tensorflow as tf
from wordcloud import WordCloud
import re
from collections import defaultdict
import itertools
from collections import Counter<jupyter_output><empty_output><jupyter_text>GETTING DATA FROM XML FILES<jupyter_code>#DATA FOLDER
directory = r'ecgen-radiology'
# extracting data from the xml documents
#https://www.geeksforgeeks.org/xml-parsing-python/
image = []
impression = []
img_finding = []
for filename in tqdm(os.listdir(directory)):
    if filename.endswith(".xml"):
        f = directory + '/' + filename
        tree = ET.parse(f)
        root = tree.getroot()
        for child in root:
            if child.tag == 'MedlineCitation':
                for attr in child:
                    if attr.tag == 'Article':
                        for i in attr:
                            if i.tag == 'Abstract':
                                for name in i:
                                    if name.get('Label') == 'FINDINGS':
                                        finding=name.text
                                        
        for p_image in root.findall('parentImage'):
            
            image.append(p_image.get('id'))
            img_finding.append(finding)
#Storing the findings and image names in a data frame
Dataset = pd.DataFrame()
Dataset['Image_names'] = image
Dataset['Finding'] = img_finding
print(Dataset.shape)
Dataset.head()
def abs_path(x):
    '''Makes the path absolute '''
    x = r'x ray data/'+ x + '.png'
    return x
Dataset['Image_names'] = Dataset['Image_names'].apply(lambda x : abs_path(x)) # making the paths absolute
Dataset.head()
#saving in a pickle file
from pickle import dump
dump(Dataset,open('Dataset.pkl','wb'))
# loading the heights and widths of each image
h = []
w = []
for i in tqdm(np.unique(Dataset['Image_names'].values)):
    image = cv2.imread(i)
    h.append(image.shape[0])
    w.append(image.shape[1])
#plotting the heights and widths of images
plt.figure(figsize=(10,4))
plt.subplot(121)
plt.xlabel('Images')
plt.ylabel('Heights')
plt.title('Heights of Images')
sns.scatterplot(range(len(h)), h)
plt.subplot(122)
plt.xlabel('Images')
plt.ylabel('Widths')
plt.title('Width of Images')
sns.scatterplot(range(len(w)), h)<jupyter_output><empty_output><jupyter_text>The Heights and widths are not same for all the images  PREPROCESSING IMAGE DATA<jupyter_code>print('Total Images:', Dataset['Image_names'].nunique())

# checking for missing values
Dataset.isnull().sum()
# drop all missing value rows
Dataset = Dataset.dropna(axis=0) 
Dataset.isnull().sum()
print('Shape after preprocessing', Dataset.shape)
Dataset.head(10)
#printing images of two points
plt.figure(figsize=(14,7))
plt.subplot(121)
img1 = cv2.imread(Dataset['Image_names'].values[4])
plt.imshow(img1)
plt.title(Dataset['Image_names'].values[4])
plt.subplot(122)
img2 = cv2.imread(Dataset['Image_names'].values[5])
plt.title(Dataset['Image_names'].values[5])
plt.imshow(img2)

#printing findings of two points
Dataset['Finding'].values[4], Dataset['Finding'].values[5]<jupyter_output><empty_output><jupyter_text>Here we see that two reports are same  and the above images are also same but with a different angle.So we need to find images and reports like this.<jupyter_code># we are creating two dictionaries with image id and number of times it is present in dataset and findings of each report. 
images = {}
findings = {}
for image, find in Dataset.values:
    i = image.split('-')#splitting the image at '-'
    i.pop(len(i)-1)#removing the numbers after - to find the x rays of same patient
    i = '-'.join(e for e in i)
    if i not in images.keys():
        images[i] = 1
        findings[i] = find
    else:
        images[i] += 1
        findings[i] = find
images[r'x ray data/CXR1001_IM-0004'], findings[r'x ray data/CXR1001_IM-0004']
print('Total Number of unique images:', len(images.keys()))<jupyter_output>Total Number of unique images: 3350<jupyter_text>Here we can see that there are about 3350 unique images present in the dataset<jupyter_code>#plotting a bar plot to see how many images we have per each person
plt.figure(figsize=(17,8))
plt.bar(range(len(images.keys())), images.values())
plt.xlabel('Total number of Persons')
plt.ylabel('Images per Person')
one = 0
two = 0
three = 0
four = 0
for i in images.values():
    if i == 1:
        one +=1
    elif i == 2:
        two += 1
    elif i == 3:
        three += 1
    elif i == 4:
        four += 1
    else:
        print('Error')
print(one, two, three, four)<jupyter_output>390 2807 143 10<jupyter_text>Here We can see that majority of people have 2 images and 4 being maximum only 10 people have 4 images.<jupyter_code>len(images)
len(findings)<jupyter_output><empty_output><jupyter_text>SPLITTING THE DATA<jupyter_code>Data = list(images.keys())
Data_train = Data[:2500]
Data_cv = Data[2500:3000]
Data_test = Data[3000:3350]  
def combining_images(images):
    
    image_per_person = defaultdict(list)  # creating a list of dictionary to store all the image paths
                                            #corresponding to a person_id
    for pid in images:
        for img in Dataset['Image_names'].values:
            if pid in img:
                image_per_person[pid].append(img)
            else:
                continue
    return image_per_person
img_train = combining_images(Data_train)
img_cv = combining_images(Data_cv)
img_test = combining_images(Data_test)
len(img_train), len(Data_train)
img_train['x ray data/CXR1_1_IM-0001']
def load_image(file):
    img = tf.io.read_file(file)
    img = tf.image.decode_png(img, channels=3)
    img = tf.image.convert_image_dtype(img, tf.float32)
    return img
# checking which points have 4 images
for k,v in images.items():
    if v == 4:
        print(k)
        break
#plotting the four images of the point
plt.figure(figsize=(9,9))
plt.subplot(221)
plt.imshow(load_image(r'C:\Users\venka\OneDrive\Desktop\cs2\x ray data/CXR1102_IM-0069-12012.png'))
plt.title('CXR1_1_IM-0001')
plt.subplot(222)
plt.imshow(load_image(r'C:\Users\venka\OneDrive\Desktop\cs2\x ray data/CXR1102_IM-0069-2001.png'))
plt.title('CXR1102_IM-0069-2001.png')
plt.subplot(223)
plt.imshow(load_image(r'C:\Users\venka\OneDrive\Desktop\cs2\x ray data/CXR1102_IM-0069-3001.png'))
plt.title('CXR1102_IM-0069-3001.png')
plt.subplot(224)
plt.imshow(load_image(r'C:\Users\venka\OneDrive\Desktop\cs2\x ray data/CXR1102_IM-0069-4004.png'))
plt.title('CXR1102_IM-0069-4004.png')<jupyter_output><empty_output><jupyter_text>We have same scan in different angleslike front and sides.We have points with 1,2,3,4 scans per person in which majority have 2 scans.So we can take pair of Images as input to generate findings and The points with 1 scan can be augmented with same scan to have 2 scans for it.  CREATING FINAL DATA<jupyter_code>def final_data(image_per_person):
    # new dataset
    Person_id, image_1, image_2, Findings = [],[],[],[]
    for pid, imgs in image_per_person.items():   #contains pid and the images associated with that pid

        if len(imgs) == 1:
            image_1.append(imgs[0])
            image_2.append(imgs[0])
            Person_id.append(pid)
            Findings.append(findings[pid])
        else:
            num = 0
            a = itertools.combinations(imgs, 2)
            for i in a:
                image_1.append(i[0])
                image_2.append(i[1])
                Person_id.append(pid + '_' + str(num))
                Findings.append(findings[pid])
                num += 1
    data = pd.DataFrame()
    data['Person_id'] = Person_id
    data['Image-1'] = image_1
    data['Image-2'] = image_2
    data['Findings'] = Findings
    
    return data
Train_data = final_data(img_train)
Test_data = final_data(img_test)
CV_Data = final_data(img_cv)
print(Train_data.shape)
print(Test_data.shape)
print(CV_Data.shape)
Train_data.head()<jupyter_output><empty_output><jupyter_text>TEXT PREPROCESSING<jupyter_code>#https://www.kdnuggets.com/2018/03/text-data-preprocessing-walkthrough-python.html
def lowercase(text):
    '''Converts to lowercase'''
    new_text = []
    for line in text:
        new_text.append(line.lower())
    return new_text

def decontractions(text):
    '''Performs decontractions in the doc'''
    new_text = []
    for phrase in text:
        phrase = re.sub(r"won't", "will not", phrase)
        phrase = re.sub(r"can\'t", "can not", phrase)
        phrase = re.sub(r"couldn\'t", "could not", phrase)
        phrase = re.sub(r"shouldn\'t", "should not", phrase)
        phrase = re.sub(r"wouldn\'t", "would not", phrase)
        # general
        phrase = re.sub(r"n\'t", " not", phrase)
        phrase = re.sub(r"\'re", " are", phrase)
        phrase = re.sub(r"\'s", " is", phrase)
        phrase = re.sub(r"\'d", " would", phrase)
        phrase = re.sub(r"\'ll", " will", phrase)
        phrase = re.sub(r"\'t", " not", phrase)
        phrase = re.sub(r"\'ve", " have", phrase)
        phrase = re.sub(r"\'m", " am", phrase)
        phrase = re.sub(r"\*+", "abuse", phrase)
        new_text.append(phrase)

    return new_text

def rem_punctuations(text):
    '''Removes punctuations'''
    punctuations = '''!()-[]{};:'"\,<>/?@#$%^&*~''' # full stop is not removed
    new_text = []
    for line in text:
        for char in line:
            if char in punctuations: 
                line = line.replace(char, "")
        new_text.append(' '.join(e for e in line.split()))
    return new_text

def rem_numbers(text):
    '''Removes numbers and irrelevant text like xxxx*'''
    new_text = []
    for line in text:
        temp = re.sub(r'x*','',line)
        new_text.append(re.sub(r'\d','',temp))
    return new_text

def words_filter(text):
    '''Removes words less than 2 characters except no and ct'''
    new_text = []
    for line in text:
        temp = line.split()
        temp2 = []
        for word in temp:
            if  len(word) <=2 and word != 'no' and word != 'ct':
                continue
            else:
                temp2.append(word)
        new_text.append(' '.join(e for e in temp2))
    return new_text

def multiple_fullstops(text):
    ''' Removes multiple full stops from the text'''
    new_text = []
    for line in text:
        new_text.append(re.sub(r'\.\.+', '.', line))
    return new_text

def fullstops(text):
    new_text = []
    for line in text:
        new_text.append(re.sub('\.', ' .', line))
    return new_text

def multiple_spaces(text):
    new_text = []
    for line in text:
        new_text.append(' '.join(e for e in line.split()))
    return new_text

def separting_startg_words(text):
    new_text = []
    for line in text:
        temp = []
        words = line.split()
        for i in words:
            if i.startswith('.') == False:
                temp.append(i)
            else:
                w = i.replace('.','. ')
                temp.append(w)
        new_text.append(' '.join(e for e in temp))
    return new_text

def rem_apostrophes(text):
    new_text = []
    for line in text:
        new_text.append(re.sub("'",'',line))
    return new_text
def Preprocessing(text):
    '''Combining all the functions'''
    pre_text = lowercase(text)
    pre_text = decontractions(pre_text)
    pre_text = rem_punctuations(pre_text)
    pre_text = rem_numbers(pre_text)
    pre_text = words_filter(pre_text)
    pre_text = multiple_fullstops(pre_text)
    pre_text = fullstops(pre_text)
    pre_text = multiple_spaces(pre_text)
    pre_text = separting_startg_words(pre_text)
    pre_text = rem_apostrophes(pre_text)
    return pre_text
Train_data['Findings'] = Preprocessing(Train_data['Findings'])
Test_data['Findings'] = Preprocessing(Test_data['Findings'])
CV_Data['Findings'] = Preprocessing(CV_Data['Findings'])
Train_data.head()
#finding number of words in each report
words = [len(i.split()) for i in Train_data['Findings'].values] 
print(max(words))
#plotting scatter plot for number of words in each report
sns.scatterplot(range(Train_data.shape[0]), words)
plt.xlabel('Number of Reports')
plt.ylabel('Number of words')
plt.title('Words in each Report')<jupyter_output><empty_output><jupyter_text>Here we can see that majority of the reports contains 20 to 60 words.<jupyter_code>#finding number of words in train,cv,test data
train_words = []
for i in Train_data['Findings'].values:
    train_words.extend(i.split())
test_words = []
for i in Test_data['Findings'].values:
    test_words.extend(i.split())
cv_words = []
for i in CV_Data['Findings'].values:
    cv_words.extend(i.split())
print('total number of words in train data',len(train_words))
print('total number of words in test data',len(test_words))
print('total number of words in cv data',len(cv_words))
#finding unique words from total words
train_c= Counter(train_words)
test_c = Counter(test_words)
cv_c   = Counter(cv_words)

print('Number of Unique words in train data',len(train_c))
print('Number of Unique words in test data',len(test_c))
print('Number of Unique words in cv data',len(cv_c))
#printing 50 most common words from train data
print(train_c.most_common(50))
#printing most common words from test data
print(test_c.most_common(50))
#printing 50 most common words from cv data
print(cv_c.most_common(50))
#common words between train ,test and cv
common_train_cv = train_c & cv_c &test_c
print(len(common_train_cv))
#printing 50 most common words present in all train,test,cv
common_train_cv.most_common(50)<jupyter_output><empty_output><jupyter_text>Here we can see that there are about 87,000 words in train data most of the words are repeated and we see that there are only 1424 unique words in train data and 735,804 in test and validation data respectively.And also we can see that most common words in all these sets are almost same.<jupyter_code>common = train_c & cv_c
print(len(common))
print('{0} % of train words present in cv data'.format(len(common)/len(train_c)*100))
common_1 = train_c & test_c
print(len(common_1))
print('{0} % of train words present in cv data'.format(len(common_1)/len(train_c)*100))<jupyter_output>45.997191011235955 % of train words present in cv data<jupyter_text>Here we can see that only 50% of the words that are present in validation data are present in train data.And only about 46% of the words that are present in test data are present in train data.<jupyter_code>words=[]
for i in Train_data['Findings'].values:
    words.extend(i.split())
w = WordCloud(height=1200, width=1200).generate(str(words))
plt.figure(figsize=(10,10))
plt.title('WordCloud of Reports')
plt.imshow(w)
#function for adding start of sentence and end of sequence in reports
def add(x):
    return 'SOS' + ' ' + x + ' ' + 'EOS'
Train_data['Findings'] = Train_data['Findings'].apply(lambda x : add(x))
Test_data['Findings'] = Test_data['Findings'].apply(lambda x : add(x))
CV_Data['Findings'] = CV_Data['Findings'].apply(lambda x : add(x))
#dumping data to csv files
Train_data.to_csv('Train_Data.csv', index=False)
Test_data.to_csv('Test_Data.csv', index=False)
CV_Data.to_csv('CV_Data.csv', index=False)
frames = [Train_data,CV_Data,Test_data]
data = pd.concat(frames)
data[-5:]
#saving in a pickle file
from pickle import dump
dump(data,open('Dataset_1.pkl','wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11989657867837171, 'fraction_numerical': 0.021345679754675006, 'mean_word_length': 3.603376695267091, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 47, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '56261', 'license': 'MIT', 'n_tokens_mistral': 5743, 'n_tokens_neox': 5415, 'n_words': 1512, 'path': 'Final_EDA.ipynb', 'repo_name': 'MahendharReddy16/Generation-Of-Medical-Reports', 'size': '16631'}