<jupyter_start><jupyter_text>Loading the packages and libraries<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
# path='/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/metadata.csv'
# df=pd.read_csv(path)
# df=pd.read_csv('metadata.csv')
df.shape
df.finding.value_counts()
df.isnull().sum()
df.head()<jupyter_output><empty_output><jupyter_text>Explanatory Data Analysis<jupyter_code>plt.figure(figsize=(10,10))
sns.distplot(df['offset'],kde=True)
plt.show()
plt.figure(figsize=(10,10))
sns.countplot(df['sex'])
plt.show()
plt.figure(figsize=(10,10))
sns.distplot(df['age'],kde=True)
plt.show()
plt.figure(figsize=(10,10))
sns.countplot(df['labels'])
plt.show()
plt.figure(figsize=(10,10))
sns.countplot(df['survival'])
plt.show()
ax = df['finding'].value_counts(dropna=False).plot.pie(y='Finding', legend = True, autopct='%2.0f%%', figsize = (20,20),shadow=True, title = 'Finding Distribution',explode=(0,0.1,0.1,0.1,0.2,0.2,0.2,0.2,0.2,0.4,0.3))
out = pd.cut(df['age'], bins=np.arange(0,110,10).tolist(), include_lowest=True)
ax = out.value_counts(sort=False, dropna=False).plot.bar(rot=0, color="r", figsize=(15,6), title= "Age Distribution")
plt.show()
ax = df['survival'].value_counts(dropna=False).plot.pie(y='survival', legend = True, autopct='%2.0f%%', figsize = (5,5), title = 'Survival Distribution')
ax = df['view'].value_counts(dropna=False).plot.pie(y='view', legend = True, autopct='%2.0f%%', figsize = (8,8), title = 'View distribution')
ax = df['modality'].value_counts(dropna=False).plot.pie(y='modality', legend = True, autopct='%2.0f%%', figsize = (5,5), title = 'Modality Distribution')
df['country'] = df['location'].apply(lambda x: x.split(',')[-1].replace(" ","") if x is not np.nan else "Unknown")
ax = df['country'].value_counts(dropna=False).plot.pie(y='country', legend = True, autopct='%2.0f%%', figsize = (20,20), title = 'Location of the patient',explode=(0.1,0.1,0.1,0.1,0.1,0.2,0.2,0.2,0.3,0.3,0.3,0.4,0.4,0.4,0.4,0.5,0.5,0.5,0.5,0.5,1.0,1.0,1.2,1.3))
plt.figure(figsize=(10,10))
sns.countplot(df['intubated'])
plt.show()
plt.figure(figsize=(10,10))
sns.countplot(df['intubation_present'])
plt.show()
plt.figure(figsize=(10,10))
sns.countplot(df['went_icu'])
plt.show()
plt.figure(figsize=(10,10))
sns.countplot(df['needed_supplemental_O2'])
plt.show()
plt.figure(figsize=(10,10))
sns.countplot(df['extubated'])
plt.show()
plt.figure(figsize=(10,10))
sns.distplot(df['temperature'])
plt.show()
plt.figure(figsize=(10,10))
sns.distplot(df['pO2_saturation'])
plt.show()
plt.figure(figsize=(10,10))
sns.distplot(df['leukocyte_count'])
plt.show()
plt.figure(figsize=(10,10))
sns.distplot(df['lymphocyte_count'])
plt.show()
plt.figure(figsize=(10,10))
sns.distplot(df['neutrophil_count'])
plt.show()
df.dtypes
cor=df.corr()
plt.figure(figsize=(20,20))
sns.heatmap(cor,annot=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Important Note:Since we are dealing with Covid-19 classification only the columns related to images are necessary and others are dropped as per the requirements Data Prepreocessing<jupyter_code>df=df.drop(axis=1,columns='Unnamed: 29')
df=df.drop(axis=1,columns='other_notes')
df.drop(axis=1,columns=['doi','url','license','clinical_notes','date'],inplace=True)
df_new=df.drop(columns=['patientid','age','sex','offset','survival','intubated','intubation_present','went_icu','in_icu','needed_supplemental_O2','extubated','temperature','pO2_saturation','leukocyte_count','neutrophil_count','lymphocyte_count','location'],axis=1)
df_new= df_new[df_new['folder']!= 'volumes']
df_new.shape<jupyter_output><empty_output><jupyter_text>Since  PA, AP , AP Supine gives the clear view we drop the remaining features of those column<jupyter_code>df_new= df_new[df_new['view']!= 'L']
df_new= df_new[df_new['view']!= 'Axial']
df_new= df_new[df_new['view']!= 'Coronal']
df_new= df_new[df_new['view']!= 'AP semi erect']
df_new['view'].unique()<jupyter_output><empty_output><jupyter_text>Here we are dealing with X-ray images so we drop CT images<jupyter_code>df_new=df_new[df_new['modality']!='CT']
df_new['modality'].unique()
df_new.view=df_new.view.astype(str)
df_new.modality=df_new.modality.astype(str)
df_new.folder=df_new.folder.astype(str)
df_new.filename=df_new.filename.astype(str)
df_new.shape
cov_df=df_new[df_new['labels']==1]
cov_df.to_csv('/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/COVID_19_pos.csv',index=False)
cov_df.shape
not_cov_df=df_new[df_new['labels']==0]
not_cov_df.shape
not_cov_df.to_csv('/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/COVID_19_neg.csv',index=False)<jupyter_output><empty_output><jupyter_text>Splitting the images to Covid-19 and Non Covid-19 labels<jupyter_code>import pandas as pd
import shutil
import os

# Virus to look for
label = 1 


imageDir = "/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/images" # Directory of images
outputDir = "/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/COVID-19" # Output directory to store selected images
if not os.path.exists(outputDir):
  os.mkdir(outputDir)
metadata_csv = cov_df
for (i, row) in metadata_csv.iterrows():
  if row['labels']==label:
    filename = row["filename"].split(os.path.sep)[-1]
    filePath = os.path.sep.join([imageDir, filename])
    shutil.copy2(filePath, outputDir)
label = 0
outputDir_1 = "/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/datas/NON COVID-19"
if not os.path.exists(outputDir_1):
  os.mkdir(outputDir_1)
metadata_csv = not_cov_df
for (i, row) in metadata_csv.iterrows():
  if row['labels']==label:
    filename = row["filename"].split(os.path.sep)[-1]
    filePath = os.path.sep.join([imageDir, filename])
    shutil.copy2(filePath, outputDir_1)
import os
import numpy as np
import shutil
import random

# # Creating Train / Val / Test folders (One time use)
root_dir = '/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/datas'
classes_dir = ['/NON COVID-19','/COVID-19']

test_ratio = 0.2

for cls in classes_dir:
    os.makedirs(root_dir +'/train' + cls)
    os.makedirs(root_dir +'/test' + cls)


    # Creating partitions of the data after shuffeling
    src = root_dir + cls # Folder to copy images from

    allFileNames = os.listdir(src)
    np.random.shuffle(allFileNames)
    train_FileNames, test_FileNames = np.split(np.array(allFileNames),[int(len(allFileNames)* (1 -  test_ratio))])


    train_FileNames = [src+'/'+ name for name in train_FileNames.tolist()]
    test_FileNames = [src+'/' + name for name in test_FileNames.tolist()]

    print('Total images: ', len(allFileNames))
    print('Training: ', len(train_FileNames))
    print('Testing: ', len(test_FileNames))

    # Copy-pasting images
    for name in train_FileNames:
        shutil.copy2(name, root_dir +'/train' + cls)

    for name in test_FileNames:
        shutil.copy2(name, root_dir +'/test' + cls)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import keras
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications import VGG16,DenseNet201,ResNet50,VGG19,Xception,InceptionResNetV2
from tensorflow.keras.layers import AveragePooling2D,MaxPooling2D,GlobalAveragePooling2D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
import os
train_path='/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/IMG/train'
test_path='/content/drive/My Drive/covid-chestxray-dataset-master/covid-chestxray-dataset-master/IMG/test'<jupyter_output><empty_output><jupyter_text>With Validation data<jupyter_code>BS = 2
train_datagen=ImageDataGenerator(rescale=1./255,fill_mode='nearest',validation_split=0.1)
train_generator = train_datagen.flow_from_directory(directory=train_path,target_size=(224,224),class_mode="categorical",batch_size=BS,shuffle=True,seed=42,subset='training')
val_generator=train_datagen.flow_from_directory(directory=train_path,target_size=(224,224),class_mode="categorical",batch_size=BS,subset='validation',shuffle=True,seed=42)
test_batches=ImageDataGenerator(rescale=1./255,fill_mode='nearest').flow_from_directory(directory=test_path,target_size=(224,224),class_mode="categorical",batch_size=BS,seed=42,shuffle=False)
INIT_LR = 1e-5
EPOCHS = 30
BS = 2
k_fold=3
cv_scores, model_history = list(), list()
for i in range(k_fold):
  print("K-FOLD:",i+1)
  baseModel = VGG19(include_top=False,weights='imagenet',input_tensor=Input(shape=(224, 224, 3)))

 

  headModel = baseModel.output
  headModel=GlobalAveragePooling2D()(headModel)
  headModel=Dense(1024,activation='relu')(headModel)
  headModel = Dense(2, activation="softmax")(headModel)

  model = Model(inputs=baseModel.input, outputs=headModel)

  print("[INFO] compiling model...")
  opt = Adam(lr=INIT_LR)
  model.compile(loss="categorical_crossentropy", optimizer=opt,metrics=["accuracy"])


  print("[INFO] training head...")
  H = model.fit(x=train_generator,batch_size=BS,epochs=EPOCHS,verbose=2,validation_data=val_generator)
  predictions=model.predict(test_batches)
  predIdxs = np.argmax(predictions, axis=1)
  cm=confusion_matrix(y_true=test_batches.classes,y_pred=predIdxs)
  print(cm)
  print(classification_report(test_batches.classes,predIdxs))
  total = sum(sum(cm))
  acc = (cm[0, 0] + cm[1, 1]) / total
  sensitivity = cm[0, 0] / (cm[0, 0] + cm[0, 1])
  specificity = cm[1, 1] / (cm[1, 0] + cm[1, 1])
  # show the confusion matrix, accuracy, sensitivity, and specificity
  print(cm)
  print("acc: {:.4f}".format(acc))
  print("sensitivity: {:.4f}".format(sensitivity))
  print("specificity: {:.4f}".format(specificity))
  cv_scores.append(acc)
  model_history.append(H.history)

print('Estimated Accuracy %.3f (%.3f)' % (np.mean(cv_scores), np.std(cv_scores)))<jupyter_output>K-FOLD: 1
[INFO] compiling model...
[INFO] training head...
Epoch 1/30
110/110 - 12s - loss: 0.5257 - accuracy: 0.7854 - val_loss: 0.4550 - val_accuracy: 0.8261
Epoch 2/30
110/110 - 12s - loss: 0.4909 - accuracy: 0.8082 - val_loss: 0.4349 - val_accuracy: 0.8261
Epoch 3/30
110/110 - 12s - loss: 0.4003 - accuracy: 0.8082 - val_loss: 1.0331 - val_accuracy: 0.8261
Epoch 4/30
110/110 - 12s - loss: 0.5162 - accuracy: 0.8219 - val_loss: 0.4091 - val_accuracy: 0.8261
Epoch 5/30
110/110 - 12s - loss: 0.3650 - accuracy: 0.8311 - val_loss: 0.3679 - val_accuracy: 0.8261
Epoch 6/30
110/110 - 12s - loss: 0.3102 - accuracy: 0.8676 - val_loss: 0.4284 - val_accuracy: 0.8261
Epoch 7/30
110/110 - 12s - loss: 0.2392 - accuracy: 0.8995 - val_loss: 0.4069 - val_accuracy: 0.8696
Epoch 8/30
110/110 - 12s - loss: 0.1182 - accuracy: 0.9543 - val_loss: 0.5798 - val_accuracy: 0.7826
Epoch 9/30
110/110 - 12s - loss: 0.0800 - accuracy: 0.9680 - val_loss: 0.6543 - val_accuracy: 0.8696
Epoch 10/30
110/110 - 12s - los[...]<jupyter_text>Without Validation data<jupyter_code>BS = 2
train_datagen=ImageDataGenerator(rescale=1./255,fill_mode='nearest')
train_generator = train_datagen.flow_from_directory(directory=train_path,target_size=(224,224),class_mode="categorical",batch_size=BS,shuffle=True,seed=42)
# val_generator=train_datagen.flow_from_directory(directory=train_path,target_size=(224,224),class_mode="categorical",batch_size=BS,subset='validation',shuffle=True,seed=42)
test_batches=ImageDataGenerator(rescale=1./255,fill_mode='nearest').flow_from_directory(directory=test_path,target_size=(224,224),class_mode="categorical",batch_size=BS,seed=42,shuffle=False)
INIT_LR = 1e-5
EPOCHS = 30
BS = 2
k_fold=3
cv_scores, model_history = list(), list()
for i in range(k_fold):
  print("K-FOLD:",i+1)
  baseModel = VGG19(include_top=False,weights='imagenet',input_tensor=Input(shape=(224, 224, 3)))

 

  headModel = baseModel.output
  headModel=GlobalAveragePooling2D()(headModel)
  headModel=Dense(1024,activation='relu')(headModel)
  headModel = Dense(2, activation="softmax")(headModel)

  model = Model(inputs=baseModel.input, outputs=headModel)

  print("[INFO] compiling model...")
  opt = Adam(lr=INIT_LR)
  model.compile(loss="categorical_crossentropy", optimizer=opt,metrics=["accuracy"])


  print("[INFO] training head...")
  H = model.fit(x=train_generator,batch_size=BS,epochs=EPOCHS,verbose=2,validation_data=val_generator)
  predictions=model.predict(test_batches)
  predIdxs = np.argmax(predictions, axis=1)
  cm=confusion_matrix(y_true=test_batches.classes,y_pred=predIdxs)
  print(cm)
  print(classification_report(test_batches.classes,predIdxs))
  total = sum(sum(cm))
  acc = (cm[0, 0] + cm[1, 1]) / total
  sensitivity = cm[0, 0] / (cm[0, 0] + cm[0, 1])
  specificity = cm[1, 1] / (cm[1, 0] + cm[1, 1])
  # show the confusion matrix, accuracy, sensitivity, and specificity
  print(cm)
  print("acc: {:.4f}".format(acc))
  print("sensitivity: {:.4f}".format(sensitivity))
  print("specificity: {:.4f}".format(specificity))
  cv_scores.append(acc)
  model_history.append(H.history)

print('Estimated Accuracy %.3f (%.3f)' % (np.mean(cv_scores), np.std(cv_scores)))<jupyter_output>K-FOLD: 1
[INFO] compiling model...
[INFO] training head...
Epoch 1/30
121/121 - 13s - loss: 0.5172 - accuracy: 0.8099 - val_loss: 0.4245 - val_accuracy: 0.8261
Epoch 2/30
121/121 - 13s - loss: 0.4706 - accuracy: 0.8099 - val_loss: 0.4952 - val_accuracy: 0.8261
Epoch 3/30
121/121 - 13s - loss: 0.4661 - accuracy: 0.8099 - val_loss: 0.3788 - val_accuracy: 0.8261
Epoch 4/30
121/121 - 13s - loss: 0.4172 - accuracy: 0.8223 - val_loss: 0.2961 - val_accuracy: 0.8261
Epoch 5/30
121/121 - 13s - loss: 0.3672 - accuracy: 0.8264 - val_loss: 0.1880 - val_accuracy: 0.9130
Epoch 6/30
121/121 - 13s - loss: 0.2752 - accuracy: 0.8678 - val_loss: 0.0871 - val_accuracy: 1.0000
Epoch 7/30
121/121 - 13s - loss: 0.2522 - accuracy: 0.9050 - val_loss: 0.1620 - val_accuracy: 0.9130
Epoch 8/30
121/121 - 13s - loss: 0.1574 - accuracy: 0.9339 - val_loss: 0.0610 - val_accuracy: 1.0000
Epoch 9/30
121/121 - 13s - loss: 0.1404 - accuracy: 0.9504 - val_loss: 0.0662 - val_accuracy: 0.9565
Epoch 10/30
121/121 - 13s - los[...]
{'fraction_non_alphanumeric': 0.16243306329809146, 'fraction_numerical': 0.06645613071536455, 'mean_word_length': 6.071359223300971, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '208816', 'license': 'MIT', 'n_tokens_mistral': 6255, 'n_tokens_neox': 5597, 'n_words': 943, 'path': 'Files/Covid_19_classifier_lung_diseases.ipynb', 'repo_name': 'KARTHIKEYANJM/Automatic-Covid-19-Classification-and-Segmentation', 'size': '14566'}