<jupyter_start><jupyter_text>Case 2 - Diabetic Retinopathy Analysis<NAME>Last edited: 8.3.2018  1. ObjectivesThe goal for this assignment was to build and train a 2D-convolutional network able to classify fundus images between healthy and affected with diabetic retinopathy. Additionally, the goal is to use advanced metrics to observe the performance of the model in various ways.  2. Required imports<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

import keras
from keras import layers
from keras import models
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator

from sklearn.metrics import accuracy_score, precision_score, f1_score, confusion_matrix
from sklearn.metrics import classification_report, recall_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
from sklearn.metrics import roc_curve, roc_auc_score
#We'll check the matplotlib version, as there were errors using version 2.1.0
import matplotlib
matplotlib.__version__<jupyter_output><empty_output><jupyter_text>3. Data description and preprocessing<jupyter_code>#<NAME> 19.2.2018:
def showsamples(img1, img2):
    plt.figure()
    imgplot = plt.imshow(img1)
    plt.figure()
    imgplot = plt.imshow(img2)
    plt.show()
    return;
#Plot some example images from the training set
img1=mpimg.imread("dataset2\\train\\nosymptoms\\1324_left.jpeg")
img2=mpimg.imread("dataset2\\train\\symptoms\\1050_left.jpeg")
showsamples(img1,img2)

#These image flags are changed to fight an error caused by the images being loaded as read-only
img1.flags.writeable = True
img2.flags.writeable = True<jupyter_output><empty_output><jupyter_text>The preprocessing function created for this assignment follows the code by <NAME> & <NAME>, Feb 15, 2018. In the function, the red channel is zeroed out while the green channel is exaggarated. This function was designed by following the principles in this article http://www.opsweb.org/?page=monochromatic by <NAME>.<jupyter_code>def image_preprocessing_func(x):
    r = x[:, :, 0]
    g = x[:, :, 1]
    b = x[:, :, 2]
    
    g = (g - np.mean(g))/(1.0*np.std(g));
    b = (b - np.mean(b))/(1.0*np.std(b));
  
    x[:, :, 0] = r*0
    x[:, :, 1] = g*60+20
    x[:, :, 2] = b
    
    return x
#Preprocessed images
img1=mpimg.imread("dataset2\\train\\nosymptoms\\1324_left.jpeg")
img2=mpimg.imread("dataset2\\train\\symptoms\\1050_left.jpeg")
img1.flags.writeable = True
img2.flags.writeable = True
img1 = image_preprocessing_func(img1)
img2 = image_preprocessing_func(img2)

showsamples(img1,img2)
train_dir = ".\\dataset2\\train"
validation_dir = ".\\dataset2\\validation"
test_dir = ".\\dataset2\\test"


train_datagen = ImageDataGenerator(rescale=1./255, preprocessing_function = image_preprocessing_func)
validation_datagen = ImageDataGenerator(rescale=1./255, preprocessing_function = image_preprocessing_func)
test_datagen = ImageDataGenerator(rescale=1./255, preprocessing_function = image_preprocessing_func)

train_generator = train_datagen.flow_from_directory(train_dir, 
        target_size=(150, 150),
        batch_size=20,
        class_mode='binary')

validation_generator = validation_datagen.flow_from_directory(
        validation_dir,
        target_size=(150, 150),
        batch_size=20,
        class_mode='binary')

test_generator = test_datagen.flow_from_directory(
        test_dir,
        target_size=(150, 150),
        batch_size=20,
        class_mode='binary')

for data_batch, labels_batch in train_generator:
    print('data batch shape:', data_batch.shape)
    print('labels batch shape:', labels_batch.shape)
    break<jupyter_output>Found 1237 images belonging to 2 classes.
Found 413 images belonging to 2 classes.
Found 413 images belonging to 2 classes.
data batch shape: (20, 150, 150, 3)
labels batch shape: (20,)<jupyter_text>4. Modeling and compilation<jupyter_code>#Build the model
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(150, 150, 3)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))

model.add(layers.Flatten())
model.add(layers.Dense(512, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))

model.summary()
#Compile the model
model.compile(loss='binary_crossentropy',
              optimizer=optimizers.Adam(),
              metrics=['acc'])<jupyter_output><empty_output><jupyter_text>5. Training and validation<jupyter_code>history = model.fit_generator(train_generator,
                              steps_per_epoch=62, #samples/batch size if there's no augmented data (62)
                              epochs=10,
                              validation_data=validation_generator,
                              validation_steps=21)
#Save the model for later use
model.save('final.h10')<jupyter_output><empty_output><jupyter_text>6. Evaluation<jupyter_code>#Save the required metrics into variables
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(len(acc))

#Plot the accuracies
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

#Plot the losses
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()
#Calculate the final result
result = model.evaluate_generator(test_generator, steps = 21)
# Loss and accuracy
print("Final result:\n Accuracy:", result[1], "Loss:", result[0])<jupyter_output>Final result:
 Accuracy: 0.782082323012 Loss: 0.415350646574<jupyter_text>Several models were tried out while solving the problem. The best models all reached validation accuracies varying between 75 and 80 percent. With deeper, more complicating models the execution time quickly increases even though the results aren't improving. Adding more layers actually made the results drop even earlier, so a simple model was determined to be the best one.At least these methods were tried while trying to improve the results:* Changing the amount of epochs (5-50)* Changing the batch size* Decreasing the amount of validation steps (with and without data augmentation)* Adding more convolutional layer* Adding more dense layers* Altering filter dimensions* Data augmentation* Trying out several preprocessing functionsMost of these modifications only made the results worse.The sole things that helped were:1. Finding a suitable preprocessing function through research2.  7. Results<jupyter_code>y_true = np.zeros(413)
y_score = np.zeros(413)
sample_count = 413
i = 0
for inputs_batch, labels_batch in test_generator:
        predicts_batch = model.predict(inputs_batch)
        L = labels_batch.shape[0]
        index = range(i, i + L)
        y_true[index] = labels_batch.ravel()
        y_score[index] = predicts_batch.ravel()
        i = i + L
        if i >= sample_count:
            break
fpr, tpr, thresholds = roc_curve(y_true, y_score)
auc = roc_auc_score(y_true, y_score)

plt.figure()
plt.plot(fpr, tpr)
plt.plot([0, 1], [0, 1], '--')
plt.grid()
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC curve   AUC = {:.3f}'.format(auc))
plt.show()

plt.figure()
plt.plot(thresholds, 1-fpr, label = 'specificity')
plt.plot(thresholds, tpr, label = 'sensitivity')
plt.legend()
plt.grid()
plt.xlabel('Threshold value')
plt.show()
# Select the threshold to maximize both specificity and sensitivity
th = 0.37

# Select the threshold to maxim
acc = accuracy_score(y_true, y_score > th)
prec = precision_score(y_true, y_score > th)
f1 = f1_score(y_true, y_score > th)
recall = recall_score(y_true, y_score > th)

print('Accuracy:  {:.3f}'.format(acc))
print('Precision: {:.3f}'.format(prec))
print('Recall:    {:.3f}'.format(recall))
print('F1:        {:.3f}'.format(f1))
print('Classification report')
print(classification_report(y_true, y_score > th, labels = [1.0, 0.0], target_names = ['Disease', 'Healthy']))
tn, fp, fn, tp = confusion_matrix(y_true, y_score > th).ravel()

print('                      Confusion matrix\n')
print('                       True condition')
print('                      ----------------')
print('                      Positive Negative      Sum')
print('Predicted | Positive  {:8} {:8} {:8}'.format(tp, fp, tp + fp))
print('condition | Negative  {:8} {:8} {:8}'.format(fn, tn, fn + tn))
print('                 Sum  {:8} {:8} {:8}'.format(tp + fn, fp + tn, tp + fp + fn + tn))
print(' ')
print('Sensitivity: {:.3f}'.format(tp/(tp+fn)))
print('Specificity: {:.3f}'.format(tn/(tn+fp)))<jupyter_output>Confusion matrix

                       True condition
                      ----------------
                      Positive Negative      Sum
Predicted | Positive        95       68      163
condition | Negative        24      226      250
                 Sum       119      294      413
 
Sensitivity: 0.798
Specificity: 0.769
{'fraction_non_alphanumeric': 0.12230752825762423, 'fraction_numerical': 0.03401578161654937, 'mean_word_length': 4.0779642663779105, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '387718', 'license': 'MIT', 'n_tokens_mistral': 3226, 'n_tokens_neox': 2943, 'n_words': 811, 'path': 'Case+2+-+Diabetic+Retinopathy+Analysis.ipynb', 'repo_name': 'SaanaVallius/cognitive-systems-for-health-technology-applications', 'size': '9418'}