<jupyter_start><jupyter_text>Mammography Abnormality Detector<jupyter_code># This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session<jupyter_output>/kaggle/input/mias-mammography/Info.txt
/kaggle/input/mias-mammography/all_mias_scans.h5
/kaggle/input/mias-mammography/all-mias.tar.gz
/kaggle/input/mias-mammography/all-mias/mdb152.pgm
/kaggle/input/mias-mammography/all-mias/mdb088.pgm
/kaggle/input/mias-mammography/all-mias/mdb113.pgm
/kaggle/input/mias-mammography/all-mias/mdb182.pgm
/kaggle/input/mias-mammography/all-mias/mdb239.pgm
/kaggle/input/mias-mammography/all-mias/mdb066.pgm
/kaggle/input/mias-mammography/all-mias/mdb220.pgm
/kaggle/input/mias-mammography/all-mias/mdb292.pgm
/kaggle/input/mias-mammography/all-mias/mdb286.pgm
/kaggle/input/mias-mammography/all-mias/mdb103.pgm
/kaggle/input/mias-mammography/all-mias/mdb194.pgm
/kaggle/input/mias-mammography/all-mias/mdb275.pgm
/kaggle/input/mias-mammography/all-mias/mdb060.pgm
/kaggle/input/mias-mammography/all-mias/mdb067.pgm
/kaggle/input/mias-mammography/all-mias/mdb023.pgm
/kaggle/input/mias-mammography/all-mias/mdb073.pgm
/kaggle/input/mias-mammography/all-mias/mdb307.p[...]<jupyter_text>Let's see how the dataset is composed<jupyter_code>from PIL import Image
im = Image.open("../input/mias-mammography/all-mias/mdb001.pgm")
im<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import tensorflow as tf
import matplotlib.pyplot as plt
import os
from IPython.display import clear_output
from tqdm.notebook import tqdm
import cv2
from sklearn.model_selection import train_test_split
import random<jupyter_output><empty_output><jupyter_text>Data Load<jupyter_code>imgs_path = '../input/mias-mammography/all-mias/'<jupyter_output><empty_output><jupyter_text>Read Data Frame<jupyter_code>print("Reading Dataframe")
info=pd.read_csv("../input/mias-mammography/Info.txt",sep=" ")
info=info.drop('Unnamed: 7',axis=1)
info<jupyter_output><empty_output><jupyter_text>Let's save the Images ids<jupyter_code>ids = {}
for i in range(len(info)):
    ids[i] = info.REFNUM[i]<jupyter_output><empty_output><jupyter_text>Let's now create our labels for training<jupyter_code># Turning our outputs Normal-Abnormal to 0-1
label = []
for i in range(len(info)):
    if info.CLASS[i] != 'NORM':
        label.append(1)
    else:
        label.append(0)
label = np.array(label)
print(f"Cant. Imagenes: {len(label)}, Normales = {len(label)-np.sum(label)}, No normales= {np.sum(label)}")<jupyter_output>Cant. Imagenes: 330, Normales = 207, No normales= 123<jupyter_text>So we need to remove 84 normal breasts so the dataset is balanced  Importing our images<jupyter_code># define the every images filepaths in to list
img_name = []

for i in range(len(label)):
        img_name.append(imgs_path + info.REFNUM[i]+ '.pgm')<jupyter_output><empty_output><jupyter_text>Removing 84 normal breasts<jupyter_code>count = 0
remove = True
temp_label = []
temp_img_name = []

for i, lbl in enumerate(label.tolist()):
    if lbl == 0 and remove == True:
        count = count + 1
        if count >= 84:
            remove = False
    else:
        temp_label.append(lbl)
        temp_img_name.append(img_name[i])
label = np.array(temp_label)
img_name = temp_img_name
img_name = np.array(img_name)
img_name.shape
#print(img_name)
print(f'image addres amount {img_name.shape}')<jupyter_output>image addres amount (246,)<jupyter_text>Let's see 25 random images of our dataset<jupyter_code># view image random images
def view_25_random_image():
    fig = plt.figure(figsize = (15, 10))
    for i in range(25):
        rand = random.randint(0,len(label))
        ax = plt.subplot(5, 5, i+1)
    
        img = cv2.imread(img_name[rand], 0)
        img = cv2.resize(img, (256,256))
        if label[rand] == 1:
            plt.title('Not Notmal')
        else:
            plt.title('Normal')
        plt.tight_layout()
        plt.axis('off')
        plt.imshow(img)
    fig.savefig('random_25_image_fig.png')

random_images = view_25_random_image()<jupyter_output><empty_output><jupyter_text>Data AugmentationSince there are aproximately only 300 images, lets make some data augmentation by rotating the images<jupyter_code>img_path = []
last_label = []
IMG_SIZE = 128

for i in range(len(img_name)):
    
    img = cv2.imread(img_name[i], 0)
    img = cv2.resize(img, (IMG_SIZE,IMG_SIZE))
    rows, cols= img.shape
    for angle in range(180):
            M = cv2.getRotationMatrix2D((cols / 2, rows / 2), angle*2, 1)    #Rotate 0 degree
            img_rotated = cv2.warpAffine(img, M, (IMG_SIZE, IMG_SIZE))
            img_path.append(img_rotated)
            if label[i] == 1:
                last_label.append(1)
            else:
                last_label.append(0)
img_path = np.array(img_path)
last_label = np.array(last_label)
img_path.shape
img_path = img_path.reshape((-1,IMG_SIZE,IMG_SIZE,1))
img_path.shape
last_label.shape<jupyter_output><empty_output><jupyter_text>Split our data for training and testing<jupyter_code># split train and test set
x_train, x_test, y_train, y_test = train_test_split(img_path, last_label, test_size = 0.2, random_state = 42)
len(x_train),len(x_test),len(y_train),len(y_test)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from tensorflow.keras.layers import *
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.activations import *
from keras.callbacks import TensorBoard
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
from time import time
reg = tf.keras.regularizers.L2(l2=1e-5)
ini = tf.keras.initializers.RandomNormal(mean=0.0, stddev=0.02, seed=None)


model = Sequential()
model.add(Conv2D(32,
                 kernel_size=(3, 3),
                 activation='relu',
                 kernel_initializer = ini,
                 input_shape=(IMG_SIZE, IMG_SIZE, 1)))
model.add(Conv2D(64,
                 kernel_size=(3,3),
                 kernel_initializer = ini,
                 activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Conv2D(64, 
                 kernel_size=(3,3),
                 kernel_initializer = ini,
                 activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Dense(64, 
                kernel_initializer = ini,
                activation='relu'))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))

optimizer = tf.keras.optimizers.Adam(
    learning_rate=1e-3)

model.compile(optimizer=optimizer,
              loss= 'binary_crossentropy',
              metrics=['accuracy'])
model.summary()<jupyter_output>Model: "sequential_6"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_18 (Conv2D)           (None, 126, 126, 32)      320       
_________________________________________________________________
conv2d_19 (Conv2D)           (None, 124, 124, 64)      18496     
_________________________________________________________________
max_pooling2d_12 (MaxPooling (None, 62, 62, 64)        0         
_________________________________________________________________
conv2d_20 (Conv2D)           (None, 60, 60, 64)        36928     
_________________________________________________________________
max_pooling2d_13 (MaxPooling (None, 30, 30, 64)        0         
_________________________________________________________________
dropout_12 (Dropout)         (None, 30, 30, 64)        0         
______________________________________________________[...]<jupyter_text>Training<jupyter_code>early_stop = EarlyStopping(monitor='val_loss', mode='min', patience=3,restore_best_weights=True, verbose=1)

check_point_filepath = './'

model_check_point = ModelCheckpoint(filepath =check_point_filepath, monitor='val_loss', verbose=1, save_best_only=True,
                                    save_weights_only=False, mode='auto', save_freq='epoch')<jupyter_output><empty_output><jupyter_text>Let's train the Neural Network<jupyter_code>train = True
if train:
    epochs=100
    history = model.fit(x_train,
                 y_train,
                 validation_split=0.2,
                 epochs=epochs,
                 batch_size=128,
                 callbacks=[early_stop, model_check_point])
else:
    model = tf.keras.models.load_model('./')<jupyter_output>Epoch 1/100
222/222 [==============================] - 118s 528ms/step - loss: 0.6993 - accuracy: 0.5031 - val_loss: 0.6924 - val_accuracy: 0.5147

Epoch 00001: val_loss improved from inf to 0.69240, saving model to ./
Epoch 2/100
222/222 [==============================] - 117s 527ms/step - loss: 0.6915 - accuracy: 0.5170 - val_loss: 0.6920 - val_accuracy: 0.5033

Epoch 00002: val_loss improved from 0.69240 to 0.69196, saving model to ./
Epoch 3/100
222/222 [==============================] - 117s 527ms/step - loss: 0.6861 - accuracy: 0.5392 - val_loss: 0.6836 - val_accuracy: 0.5410

Epoch 00003: val_loss improved from 0.69196 to 0.68362, saving model to ./
Epoch 4/100
222/222 [==============================] - 117s 527ms/step - loss: 0.6765 - accuracy: 0.5592 - val_loss: 0.6850 - val_accuracy: 0.5760

Epoch 00004: val_loss did not improve from 0.68362
Epoch 5/100
222/222 [==============================] - 117s 527ms/step - loss: 0.6601 - accuracy: 0.5851 - val_loss: 0.6502 - val_accura[...]<jupyter_text>Let's See one result<jupyter_code>from random import randint
test_num = randint(0,len(x_test))
#test_num = 274
result = model.predict(np.array([x_test[test_num]]))

plt.figure(figsize= (10,10))
plt.imshow(x_test[test_num])

print(f"The results was: {result[0][0]>= 0.5}       And the probability being: {result[0][0]}\nReal value was: {y_test[test_num]>=0.5}")
print(f"Test id: {test_num}")
history.history.keys()<jupyter_output><empty_output><jupyter_text>Plots<jupyter_code>plt.figure(figsize=(10,5))
plt.plot(range(len(history.history['accuracy'])),history.history['accuracy'], label= "accuracy")
plt.plot(range(0,len(history.history['val_accuracy'])),history.history['val_accuracy'], label= "val_accuracy")
plt.grid()
plt.xlabel("Epoch")
plt.ylabel("Accuracy")
plt.title("Accuracy vs Epoch")
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>It is likely possible that the model is overfitting or the images aren't a good way to determinig if a breast has cancer or not.<jupyter_code>plt.figure(figsize=(10,5))
plt.plot(range(0,len(history.history['loss'])),history.history['loss'], label= "loss")
plt.plot(range(0,len(history.history['val_loss'])),history.history['val_loss'], label= "val_loss")
plt.grid()
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.title("Loss vs Epoch")
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Results on the hole Test Data<jupyter_code>test_loss, test_acc = model.evaluate(x_test,  y_test, verbose=2)

print('\nTest accuracy:', test_acc)<jupyter_output>277/277 - 28s - loss: 0.1166 - accuracy: 0.9601

Test accuracy: 0.9601399898529053<jupyter_text>Save the model<jupyter_code>len(x_test)
save_model = True
if save_model:
    save_model_path = "./"
    model.save(save_model_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1393693619134321, 'fraction_numerical': 0.04675991061822395, 'mean_word_length': 4.2768558951965066, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 78, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '447985', 'license': 'MIT', 'n_tokens_mistral': 4629, 'n_tokens_neox': 4193, 'n_words': 827, 'path': 'Mammography.ipynb', 'repo_name': 'iancraz/Mammography-Abnormality-Detector', 'size': '12083'}