<jupyter_start><jupyter_text>Kaggle Steel Defects - Segmentation (locate and identify defects with masks)Link to competition: https://www.kaggle.com/c/severstal-steel-defect-detectionThis notebook was converted from my prior Kaggle notebook.  Migrated to TF 2.x and converted various methods to be more native TF.  - Pre-trained model is from <NAME>, (https://github.com/qubvel/segmentation_models)  Final Classification Report and metrics from Training images:Epoch 00010: loss: 0.1354 - dice_coef: 0.7605 - val_loss: 0.1564 - val_dice_coef: 0.7244 - lr: 5.0000e-05Total:  1416   Good class:  1053.0   Bad class:  363.0   class percent good:  0.7436440677966102Final dice_coef score:  0.5803109915075524Accuracy : 0.7436440677966102Classification Report              precision    recall  f1-score   support         4.0       0.96      0.99      0.98       907         5.0       0.00      0.00      0.00        29         6.0       0.00      0.00      0.00         2         8.0       0.85      0.53      0.65        95        12.0       0.62      0.88      0.73       101        16.0       0.00      0.00      0.00         0        17.0       0.00      0.00      0.00       168        18.0       0.00      0.00      0.00        34        19.0       0.00      0.00      0.00        16        20.0       0.23      0.38      0.29        45        21.0       0.00      0.00      0.00        14        24.0       0.00      0.00      0.00         5        28.0       0.00      0.00      0.00         0    accuracy                           0.74      1416       macro avg       0.20      0.21      0.20      1416   weighted avg       0.73      0.74      0.73      1416<jupyter_code>#"""
# Google Collab specific stuff....
from google.colab import drive
drive.mount('/content/drive')

import os
!ls "/content/drive/My Drive"

USING_COLLAB = True
%tensorflow_version 2.x
#"""
# To start, install kaggle libs
#!pip install -q kaggle

# Workaround to install the newest version
# https://stackoverflow.com/questions/58643979/google-colaboratory-use-kaggle-server-version-1-5-6-client-version-1-5-4-fai
!pip install kaggle --upgrade --force-reinstall --no-deps
# Upload your "kaggle.json" file that you created from your Kaggle Account tab
# If you downloaded it, it would be in your "Downloads" directory

from google.colab import files
files.upload()
# On your VM, create kaggle directory and modify access rights

!mkdir -p ~/.kaggle
!cp kaggle.json ~/.kaggle/
!ls ~/.kaggle
!chmod 600 /root/.kaggle/kaggle.json
#!kaggle competitions list
!kaggle competitions download -c severstal-steel-defect-detection
!unzip -uq severstal-steel-defect-detection.zip 
!ls train_images/a75bb4c01*.*
# Cleanup to add some space....
!rm -r test_images
!rm severstal-steel-defect-detection.zip
# Setup sys.path to find MachineLearning lib directory

try: USING_COLLAB
except NameError: USING_COLLAB = False

%load_ext autoreload
%autoreload 2

import sys
if "MachineLearning" in sys.path[0]:
    pass
else:
    print(sys.path)
    if USING_COLLAB:
        sys.path.insert(0, '/content/drive/My Drive/GitHub/MachineLearning/lib')  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    else:
        sys.path.insert(0, '/Users/john/Documents/GitHub/MachineLearning/lib')  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    
    print(sys.path)
#%reload_ext autoreload

from __future__ import absolute_import, division, print_function, unicode_literals

import os, sys, random, warnings, time, copy, csv, gc
import numpy as np 

import matplotlib.pyplot as plt
%matplotlib inline

import cv2
from tqdm import tqdm_notebook, tnrange, tqdm
import pandas as pd

import tensorflow as tf
print(tf.__version__)

from tensorflow.keras.models import load_model 

from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle

AUTOTUNE = tf.data.experimental.AUTOTUNE
print("AUTOTUNE: ", AUTOTUNE)

from TrainingUtils import *
from losses_and_metrics.Losses_Babakhin import make_loss, Kaggle_IoU_Precision, dice_coef_loss_bce

#warnings.filterwarnings("ignore", category=DeprecationWarning)
#warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)<jupyter_output><empty_output><jupyter_text>Examine and understand data<jupyter_code># GLOBALS/CONFIG ITEMS

# Set root directory path to data
if USING_COLLAB:
    #ROOT_PATH = "/content/drive/My Drive/ImageData/KaggleSteelDefects"  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    ROOT_PATH = ""
    MODEL_PATH= "/content/drive/My Drive/ImageData/KaggleSteelDefects"  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    
else:
    ROOT_PATH = "/Users/john/Documents/ImageData/KaggleSteelDefects"  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    MODEL_PATH= "/Users/john/Documents/ImageData/KaggleSteelDefects"  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    
# Establish global dictionary
parms = GlobalParms(MODEL_NAME="model-SteelDefects-Segmentation-V01.h5",
                    ROOT_PATH=ROOT_PATH,
                    #TRAIN_DIR="train_images",
                    TRAIN_PATH="train_images", 
                    MODEL_PATH=MODEL_PATH,
                    SMALL_RUN=False,
                    NUM_CLASSES=4,
                    CLASS_NAMES=["1", "2", "3", "4"],
                    IMAGE_ROWS=256,
                    IMAGE_COLS=800,
                    IMAGE_CHANNELS=3,
                    BATCH_SIZE=8,
                    EPOCS=20,
                    IMAGE_EXT=".jpg",
                    FINAL_ACTIVATION='sigmoid',
                    LOSS=tf.keras.losses.BinaryCrossentropy(from_logits=True))

# Other globals...
ORIG_MASK_SHAPE = (256, 1600)
STARTING_MODEL_PATH = "/content/drive/My Drive/GitHub/MachineLearning/2-Kaggle-Steel-Defect/segmodel-256-800-c4-V01.h5"

parms.print_contents()
# Simple helper method to display batches of images with labels....  

def show_image_masks(image_in, masks_in):
    if tf.is_tensor(image_in):
        image = image_in.numpy()
    else:
        image = image_in

    if tf.is_tensor(masks_in): 
        masks = masks_in.numpy()
    else:
        masks = masks_in

    #print(image.shape, masks.shape)

    # cv2.polylines and cv2.findContours display better when range is 0-255
    # https://docs.opencv.org/2.4/modules/core/doc/drawing_functions.html
    image = image * 255
    palet = [(249, 192, 12), (0, 185, 241), (114, 0, 218), (249,50,12)]
    title = "Labels: "
    fig, ax = plt.subplots(1,1,figsize=(20,10))
      
    for j in range(parms.NUM_CLASSES):
        msk = np.ascontiguousarray(masks[:, :, j], dtype=np.uint8)
        if np.count_nonzero(msk) > 0:
            title = title + str(j+1) + ",  "
            contours, _ = cv2.findContours(msk, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
            for i in range(0, len(contours)):
                cv2.polylines(image, contours[i], True, palet[j], 2) 

    title = title[:-3]  
    ax.set_title(title)

    #ax.imshow(tf.keras.preprocessing.image.array_to_img(image), cmap=plt.get_cmap('gray'))
    #print(image.shape, image.dtype, np.max(image), np.min(image))
    ax.imshow(image/255, cmap=plt.get_cmap('gray'))


def show_batch_image_masks(image, masks):
    for i in range(len(image)):
        show_image_masks(image[i], masks[i])


# Helper methods to create mask's or rle's
def mask2rle(img):
    '''
    img: numpy array, 1 - mask, 0 - background
    Returns run length as string formated
    '''
    pixels= img.T.flatten()
    pixels = np.concatenate([[0], pixels, [0]])
    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
    runs[1::2] -= runs[::2]
    return ' '.join(str(x) for x in runs)

def rle2mask(rle, input_shape):
    width, height = input_shape[:2]
    
    mask= np.zeros( width*height ).astype(np.uint8)
    
    array = np.asarray([int(x) for x in rle.split()])
    starts = array[0::2]
    lengths = array[1::2]

    current_position = 0
    for index, start in enumerate(starts):
        mask[int(start):int(start+lengths[index])] = 1
        current_position += lengths[index]
        
    return mask.reshape(height, width).T

def build_masks(rles, input_shape):
    depth = len(rles)
    masks = np.zeros((*input_shape, depth))
    
    for i, rle in enumerate(rles):
        if type(rle) is str:
            masks[:, :, i] = rle2mask(rle, input_shape)
    
    return masks

def build_rles(masks):
    width, height, depth = masks.shape
    
    rles = [mask2rle(masks[:, :, i])
            for i in range(depth)]
    
    return rles

# Load train DEFECT csv
image_defect_df = pd.read_csv(os.path.join(parms.ROOT_PATH, "train.csv"))

# Load image file sizes for possible stratification usage
image_defect_df['ImageSize'] = image_defect_df['ImageId'].map(lambda image_id: round(os.stat(os.path.join(parms.TRAIN_PATH, image_id)).st_size))
#image_defect_df['ImageSize'] = 50

print(image_defect_df.loc[image_defect_df["ImageId"] == "0025bde0c.jpg"])
image_defect_df.head()
# Stratifing by image_size, my prior notebook used the number of white pixels, this was easier and gave a better spread
image_defect_df_cut = pd.cut(image_defect_df["ImageSize"], bins=[0, 85000, 104000, 115000, 1000000]) 
ax = image_defect_df_cut.value_counts(sort=False).plot.bar(rot=0, color="b", figsize=(20,6)) 
plt.show() 
# Apply method to create the group number
def group_by_image_size(x):
    #[0, 85000, 104000, 115000, 1000000])
    if x < 85000:
        return 0
    elif x < 104000:
        return 1
    elif x < 115000:
        return 2
    else:
        return 3

image_defect_df['ImageGroup'] = image_defect_df['ImageSize'].apply(group_by_image_size)
image_defect_df.head()

# Select a balanced subset for training
SAMPLES_PER_GROUP = 200000
balanced_train_df = image_defect_df.groupby('ImageGroup').apply(lambda x: x.sample(SAMPLES_PER_GROUP) if len(x) > SAMPLES_PER_GROUP else x)
balanced_train_df['ImageGroup'].hist(bins=balanced_train_df['ImageGroup'].max()+1)
print(balanced_train_df.shape[0], 'ImageGroup')<jupyter_output><empty_output><jupyter_text>Build an input pipeline<jupyter_code># Split train and val, stratify by number of targets

train_df, valid_df = train_test_split(balanced_train_df, 
                                      test_size = 0.2,
                                      stratify = balanced_train_df['ImageGroup'])

# Add some more training examples from the sparse examples
#print('Original Training len: ', train_df.shape[0], "  Validation len: ", valid_df.shape[0])
#add_more_df = train_df.loc[train_df["DefectCount"] > 1]
#add_more_df = pd.concat([add_more_df, add_more_df])
#train_df = pd.concat([train_df, add_more_df])
#train_df.reset_index(drop=True)

train_df = shuffle(train_df) # Shuffle

print('After Adjust, Training len: ', train_df.shape[0], "  Validation len: ", valid_df.shape[0])
# set lengths and steps
train_len = len(train_df)
val_len = len(valid_df)
images_list_len = train_len + val_len

steps_per_epoch = np.ceil(train_len // parms.BATCH_SIZE) # set step sizes based on train & batch
validation_steps = np.ceil(val_len // parms.BATCH_SIZE) # set step sizes based on val & batch

print("Total number: ", images_list_len, "  Train number: ", train_len, "  Val number: ", val_len)
print("Steps/EPOC: ", steps_per_epoch, "  Steps/Validation: ", validation_steps)
# Final look at the distribution since we added more of the sparse cases
print(train_df["ImageGroup"].value_counts())
print(valid_df["ImageGroup"].value_counts())<jupyter_output><empty_output><jupyter_text>Training and Validation setup<jupyter_code># Read, decode the image, convert to float
def read_decode_image(image_id: tf.Tensor) -> tf.Tensor:
    file_path = parms.TRAIN_PATH + "/" + image_id
    image = tf.io.read_file(file_path)
    image = tf.image.decode_jpeg(image, channels=parms.IMAGE_CHANNELS)
    image = tf.image.convert_image_dtype(image, parms.IMAGE_DTYPE)
    return image

# Build mask(s) from rles
def load_masks(image_id_in: tf.Tensor) -> tf.Tensor:
    image_id = image_id_in.numpy().decode("utf-8")
    image_df = image_defect_df.loc[image_defect_df['ImageId'] == image_id]
    #print("df ", image_id, image_df)

    rles = [None] * parms.NUM_CLASSES # Create blank list
    for i, image_row in image_df.iterrows():
        indx = int(image_row["ClassId"]) - 1
        #print("row ", indx, image_row)
        rles[indx] = image_row["EncodedPixels"] # Fill in any encoded masks
        
    masks = build_masks(rles, input_shape=ORIG_MASK_SHAPE)

    return masks

# Augmentations for training dataset, done after cache
def image_aug(image: tf.Tensor, masks: tf.Tensor) -> tf.Tensor:
    # Must use custom precent, random.uniform, because both image and mask must match
    
    if tf.random.uniform(()) > 0.5:
        image = tf.image.flip_left_right(image)
        masks = tf.image.flip_left_right(masks)
        
    if tf.random.uniform(()) > 0.5:
        image = tf.image.flip_up_down(image)
        masks = tf.image.flip_up_down(masks)
    
    return image, masks

# pre-cache mapped method to load image and masks
def process_load_image_masks(image_id: tf.Tensor) -> tf.Tensor:
    image = read_decode_image(image_id)  

    [masks,] = tf.py_function(load_masks, [image_id], [tf.int32])  #parms must be tensors
    masks.set_shape((*ORIG_MASK_SHAPE, parms.NUM_CLASSES))
    
    image = tf.image.resize(image, [parms.IMAGE_ROWS, parms.IMAGE_COLS])
    masks = tf.image.resize(masks, [parms.IMAGE_ROWS, parms.IMAGE_COLS])

    return image, masks

# post-cache mapped method, does image augmentation
def process_train_post_cache(image: tf.Tensor, masks: tf.Tensor) -> tf.Tensor:
    image, masks = image_aug(image, masks)
    return image, masks

# Create Dataset from pf
train_dataset = tf.data.Dataset.from_tensor_slices(train_df["ImageId"].values)
                                               
# Verify image and label were loaded
for image_id in train_dataset.take(2):
    train_image_id = image_id.numpy().decode("utf-8")
    print("Image ID: ", image_id.numpy().decode("utf-8"))

# map training images to processing, includes any augmentation
train_dataset = train_dataset.map(process_load_image_masks, num_parallel_calls=AUTOTUNE)

# Verify the mapping worked
for image, masks in train_dataset.take(1):
    print("Image shape: {}  Max: {}  Min: {}".format(image.numpy().shape, np.max(image.numpy()), np.min(image.numpy())))
    print("Masks shape: {}  Max: {}  Min: {}".format(masks.numpy().shape, np.max(masks.numpy()), np.min(masks.numpy())))
    some_image = image.numpy()
    some_masks = masks.numpy()

# Remove cache if running under Kaggle
#train_dataset = train_dataset \
train_dataset = train_dataset.cache("./steel_train_seg.tfcache") \
                             .map(process_train_post_cache, num_parallel_calls=AUTOTUNE) \
                             .batch(parms.BATCH_SIZE) \
                             .prefetch(1) \
                             .repeat()

# Uncomment to show the batch of images, execute this cell multiple times to see the images
#for batch_image, batch_masks in train_dataset.take(1):
#    show_batch_image_masks(batch_image, batch_masks)

show_batch_image_masks([some_image], [some_masks])
# Double check that training labels and image_id are all good, can use different image_id's
image_defect_df.loc[image_defect_df["ImageId"] == train_image_id]
# Create Dataset from pd
val_dataset = tf.data.Dataset.from_tensor_slices(valid_df["ImageId"].values)


# Verify image and label were loaded
for image_id in val_dataset.take(2):
    val_image_id = image_id.numpy().decode("utf-8")
    print("Image ID: ", image_id.numpy().decode("utf-8"))

    # map training images to processing, includes any augmentation
val_dataset = val_dataset.map(process_load_image_masks, num_parallel_calls=AUTOTUNE)

# Verify the mapping worked
for image, masks in val_dataset.take(1):
    print("Image shape: {}  Max: {}  Min: {}".format(image.numpy().shape, np.max(image.numpy()), np.min(image.numpy())))
    print("Masks shape: {}  Max: {}  Min: {}".format(masks.numpy().shape, np.max(masks.numpy()), np.min(masks.numpy())))
    some_image = image
    some_masks = masks

# Remove cache if running under Kaggle
#val_dataset = val_dataset.cache("./steel_val_seg.tfcache2") \
val_dataset = val_dataset \
                         .batch(parms.BATCH_SIZE) \
                         .prefetch(1) \
                         .repeat()

# Double check that val labels and image_id are all good, can use different image_id's
image_defect_df.loc[image_defect_df["ImageId"] == val_image_id]
# Final check before model training.  Test Validation or Train by changing the dataset

#for batch_image, batch_masks in train_dataset.take(1):
for batch_image, batch_masks in val_dataset.take(1):  
    show_batch_image_masks(batch_image, batch_masks)
    
#show_batch_image_masks([some_image], [some_masks])<jupyter_output><empty_output><jupyter_text>Build  model- add and validate pretrained model as a baseline<jupyter_code># Create any call backs for training...These are the most common.

from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, CSVLogger

reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=3, verbose=1, min_lr=1e-6)
earlystopper = EarlyStopping(patience=6, verbose=1)
checkpointer = ModelCheckpoint(parms.MODEL_PATH, monitor='val_dice_coef', verbose=1, mode="max", save_best_only=True)

# Create model and compile it

from tensorflow.keras.models import Sequential, load_model, Model
from tensorflow.keras.layers import Dense, Dropout, Flatten, Input, Conv2D, MaxPooling2D, BatchNormalization, UpSampling2D, Conv2DTranspose, Concatenate, Activation
from tensorflow.keras.losses import binary_crossentropy, categorical_crossentropy
from tensorflow.keras.optimizers import Adadelta, Adam, Nadam, SGD
########
K = tf.keras.backend


loss_function = "bce_dice"  # bce_dice, lovasz
loss = make_loss(loss_function)    

def dice_coef_np(y_true, y_pred, smooth=1):
    y_true_f = np.ndarray.flatten(y_true)
    y_pred_f = np.ndarray.flatten(y_pred)
    intersection = np.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (np.sum(y_true_f) + np.sum(y_pred_f) + smooth)

def dice_coef(y_true, y_pred, smooth=1):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)

def compile_model(parms, model):
    model.compile(
        loss= loss,
        optimizer=Adam(lr=0.00005),  #
        metrics=[dice_coef])    
    return model

#def compile_model(parms, model):
#    model.compile(optimizer=tf.keras.optimizers.RMSprop(learning_rate=0.0001),
#                  loss= make_loss(loss_function),
#                  metrics=[lb_metric])
#    return model<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code># Reload the model from prior runs
#model = load_model(parms.MODEL_PATH, custom_objects={'loss': loss, 'dice_coef': dice_coef})

# train from empty seg model, comment out if loading existing model
model = load_model(STARTING_MODEL_PATH)
print("Loaded: ", STARTING_MODEL_PATH)

model = compile_model(parms, model)
#!ls
#!rm steel_train_seg.tfcache_0.lockfile
#!ls
# Train model

history = model.fit(train_dataset,
                    validation_data=val_dataset,
                    epochs=parms.EPOCS, 
                    steps_per_epoch=steps_per_epoch,
                    validation_steps=validation_steps,
                    callbacks=[reduce_lr, earlystopper, checkpointer] 
                    )

# Plot the training history
history_df = pd.DataFrame(history.history)
plt.figure()
history_df[['loss', 'val_loss']].plot(title="Loss")
plt.xlabel('Epocs')
plt.ylabel('Loss')
history_df[['dice_coef', 'val_dice_coef']].plot(title="Accuracy")
plt.xlabel('Epocs')
plt.ylabel('Accuracy')
plt.show()<jupyter_output><empty_output><jupyter_text>Validate model's predictions- Create actual_lables and predict_labels- Calculate Confusion Matrix & Accuracy- Display results<jupyter_code>#Load saved model
   
model = load_model(parms.MODEL_PATH, custom_objects={'loss': loss, 'dice_coef': dice_coef})
print("loaded: ", parms.MODEL_PATH)
# Show a batch of images with predictions and scores....
for batch_image, batch_masks in train_dataset.take(1):
    score_total = 0
    pred_masks = model.predict(batch_image)
    for z in range(parms.BATCH_SIZE):
        show_image_masks(batch_image[z], batch_masks[z])
        pred_masks_img = np.where(pred_masks[z] > 0.5, 1, 0)
        if np.count_nonzero(pred_masks_img) == 0:
            print("none predicted found, adjusting np.where....")
            pred_masks_img = np.where(pred_masks_img > 0.20, 1, 0)     
        show_image_masks(batch_image[z], pred_masks_img)
        score = dice_coef_np(batch_masks[z].numpy(), pred_masks_img)
        score_total = score_total + score
        print(z, "  Score: ", score)

    print("Final batch score: ", score_total / parms.BATCH_SIZE)

def predictions_using_dataset_masks(model_actual,
                              dataset,
                              steps,
                              batch_size,
                              create_bad_results_list=False,
                              score_min=0.10,
                              predict_one_min=0.5):
    """
      Uses generator to predict results.  Builds actual_labels, predict_labels
      and predict_probabilities

      Args:
        model_actual : trained model to use for predictions
        ds_iter : dataset iterator
        steps : number of batches to process
        create_bad_results_list : bool default True.  Lets you trun on/off
            the creation of the bad results lists.

      Returns:
        actual_labels : list of actual labels
        predict_labels : list of predicted labels
        predict_probabilities : list of predicted probability array
        bad_results : list of bad results [actual_labels, predict_labels,
                      predict_probabilities, image]
    """

    bad_cnt = 0.0
    good_cnt = 0.0
    total_cnt = 0
    actual_labels = []
    predict_labels = []
    predict_probabilities = []
    bad_results = []
    score_total = 0

    for image_batch, label_batch in tqdm(dataset.take(steps)):
        for j in range(batch_size):
            actual_label = []
            pred_label = []
            image = image_batch[j]
            label = label_batch[j].numpy()
            #print("label ", label.shape, np.max(label), np.min(label))

            total_cnt += 1
   
            image = np.expand_dims(image, axis=0)
            # image = tf.reshape(image, (1, *image.shape))

            predict_probabilities_tmp = model_actual.predict(image)[0]

            pred_masks = np.where(predict_probabilities_tmp > 0.5, 1, 0)
            if np.count_nonzero(pred_masks) == 0:
                #print("none predicted found, adjusting np.where....")
                pred_masks = np.where(predict_probabilities_tmp > 0.20, 1, 0) 

            #show_image_masks(image[0], label)
            #show_image_masks(image[0], pred_masks)

            score = dice_coef_np(label, pred_masks)
            score_total = score_total + score
            #print(j, "  Score: ", score)
            #print("predict ", np.max(predict_probabilities_tmp[:,:,0]),
            #                  np.max(predict_probabilities_tmp[:,:,1]),
            #                  np.max(predict_probabilities_tmp[:,:,2]),
            #                  np.max(predict_probabilities_tmp[:,:,3]))

            predict_label = np.zeros(parms.NUM_CLASSES + 1)
            actual_label =  np.zeros(parms.NUM_CLASSES + 1)
            for z in range(parms.NUM_CLASSES):
                if np.max(predict_probabilities_tmp[:, :, z]) > predict_one_min:
                    predict_label[z] = 1

                if np.max(label[:, :, z]) == 1:
                    actual_label[z] = 1
            
            if score < score_min:
                predict_label[4] = 1
                actual_label[4] = 1

            correct_flag = np.array_equal(actual_label, predict_label)

            np_tmp = np.array([1,2,4,8,16])
            actual_label_amt = np.sum(actual_label * np_tmp)
            predict_label_amt = np.sum(predict_label * np_tmp)

            actual_labels.append(actual_label_amt)
            predict_labels.append(predict_label_amt)
            predict_probabilities.append(predict_probabilities_tmp)
            #print(correct_flag, "  Label ", actual_label, "  Pred Label ", predict_label)

            if correct_flag:
                good_cnt = good_cnt + 1
            else:
                bad_cnt = bad_cnt + 1

            if score < score_min: # or could use correct_flag
                if create_bad_results_list:
                    bad_results.append([[actual_label],
                                        [predict_label],
                                        predict_probabilities_tmp,
                                        score,
                                        image])
    print(" ")
    print("Total: ", total_cnt, "  Good class: ", good_cnt, "  Bad class: ",
          bad_cnt, "  class percent good: ", str(good_cnt/total_cnt))
    print("Final dice_coef score: ", str(score_total / total_cnt))

    return actual_labels, predict_labels, predict_probabilities, \
        bad_results

# Use model to generate predicted labels and probabilities

#create_bad_results_list=True
labels, predict_labels, predict_probabilities, bad_results = predictions_using_dataset_masks(model, val_dataset, validation_steps, parms.BATCH_SIZE)
#labels, predict_labels, predict_probabilities, bad_results = predictions_using_dataset_masks(model, val_dataset, 1, parms.BATCH_SIZE)

def show_confusion_matrix2(labels,
                          predict_labels,
                          show_graph=True):
    """
      Shows various accuracry measurements.

      Args:
        labels : actual labels
        predict_labels : predicted labels
        class_names : list of class names
        show_graph : flag to show or not show the actual graph.  set
                     to False for large number of classes.
      Returns:
        nothing
    """

    # Accuracy score
    print("Accuracy : " + str(accuracy_score(labels, predict_labels)))

    print("")

    # Classification report
    print("Classification Report")
    print(classification_report(np.array(labels),
                                np.array(predict_labels)))
    if show_graph:
        # Plot confusion matrix
        cnf_matrix = confusion_matrix(labels, predict_labels)
        print(cnf_matrix)
        plot_confusion_matrix(cnf_matrix, [])

show_confusion_matrix2(labels, predict_labels)
# defect 1->0,  2->1, 4->2, 8->3,  3->0+1, 16->no label & bad score  Larger than 16-> bad score, subtract 16 to get labels
#Create a df from the bad results list, can save as csv or use for further analysis
bad_results_df = pd.DataFrame(bad_results, columns =['actual', 'predict', 'prob', 'image'])
bad_results_df.head()
bad_act, bad_pred, bad_prob, bad_images = zip(*bad_results)

# display images....        
def show_bad_batch(image_batch, bad_act, bad_pred, number_to_show=25):
    plt.figure(figsize=(10,10))
    show_number = number_to_show
    if len(image_batch) < number_to_show:
        show_number = len(image_batch)
      
    for n in range(show_number):
        ax = plt.subplot(5,5,n+1)
        plt.imshow(tf.keras.preprocessing.image.array_to_img(np.squeeze(image_batch[n])))
        #s = parms.CLASS_NAMES[bad_pred[n][0]]
        s = "Act: "+ str(bad_act[n][0]) + " Pred: " + str(bad_pred[n][0])
        plt.title(s)
        plt.axis('off')

show_bad_batch(bad_images, bad_act, bad_pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11025613236516021, 'fraction_numerical': 0.024999096853437375, 'mean_word_length': 2.929869392390687, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 52, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '378309', 'license': 'MIT', 'n_tokens_mistral': 9647, 'n_tokens_neox': 9117, 'n_words': 2014, 'path': '2-Kaggle-Steel-Defect/SteelDefects_Segmentation_V1.ipynb', 'repo_name': 'bo9zbo9z/MachineLearning', 'size': '27692'}