<jupyter_start><jupyter_text>Load the model weights for the fine-tuned resnet<jupyter_code># Install the OpenSlide C library and Python bindings
# After installing these libraries, use `Runtime -> restart and run all` on the menu
!apt-get install openslide-tools
!pip install openslide-python
from openslide import open_slide, __library_version__ as openslide_version
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import os
import shutil
import tensorflow as tf
import random
import glob
import cv2
import math
import re
import pathlib 
import time

from scipy.ndimage import rotate
from skimage.color import rgb2gray
from matplotlib import cm
# Note: x,y coords are with respect to level 0.
# Function to read a region from slide and return a numpy RBG array
def read_slide(slide, x, y, level, width, height, as_float=False):
    im = slide.read_region((x,y), level, (width, height))
    im = im.convert('RGB') # drop the alpha channel
    if as_float:
        im = np.asarray(im, dtype=np.float32)
    else:
        im = np.asarray(im)
    assert im.shape == (height, width, 3)
    return im

# Function to read a region from mask and return a single channel numpy array
def read_mask(slide, x, y, level, width, height, as_float=False):
  im = read_slide(slide, x, y, level, width, height, as_float=False)
  im = im[:,:,0]
  return im

# Function to identify tissue pixels (by filtering out grey areas)
def find_tissue_pixels(image, intensity=0.8):
    im_gray = rgb2gray(image)
    assert im_gray.shape == (image.shape[0], image.shape[1])
    indices = np.where(im_gray <= intensity)
    return list(zip(indices[0], indices[1]))
levels = [4,5]
patch_shape = (224,224)
BATCH_SIZE = 256
BUFFER_SIZE = 1000<jupyter_output><empty_output><jupyter_text>Define and load the fine-tuned model<jupyter_code>def import_resnet_tuned():
  model = tf.keras.applications.ResNet50V2(include_top=False, weights='imagenet')
  for layer in model.layers[:143]:
    layer.trainable = False
  return model
def define_stacked_model_tuned(members):
  for i in range(len(members)):
    model = members[i+1]
    for layer in model.layers:
      #layer.trainable = False
      layer._name = 'ensemble_' + str(i+1) + '_' + layer.name

  # define multi input
  stacked_inputs = [model.input for model in members.values()]

  # concatenate output from each model
  stacked_outputs = [model.output for model in members.values()]
  merge = tf.keras.layers.Concatenate()(stacked_outputs)

  # defining the hidden and output layer 
  hidden = tf.keras.layers.GlobalAveragePooling2D()(merge)
  output = tf.keras.layers.Dense(1, activation='sigmoid')(hidden)
  model = tf.keras.Model(inputs=stacked_inputs, outputs=output)

  return model
members = {}
for i in range(len(levels)):
  members[i+1] = import_resnet_tuned()

ensemble_model_tuned = define_stacked_model_tuned(members)
ensemble_model_tuned.compile(loss='binary_crossentropy', optimizer='adam', metrics=[tf.keras.metrics.Recall(), 'accuracy', tf.keras.metrics.AUC()])
from google.colab import drive
drive.mount("/content/gdrive", force_remount=True)
# !!!if files are saved locally then this step is not necessary
checkpoint_dir_ensemble_tuned = os.path.join('/content/gdrive/MyDrive/COMS4995_Project', 'checkpoints')
checkpoint_dir_ensemble_tuned = os.path.join(checkpoint_dir_ensemble_tuned, "final_project_ensemble_model_tuned")
# !!!this has to be changed to the actual path where the model weights are saved
latest = tf.train.latest_checkpoint(checkpoint_dir_ensemble_tuned)
if latest != None:
  print("Loading weights from", latest)
  ensemble_model_tuned.load_weights(latest)
else:
  print("Checkpoint not found. Starting from scratch")<jupyter_output>Loading weights from /content/gdrive/MyDrive/COMS4995_Project/checkpoints/final_project_ensemble_model_tuned/cp-00000009.ckpt<jupyter_text>Define and load the non-finetuned model<jupyter_code>def define_stacked_model(members):
  for i in range(len(members)):
    model = members[i+1]
    for layer in model.layers:
      layer.trainable = False
      layer._name = 'ensemble_' + str(i+1) + '_' + layer.name

  # define multi input
  stacked_inputs = [model.input for model in members.values()]

  # concatenate output from each model
  stacked_outputs = [model.output for model in members.values()]
  merge = tf.keras.layers.Concatenate()(stacked_outputs)

  # defining the hidden and output layer 
  hidden = tf.keras.layers.GlobalAveragePooling2D()(merge)
  output = tf.keras.layers.Dense(1, activation='sigmoid')(hidden)
  model = tf.keras.Model(inputs=stacked_inputs, outputs=output)

  return model
ensemble_model = define_stacked_model(members)
ensemble_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[tf.keras.metrics.Recall(), 'accuracy', tf.keras.metrics.AUC()])
checkpoint_dir = os.path.join('/content/gdrive/MyDrive/COMS4995_Project', 'checkpoints')
checkpoint_dir = os.path.join(checkpoint_dir, "final_project_ensemble_model")
latest = tf.train.latest_checkpoint(checkpoint_dir)
if latest != None:
  print("Loading weights from", latest)
  ensemble_model.load_weights(latest)
else:
  print("Checkpoint not found. Starting from scratch")<jupyter_output>Loading weights from /content/gdrive/MyDrive/COMS4995_Project/checkpoints/final_project/cp-00000004.ckpt<jupyter_text>Plotting the Heatmap  Pre-processing the slide we want to plot the heatmap on<jupyter_code>test_indexes = ['064', '001', '012']
#['064', '001', '012']
#We test our predictions with slide 064 first
raw_path = '/content/gdrive/MyDrive/COMS4995_Project/Project_Slides_Raw' 
# !!!this has to be changed to the actual path where the slide is
slide = '064'
levels = [4,5]
patch_shape = (224,224)
slide_path = raw_path + f'/tumor_{slide}.tif'
tumor_mask_path = raw_path + f'/tumor_{slide}_mask.tif'

test_slide = open_slide(slide_path)
test_mask = open_slide(tumor_mask_path)
test_image = read_slide(test_slide,
                                x=0,
                                y=0,
                                level=levels[0],
                                width=test_slide.level_dimensions[levels[0]][0],
                                height=test_slide.level_dimensions[levels[0]][1])

tumor_image = read_mask(test_mask,
                                x=0,
                                y=0,
                                level=levels[0],
                                width=test_mask.level_dimensions[levels[0]][0],
                                height=test_mask.level_dimensions[levels[0]][1])

#we have to use this resized plot to plot our heatmaps because having too large an image with too much range in its 
def resized_plot(slide_image, basewidth=1000): 
  from PIL import Image
  
  arr = np.copy(slide_image)
  img = Image.fromarray(arr)
  wpercent = (basewidth/float(img.size[0]))
  hsize = int((float(img.size[1])*float(wpercent)))
  img = img.resize((basewidth,hsize), Image.ANTIALIAS)
  a = np.asarray(img)
  return a
plt.figure(figsize=(10,10), dpi=100)
plt.imshow(resized_plot(test_image))
plt.imshow(resized_plot(tumor_image), cmap='jet', alpha=0.5) # Red regions contains cancer.
patches_dict = {}
for level in levels:
  patches_dict['level' + str(level)] = []
patches_dict['labels'] = []
for vert_slide in range(math.ceil(test_slide.level_dimensions[levels[0]][1]/patch_shape[1])):
  y_coord = int(vert_slide * test_slide.level_downsamples[levels[0]] * patch_shape[1])

  for horiz_slide in range(math.ceil(test_slide.level_dimensions[levels[0]][0]/patch_shape[0])):
    x_coord = int(horiz_slide * test_slide.level_downsamples[levels[0]] * patch_shape[0])
    
    level_name = 'level' + str(levels[0])

    slide_image = read_slide(test_slide,
                                x=x_coord,
                                y=y_coord,
                                level=levels[0],
                                width=patch_shape[0],
                                height=patch_shape[1])

    patches_dict[level_name].append(slide_image)

    
    mask_image = read_mask(test_mask,
                                  x=x_coord,
                                  y=y_coord,
                                  level=levels[0],
                                  width=patch_shape[0],
                                  height=patch_shape[1])
    
    tissue_pixels = find_tissue_pixels(slide_image)
    mask_tissue_image = np.array([mask_image[pixels] for pixels in tissue_pixels])
    
    if mask_tissue_image.any() == False:
      patches_dict['labels'].append(0)
    else:
      patches_dict['labels'].append(1)
    
    for i in range(len(levels)-1):
          level_name = 'level' + str(levels[i+1])
          slide_image_zoomout = read_slide(test_slide,
                                            x=x_coord,
                                            y=y_coord,
                                            level=levels[i+1],
                                            width=patch_shape[0],
                                            height=patch_shape[1])
          
          patches_dict[level_name].append(slide_image_zoomout)
dataset = tf.data.Dataset.from_tensor_slices((np.array(patches_dict['level4'])/255,np.array(patches_dict['level5'])/255))
dataset = dataset.batch(BATCH_SIZE)<jupyter_output><empty_output><jupyter_text>Running the Predictions on processed data:<jupyter_code>predictions = []
for x1,x2 in dataset:
    predictions.extend(ensemble_model_tuned.predict_on_batch(x=[x1,x2]).tolist())
vert_patches = math.ceil(test_slide.level_dimensions[levels[0]][1]/patch_shape[1])
hor_patches = math.ceil(test_slide.level_dimensions[levels[0]][0]/patch_shape[1])
predictions = np.array(predictions)
predictions.resize(vert_patches,hor_patches)
test_heatmap = np.copy(tumor_image)
test_heatmap = test_heatmap.astype('float')

for j in range(math.ceil(test_slide.level_dimensions[levels[0]][0]/patch_shape[0])):
  for i in range(math.ceil(test_slide.level_dimensions[levels[0]][1]/patch_shape[1])):
    y = i*patch_shape[0]
    x = j*patch_shape[1]
    width = patch_shape[0]
    height = patch_shape[1]
    test_heatmap[y:y+height,x:x+width] = predictions[i,j]
assert test_heatmap.shape == test_image[:,:,0].shape
test_heatmap_filtered = np.copy(test_heatmap)
zero_filter = test_heatmap < 0.6
test_heatmap_filtered[zero_filter] = 0
plt.figure(figsize=(10,10), dpi=100)
plt.imshow(resized_plot(test_image))
plt.imshow(resized_plot(test_heatmap_filtered), cmap='jet', alpha=0.5) # Red regions contains cancer.<jupyter_output><empty_output><jupyter_text>Non finetuned model heatmap<jupyter_code>predictions_untuned = []
for x1,x2 in dataset:
    predictions_untuned.extend(ensemble_model.predict_on_batch(x=[x1,x2]).tolist())
predictions_untuned = np.array(predictions_untuned)
predictions_untuned.resize(vert_patches,hor_patches)
test_heatmap = np.copy(tumor_image)
test_heatmap = test_heatmap.astype('float')

for j in range(math.ceil(test_slide.level_dimensions[levels[0]][0]/patch_shape[0])):
  for i in range(math.ceil(test_slide.level_dimensions[levels[0]][1]/patch_shape[1])):
    y = i*patch_shape[0]
    x = j*patch_shape[1]
    width = patch_shape[0]
    height = patch_shape[1]
    test_heatmap[y:y+height,x:x+width] = predictions_untuned[i,j]
assert test_heatmap.shape == test_image[:,:,0].shape
test_heatmap_filtered = np.copy(test_heatmap)
zero_filter = test_heatmap < 0.6
test_heatmap_filtered[zero_filter] = 0
plt.figure(figsize=(10,10), dpi=100)
plt.imshow(resized_plot(test_image))
plt.imshow(resized_plot(test_heatmap_filtered), cmap='jet', alpha=0.5) # Red regions contains cancer.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11304199676292699, 'fraction_numerical': 0.01984836868557799, 'mean_word_length': 3.607535321821036, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '392169', 'license': 'MIT', 'n_tokens_mistral': 4120, 'n_tokens_neox': 3866, 'n_words': 760, 'path': 'Plot_Heatmap.ipynb', 'repo_name': 'dkow72/coms4995_adl_proj', 'size': '11739'}