<jupyter_start><jupyter_text>Using Deep Learning to Quantify Fibrosis in Muscle Tissue Model Selection By <NAME>  Edit Here<jupyter_code>run = 'aug18ep150'
test_file = '/home/sergei/projects/def-rogertam/sergei/micro_scans/data/traintest/diaphragmsmarinefascia4/validation_finalxxx.npz'
training_data_directory = '/home/sergei/projects/def-rogertam/sergei/micro_scans/data/traintest/diaphragmsmarinefascia4/train_finalxxx.npz'
model_run = f'/home/sergei/projects/def-rogertam/sergei/micro_scans/models/{run}/'
epochs = 150
print(f"Welcome to the evaluation for the run conducted on {run}! I hope you are happy with the results!")
inv_data_gen_args = dict()
# , samplewise_center=True
# load image data

# load and prepare training images
from numpy import load
def load_real_samples(filename):
    # load compressed arrays
    data = load(filename)
    # unpack arrays
    X1, X2 = data['arr_0'], data['arr_1']
    # scale from [0,255] to [-1,1]
    X1 = (X1 - 127.5) / 127.5
    X2 = (X2 - 127.5) / 127.5
    return [X1, X2]

seed = 12



print()
print()
print()
print()<jupyter_output><empty_output><jupyter_text>End of Edit Here<jupyter_code>from keras.models import load_model
from numpy import load
from numpy import vstack
from matplotlib import pyplot
from numpy.random import randint
import cv2  
import random
import matplotlib.pyplot as plt
import numpy as np  
import scipy.stats as stats
from tqdm import tqdm
import sys
sys.path.insert(1, '/project/def-rogertam/sergei/micro_scans/code_files/product')
import threshold_fx
from keras.preprocessing.image import ImageDataGenerator
corrs = []
abso = []
real = []
invariant_datagen = ImageDataGenerator(**inv_data_gen_args)

dataset = load_real_samples(training_data_directory)
print('Loaded', dataset[0].shape, dataset[1].shape)
trainA, trainB = dataset
invariant_datagen.fit(trainA, augment=True, seed=seed)

data = load(test_file)
[A1, A2] = data['arr_0'], data['arr_1'] 
X1 = (A1 - 127.5) / 127.5
X2 = (A2 - 127.5) / 127.5
print('Loaded a test set of shape', X1.shape)
for x in tqdm(range(0, 10)):
    l1 = []
    l2 = []
    model = load_model(model_run + 'model_00' + str(x) + '000.h5', compile=False)
    #print(f'using model {x}\nThe test shape is of size {X1.shape}.')
    for l in range(X1.shape[0] // 2):
        #print('*' * 30)
        #print(f'Starting {l}')
        src_image, tar_image = X1[[l]], X2[[l]]
        invariant_generator = invariant_datagen.flow(
        src_image,
        seed=seed)
        invariant_generator.next()
        
        
        invariant =  invariant_generator[0].copy()
 
        src_image = invariant        
        
        gen_image = model.predict(src_image)
        gen = (gen_image + abs(gen_image.min())) / 2.0

        white, black, thresh1 = threshold_fx.threshold_fx(A1, l, 0)
        if white != 0:
            gen_calc, threshgen = threshold_fx.threshold_gen(gen, white)
            true_calc, threshtar = threshold_fx.threshold_gen(tar_image, white)
        else:
            gen_calc = 0
            true_calc = 0
            threshgen = np.zeros((thresh1.shape))
            threshtar = np.zeros((thresh1.shape))
        
      #  srcs = (cv2.cvtColor(src_image.squeeze(), cv2.COLOR_BGR2RGB))
      #  num = random.randint(1, 100)
        #print(f"num is {num}")
       # if num > 90:
      #      srcs = srcs + 1.0 / 2.0
      #      print(srcs.max(), srcs.min())
       #     plt.imshow(srcs)
       #     plt.title('Source')
       #     plt.show()
        #    plt.imshow(thresh1, cmap='gray')
       #     plt.title(f'Threshold={white}')
      #      plt.show()
      #      plt.imshow(threshgen, cmap='gray')
       #     plt.title(f'Generated={gen_calc}')
       #     plt.show()
       #     plt.imshow(threshtar, cmap='gray')
       #     plt.title(f'Ground Truth={true_calc}')
        #   plt.show()
        


        
        l1.append(gen_calc)
        l2.append(true_calc )

        
    #print('$' * 1000)
    diff = 0
    for i in range(len(l1)):
        diff += abs(l1[i] - l2[i])
    diffr = 0
    for i in range(len(l1)):
        diffr += (l1[i] - l2[i])
    correlation, p_value = stats.pearsonr(l1, l2)
    corrs.append(correlation)
    abso.append(diff)
    real.append(diffr)
    #print(f'Diff = {diff}')
    #print(f'Diffr = {diffr}')
   # print(f'Correlation = {correlation}')
print('Done the first 10!')
for x in tqdm(range(10, 100)):
    l1 = []
    l2 = []
    model = load_model(model_run + 'model_0' + str(x) + '000.h5', compile=False)
   # print(f'using model {x}\nThe test shape is of size {X1.shape}.')

    for l in range(X1.shape[0] // 2):
        #print('*' * 30)
        #print(f'Starting {l}')
        src_image, tar_image = X1[[l]], X2[[l]]
        invariant_generator = invariant_datagen.flow(
        src_image,
        seed=seed)
        invariant_generator.next()
        
        
        invariant =  invariant_generator[0].copy()
 
        src_image = invariant      
        
        
        
        
        gen_image = model.predict(src_image)
        gen = (gen_image + abs(gen_image.min())) / 2.0

        white, black, thresh1 = threshold_fx.threshold_fx(A1, l, 0)
        if white != 0:
            gen_calc, threshgen = threshold_fx.threshold_gen(gen, white)
            true_calc, threshtar = threshold_fx.threshold_gen(tar_image, white)
        else:
            gen_calc = 0
            true_calc = 0
            threshgen = np.zeros((thresh1.shape))
            threshtar = np.zeros((thresh1.shape))
        
        srcs = (cv2.cvtColor(src_image.squeeze(), cv2.COLOR_BGR2RGB))
        num = random.randint(1, 100)
        #print(f"num is {num}")
      #  if num > 90:
      #      srcs = srcs + 1.0 / 2.0
      #      print(srcs.max(), srcs.min())
       #     plt.imshow(srcs)
       #     plt.title('Source')
      #      plt.show()
     #       plt.imshow(thresh1, cmap='gray')
      #      plt.title(f'Threshold={white}')
       #     plt.show()
      #      plt.imshow(threshgen, cmap='gray')
     #       plt.title(f'Generated={gen_calc}')
       #     plt.show()
      #      plt.imshow(threshtar, cmap='gray')
       #     plt.title(f'Ground Truth={true_calc}')
       #     plt.show()
        


        
        l1.append(gen_calc)
        l2.append(true_calc )

        
   # print('$' * 1000)
    diff = 0
    for i in range(len(l1)):
        diff += abs(l1[i] - l2[i])
    diffr = 0
    for i in range(len(l1)):
        diffr += (l1[i] - l2[i])
    correlation, p_value = stats.pearsonr(l1, l2)
    corrs.append(correlation)
    abso.append(diff)
    real.append(diffr)
    #print(f'Diff = {diff}')
    #print(f'Diffr = {diffr}')
   # print(f'Correlation = {correlation}')
print('Thanks for waiting!')
for x in tqdm(range(100, 228)):
    l1 = []
    l2 = []
    model = load_model(model_run + 'model_' + str(x) + '000.h5', compile=False)
   # print(f'using model {x}\nThe test shape is of size {X1.shape}.')

    for l in range(X1.shape[0] // 2):
        #print('*' * 30)
        #print(f'Starting {l}')
        src_image, tar_image = X1[[l]], X2[[l]]
        invariant_generator = invariant_datagen.flow(
        src_image,
        seed=seed)
        invariant_generator.next()
        
        
        invariant =  invariant_generator[0].copy()
 
        src_image = invariant      
        
        
        
        
        
        
        gen_image = model.predict(src_image)
        gen = (gen_image + abs(gen_image.min())) / 2.0

        white, black, thresh1 = threshold_fx.threshold_fx(A1, l, 0)
        if white != 0:
            gen_calc, threshgen = threshold_fx.threshold_gen(gen, white)
            true_calc, threshtar = threshold_fx.threshold_gen(tar_image, white)
        else:
            gen_calc = 0
            true_calc = 0
            threshgen = np.zeros((thresh1.shape))
            threshtar = np.zeros((thresh1.shape))
        
        srcs = (cv2.cvtColor(src_image.squeeze(), cv2.COLOR_BGR2RGB))
        num = random.randint(1, 100)
        #print(f"num is {num}")
      #  if num > 90:
      #      srcs = srcs + 1.0 / 2.0
      #      print(srcs.max(), srcs.min())
       #     plt.imshow(srcs)
       #     plt.title('Source')
      #      plt.show()
     #       plt.imshow(thresh1, cmap='gray')
      #      plt.title(f'Threshold={white}')
       #     plt.show()
      #      plt.imshow(threshgen, cmap='gray')
     #       plt.title(f'Generated={gen_calc}')
       #     plt.show()
      #      plt.imshow(threshtar, cmap='gray')
       #     plt.title(f'Ground Truth={true_calc}')
       #     plt.show()
        


        
        l1.append(gen_calc)
        l2.append(true_calc )

        
   # print('$' * 1000)
    diff = 0
    for i in range(len(l1)):
        diff += abs(l1[i] - l2[i])
    diffr = 0
    for i in range(len(l1)):
        diffr += (l1[i] - l2[i])
    correlation, p_value = stats.pearsonr(l1, l2)

    corrs.append(correlation)
    abso.append(diff)
    real.append(diffr)
    #print(f'Diff = {diff}')
    #print(f'Diffr = {diffr}')
   # print(f'Correlation = {correlation}')

print("Smile more!")
########################################################################################################################
print('@' * 50)
print("Here is what the results are:")
print(f'The original corrs is {corrs}. The correlation of l1 and l2 for the nth weight file is in the nth position of this run.')
cor = sorted(corrs)
print(f'We sort it to get {cor}. This is the sorted version. Check out the end to see the best correlation!')
positionscor = []
for x in range(len(cor)):
    positionscor.append(corrs.index(cor[x]))
print(f'The following list shows which model occupies which spot in the sorted list of correlations. The end of the list is the best model! positionscor is {positionscor}.')



print(f'Original abso is {abso}')
ab = sorted(abso)
print(f'We sort it to get {ab}')
positionsabs = []
for x in range(len(ab)):
    positionsabs.append(abso.index(ab[x]))
positionsabs.reverse()
print(f'positionsabs is {positionsabs}')



print('$' * 50)
finals = list()
number_models = len(ab)
for k in range(len(ab)):
    corspot = (positionscor.index(k) + 1)/ number_models
    try:
        absspot = (positionsabs.index(k) + 1)/ number_models
        metric = 0.5 * corspot + 0.5 * absspot 
        finals.append(metric)
        print(f'the model is {k}, it holds position {positionscor.index(k)} in positionscor / {number_models} = {corspot}, and position {positionsabs.index(k)} in positionsabs / {number_models} = {absspot}. We weigh this as {0.5 * corspot} + {0.5 * absspot} = {metric}')

    except:
        continue
    print(f'Finals is {finals}')

sorted_finals = sorted(finals)
print(f"Sorted finals is {sorted_finals}")
print()
print()
print()
print()
print()
print(f'The best model is {finals.index(sorted_finals[-1])}, followed by {finals.index(sorted_finals[-2])}.')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15001839587932303, 'fraction_numerical': 0.025294334069168506, 'mean_word_length': 2.4682615629984053, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '591973', 'license': 'Apache-2.0', 'n_tokens_mistral': 4035, 'n_tokens_neox': 3850, 'n_words': 811, 'path': 'Main Code/06-model_selection.ipynb', 'repo_name': 'sergeiissaev/fibrosis_training', 'size': '10879'}