<jupyter_start><jupyter_text>Import libraries<jupyter_code>import torch
import os
import os.path as op
import nibabel as nib
import numpy as np
from nilearn import plotting

import torch.nn as nn
from torch.utils.data import DataLoader

from datasets import SimulationDataset<jupyter_output>/Users/greg/code/gkiar/3d-cnn-ae/env/cn/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.
  warnings.warn(msg, category=DeprecationWarning)<jupyter_text>Create helper functions<jupyter_code>def groom(im):
    if isinstance(im, nib.Nifti1Image):
        sample = im
    else:
        sample = nib.load(im)
    sample = sample.get_data().copy().astype(float)
    sample = np.nan_to_num(sample)
    sample *= 1.0/np.abs(sample).max()
    sample = torch.tensor(sample).view((1), (1), *sample.shape)
    return sample.type('torch.FloatTensor')
def recreate_im(immat, affine, header):
    tmpdat = immat.detach().numpy().reshape(*immat.shape[2:])
    return nib.Nifti1Image(tmpdat, affine=affine, header=header) 
def my_mse(im1, im2):
    return np.mean(np.power(im1-im2, 2))
tensor2np = lambda x: x.detach().numpy().reshape(*x.shape[2:])<jupyter_output><empty_output><jupyter_text>Load model and images, show reconstruction<jupyter_code>bp = '../data/training_8_deconv/'

model_file = op.join(bp, 'model.py')
model_file = model_file if op.isfile(model_file) else None

params_file = sorted([op.join(bp, f)
                      for f in os.listdir(bp)
                      if f.endswith('.pt')])[-1]

print("Directory: {0}".format(bp))
print("Using current model: {0}".format(model_file is None))
print("Parameters: {0}".format(params_file))
if model_file is not None:
    exec(open(model_file).read())
else:
    from model import Autoencoder3D
model = torch.load(params_file, map_location="cpu")
if isinstance(model, dict):
    tmpmodel = Autoencoder3D()
    tmpmodel.load_state_dict(model['model_state_dict'])
    model = tmpmodel
    
model = model.eval()
tmpimg = '/Users/greg/code/gkiar/analytical-stability/code/realoutdir/10/executive_ri_z.nii.gz'

tmp = nib.load(tmpimg)
affine = tmp.affine.copy()
header = tmp.header
inn = groom(tmpimg)
out = model(inn)

inim = recreate_im(inn, affine, header)
outim = recreate_im(out, affine, header)
print("Orig:")
print(inn.min(), inn.max())
print(my_mse(tensor2np(inn), np.zeros_like(tensor2np(inn))))

print("\nReconst:")
print(out.min(), out.max())
print(my_mse(tensor2np(out), np.zeros_like(tensor2np(out))))
distance = nn.MSELoss()

print(distance(inn, out))
print(my_mse(tensor2np(inn), tensor2np(out)))
# plotting.plot_roi(inim,  cut_coords=[-13, 1, 19], threshold=0, vmin=-10, vmax=10)
# plotting.plot_roi(outim, cut_coords=[-13, 1, 19], threshold=0, vmin=-10, vmax=10)

plotting.plot_stat_map(inim,  cut_coords=[-13, 1, 19], threshold=0.01)
plotting.plot_stat_map(outim, cut_coords=[-13, 1, 19], threshold=0.01)
# plotting.plot_roi(inim,  cut_coords=[-13, 1, 19], threshold=0, vmin=-10, vmax=10)
# plotting.plot_roi(outim, cut_coords=[-13, 1, 19], threshold=0, vmin=-10, vmax=10)

plotting.plot_stat_map(inim,  cut_coords=[-13, 1, 19], threshold=0)
plotting.plot_stat_map(outim, cut_coords=[-13, 1, 19], threshold=0)<jupyter_output><empty_output><jupyter_text>View simulated data<jupyter_code>data_shape = (48, 56, 48)
n = 5
simdat = SimulationDataset(shape=data_shape, n_samples=3)
simdat_loader = DataLoader(simdat, batch_size=1)

simexamples = list(enumerate(simdat_loader))
for d in simexamples:
    simni = recreate_im(d[1], affine, header)
    plotting.plot_stat_map(simni)
# minfl = '../data/sim_simplified_data/model_final.pt'
# minfl = '../data/sim_tanh/model_final.pt'
minfl = '../data/sim_1_interp/model_epoch_5.pt'
simmodel = torch.load(minfl, map_location="cpu")
if isinstance(simmodel, dict):
    tmpmodel = Autoencoder3D()
    tmpmodel.load_state_dict(simmodel['model_state_dict'])
    simmodel = tmpmodel

simmodel = simmodel.eval()
s_inn = groom(simni)
s_out = simmodel(s_inn)

s_inim = recreate_im(s_inn, affine, header)
s_outim = recreate_im(s_out, affine, header)
print("Orig:")
print(s_inn.min(), s_inn.max())
print(my_mse(tensor2np(s_inn), np.zeros_like(tensor2np(s_inn))))

print("\nReconst:")
print(s_out.min(), s_out.max())
print(my_mse(tensor2np(s_out), np.zeros_like(tensor2np(s_out))))
distance = nn.MSELoss()

print(distance(s_inn, s_out))
print(my_mse(tensor2np(s_inn), tensor2np(s_out)))
plotting.plot_stat_map(s_inim,  cut_coords=[-13, 1, 19], threshold=0)
plotting.plot_stat_map(s_outim, cut_coords=[-13, 1, 19], threshold=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15068774378977623, 'fraction_numerical': 0.029152124820365426, 'mean_word_length': 5.710743801652892, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '613047', 'license': 'MIT', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1848, 'n_words': 347, 'path': 'code/applying_model.ipynb', 'repo_name': 'gkiar/3d-cnn-siamese-ae', 'size': '4871'}