<jupyter_start><jupyter_text>'H', 'N', 'F', 'T', 'I', 'M', 'B', 'D' ,'C', 'G','Y'   0     1    2    3    4    5    6    7    8    9   10<jupyter_code>out = model(test_tensor.to(device))
print out.shape
a,r = torch.max(out,1)
print(r)
#print r
#ap = torch.nn.AdaptiveAvgPool2d(output_size=(4,4))
#_,b = torch.max(ap(out),1)
#print(b)

#ap = torch.nn.AdaptiveAvgPool2d(output_size=(2,2))
#_,b = torch.max(ap(out),1)
#print(b)
import sys
sys.path.append('../')
from Data.get_dataloader import get_dataloader
batch_size = 1
patch_size = 448
train_path = '/mnt/DATA_CRLM/Patches/Patches_Level0/Patches_448/All/'
test_path = '/mnt/DATA_CRLM/Patches/Patches_Level0/Patches_448/Test/'
save_path = '/mnt/DATA_CRLM/Patches/Checkpoints/PatchCNN_448/'

device_ids=[0,]

device = torch.device('cuda:{}'.format(','.join([str(i) for i in device_ids])) if torch.cuda.device_count>0 else torch.device('cpu'))

dataloaders = {'train':get_dataloader(batch_size=batch_size,patch_size=patch_size,num_workers=2,
                        rootdir = train_path ),\
               'val':get_dataloader(batch_size= batch_size,patch_size=patch_size,num_workers=2,
                      rootdir=test_path)}

test_dl = dataloaders['train']
model.cuda()
softmax = torch.nn.Softmax2d()
for batch_index,(inputs, labels) in enumerate(test_dl):
    print batch_index, labels.cpu().item()
    print inputs.shape
    print torch.max(softmax(model(inputs[:,:3,:,:].to(device))),1)
    break


import os
train_path = '/mnt/DATA_CRLM/Patches/Patches_Level0/Patches_448/All/'
index =36000

model.to(device)

pairwise_dis = torch.nn.PairwiseDistance(p=2)
flist = os.listdir(train_path)
flist.sort()
print len(flist)

tmp_train_img = flist[index]
print tmp_train_img

test_img = plt.imread(train_path+tmp_train_img)

plt.imshow(test_img)
plt.axis('off')
plt.show()

if test_img.max()>2:
    test_img = test_img/255.0
test_tensor = torch.from_numpy(((test_img[:512,:512,:3]-np.array([0.485, 0.456, 0.406]))/ np.array([0.229, 0.224, 0.225])).transpose(2,0,1)).float()
test_tensor = test_tensor.unsqueeze(0)

feature_ref = softmax(model(test_tensor.to(device)))
print feature_ref.shape
print torch.max(feature_ref,1)
#tindex = 3916
simi_index = []
for tindex in range(36000,60000,1000):
    tmp_train_img = flist[tindex]
    #print tmp_train_img

    test_img = plt.imread(train_path+tmp_train_img)

    if test_img.max()>2:
        test_img = test_img/255.0
    test_tensor = torch.from_numpy(((test_img[:512,:512,:3]-np.array([0.485, 0.456, 0.406]))/ np.array([0.229, 0.224, 0.225])).transpose(2,0,1)).float()
    test_tensor = test_tensor.unsqueeze(0)

    feature_tmp = softmax(model(test_tensor.to(device)))
    
    pwdis = torch.mean(pairwise_dis(feature_tmp,feature_ref)).cpu().item()
    if pwdis<0.0005:
        simi_index.append(tmp_train_img)
        print tmp_train_img, pwdis
    

#     plt.imshow(test_img)
#     plt.axis('off')
#     plt.show()
print len(simi_index)
test_img = plt.imread(train_path+simi_index[23])
plt.imshow(test_img)
plt.axis('off')
plt.show()
from sklearn.manifold import TSNE
tsne = TSNE(n_components=2, random_state=0)
label_list = ['H', 'N', 'F', 'T', 'I', 'M', 'B', 'D' ,'C', 'G','Y']
label_dict = {}
label_point = [(35981,81083),(93094,105493),(3888,31980),(105494,128220),(81084,82911),(82912,93093),(0,1030),(3617,3887),(1031,3616),(37981,35980),(128221,128601)]
# for i,f in enumerate(flist):
#     if i == len(flist)-2:
#         break
#     else:
#         if flist[i][0]!=flist[i+1][0]:
#             print i, flist[i][0], i+1, flist[i+1][0]
label_dict = dict(zip(label_list,label_point))<jupyter_output><empty_output><jupyter_text>Plot Scatters  Plot feature map<jupyter_code>X = []
Y =[]

sample_interval = 200
for tlabel in label_list:
    x,y =label_dict[tlabel]
    for tindex in range(x,y,(y-x)/sample_interval):
        tmp_train_img = flist[tindex]
        #print tmp_train_img

        test_img = plt.imread(train_path+tmp_train_img)

        if test_img.max()>2:
            test_img = test_img/255.0
        test_tensor = torch.from_numpy(((test_img[:512,:512,:3]-np.array([0.485, 0.456, 0.406]))/ np.array([0.229, 0.224, 0.225])).transpose(2,0,1)).float()
        test_tensor = test_tensor.unsqueeze(0)

        feature_tmp = softmax(model(test_tensor.to(device)))

        pwdis = torch.mean(pairwise_dis(feature_tmp,feature_ref)).cpu().item()

        X.append(feature_tmp.view(-1).detach().cpu().numpy())
        Y.append(label_list.index(tlabel))

X_2d = tsne.fit_transform(X)
y = np.array(Y)
plt.figure(figsize=(10, 8))
colors = 'r', 'g', 'b', 'c', 'm', 'y', 'k', 'w', 'orange', 'purple','brown'
for i, c, label in zip(range(len(label_list)), colors, label_list):
    plt.scatter(X_2d[y == i, 0], X_2d[y == i, 1], c=c, label=label)
#plt.legend(loc='lower right', bbox_to_anchor=(1.2, 0))
plt.legend(loc='lower right')
#plt.show()
plt.savefig('./tsne_train.png')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18248323511481407, 'fraction_numerical': 0.06685633001422475, 'mean_word_length': 4.487179487179487, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '130845', 'license': 'Apache-2.0', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2018, 'n_words': 303, 'path': 'Load/Load_trained_224.ipynb', 'repo_name': 'giussepi/cyto_CRLM', 'size': '4921'}