<jupyter_start><jupyter_text>Resnet으로 train한 모델로 1channel짜리 feature 뽑아보기<jupyter_code>import numpy as np
import torch
from torch import nn
from torch.utils.data import DataLoader
import torch.backends.cudnn as cudnn
import torchvision
from torchvision.transforms import transforms
import matplotlib.pyplot as plt 

# check if gpu training is available
if torch.cuda.is_available():
    device = torch.device('cuda')
    cudnn.deterministic = True
    cudnn.benchmark = True
else:
    device = torch.device('cpu')
    
print("Using...", device)
n_channel=1
# 1ch. 
if n_channel == 1:
    from atm.simclr import resnet
    fn_save = './runs/Sep23_14-03-04_hoseung-kasi/Resnet_ch1_cifar10_bn256_400_399.pth'
    transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Lambda(lambda x: x.mean(dim=0, keepdim=True)),
     transforms.Normalize((0.5), (0.2))])
elif n_channel == 3:
    from torchvision.models import resnet
    fn_save = './runs/resnet_ch3_399.pth'
    transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
    
model = resnet.resnet50(pretrained=False, num_classes=10)
model.to(device)
checkpoint = torch.load(fn_save, map_location=device)
state_dict = checkpoint['state_dict']

log = model.load_state_dict(state_dict, strict=False)

print(log)
model.fc
# Remove the last FC
model.fc = nn.Identity()<jupyter_output><empty_output><jupyter_text>Get the features<jupyter_code>batch_size = 64

test_set = torchvision.datasets.CIFAR10(root='./data', train=False,
#test_set = torchvision.datasets.STL10(root='./data', split='test',
                                       download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size,
                                         shuffle=False, num_workers=2)
feature_arr = []
label_arr = []
for images, l in test_loader:
    images = images.to(device)

    #with autocast(enabled=args.fp16_precision):
    features = model(images)
        
    feature_arr.append(features.cpu().detach().numpy())
    label_arr.append(l)

feature_arr = np.concatenate(feature_arr)
label_arr = np.concatenate(label_arr)

print("feature vector dim must be 2048", features.shape)
print("feature min max", features.min(), features.max()) # 0 ~ 2.x?<jupyter_output>feature vector dim must be 2048 torch.Size([16, 2048])
feature min max tensor(0., device='cuda:0', grad_fn=<MinBackward1>) tensor(3.2481, device='cuda:0', grad_fn=<MaxBackward1>)<jupyter_text>t-SNE<jupyter_code>from sklearn.manifold import TSNE

perp = 75
n_iter = 1000
tsne = TSNE(n_components=2, verbose=1, perplexity=perp, n_iter=n_iter)
# n_components : the number of reduced dimension.
# perplexity : (roughly) number of neighbors.
tsne_results = tsne.fit_transform(feature_arr)

plt.figure(figsize=(16,10))
plt.scatter(tsne_results[:,0], tsne_results[:,1], c=label_arr, cmap='Set1')
plt.savefig(f"1ch_stl_400epochs_perp{perp}_iter{n_iter}.png")
plt.show()<jupyter_output>[t-SNE] Computing 226 nearest neighbors...
[t-SNE] Indexed 10000 samples in 0.009s...
[t-SNE] Computed neighbors for 10000 samples in 2.472s...
[t-SNE] Computed conditional probabilities for sample 1000 / 10000
[t-SNE] Computed conditional probabilities for sample 2000 / 10000
[t-SNE] Computed conditional probabilities for sample 3000 / 10000
[t-SNE] Computed conditional probabilities for sample 4000 / 10000
[t-SNE] Computed conditional probabilities for sample 5000 / 10000
[t-SNE] Computed conditional probabilities for sample 6000 / 10000
[t-SNE] Computed conditional probabilities for sample 7000 / 10000
[t-SNE] Computed conditional probabilities for sample 8000 / 10000
[t-SNE] Computed conditional probabilities for sample 9000 / 10000
[t-SNE] Computed conditional probabilities for sample 10000 / 10000
[t-SNE] Mean sigma: 0.218101
[t-SNE] KL divergence after 250 iterations with early exaggeration: 74.067673
[t-SNE] KL divergence after 1000 iterations: 1.099962<jupyter_text>오.... 쪼금 나옴!   EvaluateFreeze all the layers.<jupyter_code># freeze all layers
for name, param in model.named_parameters():
    if name not in ['fc.weight', 'fc.bias']:
        param.requires_grad = False

parameters = list(filter(lambda p: p.requires_grad, model.parameters()))
assert len(parameters) == 2  # fc.weight, fc.bias

print("un-freezed parameters:", len(parameters))
optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, weight_decay=0.0008)
criterion = torch.nn.CrossEntropyLoss().to(device)
def accuracy(output, target, topk=(1,)):
    """Computes the accuracy over the k top predictions for the specified values of k"""
    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)

        _, pred = output.topk(maxk, 1, True, True)
        pred = pred.t()
        correct = pred.eq(target.view(1, -1).expand_as(pred))

        res = []
        for k in topk:
            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
            res.append(correct_k.mul_(100.0 / batch_size))
        return res
top1_accuracy = 0
top5_accuracy = 0
for counter, (x_batch, y_batch) in enumerate(test_loader):
    x_batch = x_batch.to(device)
    y_batch = y_batch.to(device)

    logits = model(x_batch)

    top1, top5 = accuracy(logits, y_batch, topk=(1,5))
    top1_accuracy += top1[0]
    top5_accuracy += top5[0]
    
top1_accuracy /= (counter + 1)
top5_accuracy /= (counter + 1)
print(f"Top1 Test accuracy: {top1_accuracy.item()}\tTop5 test acc: {top5_accuracy.item()}")<jupyter_output>Top1 Test accuracy: 45.55000305175781	Top5 test acc: 88.35000610351562<jupyter_text>Trained model show a reasonable performance.<jupyter_code>epochs = 100
for epoch in range(epochs):
    top1_train_accuracy = 0
    for counter, (x_batch, y_batch) in enumerate(train_loader):
        x_batch = x_batch.to(device)
        y_batch = y_batch.to(device)

        logits = model(x_batch)
        loss = criterion(logits, y_batch)
        top1 = accuracy(logits, y_batch, topk=(1,))
        top1_train_accuracy += top1[0]

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    top1_train_accuracy /= (counter + 1)
    top1_accuracy = 0
    top5_accuracy = 0
    for counter, (x_batch, y_batch) in enumerate(test_loader):
        x_batch = x_batch.to(device)
        y_batch = y_batch.to(device)

        logits = model(x_batch)

        top1, top5 = accuracy(logits, y_batch, topk=(1,5))
        top1_accuracy += top1[0]
        top5_accuracy += top5[0]

    top1_accuracy /= (counter + 1)
    top5_accuracy /= (counter + 1)
    print(f"Epoch {epoch}\tTop1 Train accuracy {top1_train_accuracy.item()}\tTop1 Test accuracy: {top1_accuracy.item()}\tTop5 test acc: {top5_accuracy.item()}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12199084330231871, 'fraction_numerical': 0.05257716733126569, 'mean_word_length': 4.076461769115443, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '177432', 'license': 'MIT', 'n_tokens_mistral': 2633, 'n_tokens_neox': 2357, 'n_words': 539, 'path': 'notebooks/test_trained_resnet_1ch.ipynb', 'repo_name': 'Hoseung/ATM', 'size': '6771'}