<jupyter_start><jupyter_text>KD- hyperparameter might be matter- student model itself can converge on train data if training it alone- but in KD model, student model's performance is poor.- it seems to be required to do hyperparameter grid search<jupyter_code>y_train_ = torch.LongTensor(y_train)
x_train_ = torch.tensor(x_train, dtype=torch.float32)

for p in teacher_model.parameters():
    p.requires_grad_(False)
    
teacher_model.eval()
student_model.train()

loss_fn = torch.nn.CrossEntropyLoss()
lr = 1e-3
optimizer = torch.optim.Adam(teacher_model.parameters(), lr=lr)
loss_ls_ls = []
for lambda_factor in np.linspace(1,10,10):
    loss_ls = []
    for i, t in zip(range(10000), np.linspace(1, 10, 10000)):
        student_model = torch.nn.Sequential(
        torch.nn.Linear(2,50),
        torch.nn.ReLU(),
        torch.nn.Linear(50,25),
        torch.nn.ReLU(),
        torch.nn.Linear(25,3),
    )
    for m in student_model.children():
        if isinstance(m, torch.nn.Linear):
            torch.nn.init.xavier_normal_(m.weight.data)
            if m.bias is not None:
                torch.nn.init.normal_(m.bias.data)
        
        y_output = student_model(x_train_)
        y_lesson = teacher_model(x_train_)
        l1 = torch.nn.LogSoftmax()(y_output/t)
        l2 = torch.nn.LogSoftmax()(y_lesson/t)
        kl_loss = F.kl_div(l1, l2)
        loss = loss_fn(torch.nn.LogSoftmax()(y_output), y_train_) + lambda_factor*(t**2)*kl_loss
        loss_ls.append(loss.detach().cpu().numpy().item())

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
    loss_ls_ls.append(loss_ls)
    
    student_model.eval()
    student_output = torch.nn.LogSoftmax()(student_model(x_train_))
    loss = loss_fn(student_output, y_train_)
    print(loss)
    print('student model accuracy(%):',sum(torch.argmax(student_output, dim=1) == y_train_).item()/len(y_train_) * 100)

    teacher_output = torch.nn.LogSoftmax()(teacher_model(x_train_))
    loss = loss_fn(teacher_output, y_train_)
    print(loss)
    print('teacher model accuracy(%):', sum(torch.argmax(teacher_output,dim=1) == y_train_).item()/len(y_train_) * 100)

    print(torch.argmax(student_output, dim=1))
    plt.plot(loss_ls)<jupyter_output>tensor(1.6960, grad_fn=<NllLossBackward>)
student model accuracy(%): 33.33333333333333
tensor(0.0047)
teacher model accuracy(%): 100.0
tensor([2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, [...]
{'fraction_non_alphanumeric': 0.1810079901659496, 'fraction_numerical': 0.10295021511985249, 'mean_word_length': 2.7242562929061784, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '62243', 'license': 'MIT', 'n_tokens_mistral': 1689, 'n_tokens_neox': 1396, 'n_words': 406, 'path': 'Knowledge_Distillation/kd_implement_code.ipynb', 'repo_name': 'HenryPaik1/paper_implement', 'size': '3254'}