<jupyter_start><jupyter_text>Model Baseline<jupyter_code>importlib.reload(mds)

Loss_test_0 = np.zeros((EPOCH,))
Acc_test_0 = np.zeros((EPOCH,))
Acc_train_0 = np.zeros((EPOCH,))

print('\n\t\t\t\t\tTraining Baseline\n')

T = 0
RHO = 0

model_0 = mds.ML_ISTA_NET(m1,m2,m3)
if cudaopt:
    model_0.cuda()

optimizer = torch.optim.Adam(model_0.parameters(), lr = 0.0001, eps = EPS)
bar = progressbar.ProgressBar()

for epoch in range(EPOCH):

    bar.update((epoch+1)/EPOCH*100)
    # train 1 epoch
    model_0.train()
    train_correct = 0
    for step, (x, y) in enumerate(train_loader):
        b_x = Variable(x)   # batch x, shape (batch, 28*28)
        b_y = Variable(y)               # batch label
        if cudaopt:
            b_y, b_x = b_y.cuda(), b_x.cuda()
        encoded, scores = model_0(b_x,T,RHO)
        
        train_pred = scores.data.max(1, keepdim=True)[1]
        train_correct += train_pred.eq(b_y.data.view_as(train_pred)).long().cpu().sum()
        
        loss = F.nll_loss(scores, b_y)      # negative log likelyhood
        optimizer.zero_grad()               # clear gradients for this training step
        loss.backward()                     # backpropagation, compute gradients
        optimizer.step()                    # apply gradients
    
    Acc_train_0[epoch] =  100 * float(train_correct) /float(len(train_loader.dataset))
    
    # testing
    model_0.eval()
    correct = 0
    test_loss = 0
    for step, (x, y) in enumerate(test_loader):
        b_x = Variable(x)   # batch x, shape (batch, 28*28)
        b_y = Variable(y)               # batch label
        if cudaopt:
            b_y, b_x = b_y.cuda(), b_x.cuda()
        gamma, scores = model_0(b_x,T,RHO)
        test_loss += F.nll_loss(scores, b_y, size_average=False).data[0]
        pred = scores.data.max(1, keepdim=True)[1]
        correct += pred.eq(b_y.data.view_as(pred)).long().cpu().sum()
        
    test_loss /= len(test_loader.dataset)
    Loss_test_0[epoch] = test_loss
    Acc_test_0[epoch] =  100 * float(correct) /float(len(test_loader.dataset))
    
# torch.save(model_0.state_dict(), 'cnn_model.pt')<jupyter_output>/ |#                                                  | 0 Elapsed Time: 0:00:00<jupyter_text>Joint ISTA<jupyter_code>importlib.reload(mds)

Loss_test_jista_r = np.zeros((EPOCH,))
Acc_test_jista_r = np.zeros((EPOCH,))
Acc_train_jista_r = np.zeros((EPOCH,))

print('\n\t\t\t\t\tTraining ML-JISTA \n')


T = 0
RHO = 0

model_jnn = mds.ML_JISTA_NET_J(m1,m2,m3)
if cudaopt:
    model_jnn.cuda()

optimizer = torch.optim.Adam(model_jnn.parameters(), lr = 0.0001, eps = EPS)
bar = progressbar.ProgressBar()

for epoch in range(EPOCH):
#     print("Epoch: " + str(int(epoch)))
    bar.update((epoch+1)/EPOCH*100)
    # train 1 epoch
    model_jnn.train()
    
    train_correct = 0
    for step, (x, y) in enumerate(train_loader):
        b_x = Variable(x)   # batch x, shape (batch, 28*28)
        b_y = Variable(y)               # batch label
        if cudaopt:
            b_y, b_x = b_y.cuda(), b_x.cuda()
        encoded, scores = model_jnn.forward_joint(b_x, b_y, T, RHO)
        
        train_pred = scores.data.max(1, keepdim=True)[1]
        train_correct += train_pred.eq(b_y.data.view_as(train_pred)).long().cpu().sum()
        
        loss = F.nll_loss(scores, b_y)      # negative log likelyhood
        optimizer.zero_grad()               # clear gradients for this training step
        loss.backward()                     # backpropagation, compute gradients
        optimizer.step()                    # apply gradients
    
    Acc_train_jista_r[epoch] =  100 * float(train_correct) /float(len(train_loader.dataset))    
        
    # testing
    model_jnn.eval()
    correct = 0
    test_loss = 0
    for step, (x, y) in enumerate(test_loader):
        b_x = Variable(x)   # batch x, shape (batch, 28*28)
        b_y = Variable(y)               # batch label
        if cudaopt:
            b_y, b_x = b_y.cuda(), b_x.cuda()
        gamma, scores = model_jnn.forward(b_x,T,RHO)
        test_loss += F.nll_loss(scores, b_y, size_average=False).data[0]
        pred = scores.data.max(1, keepdim=True)[1]
        correct += pred.eq(b_y.data.view_as(pred)).long().cpu().sum()
        
    test_loss /= len(test_loader.dataset)
    Loss_test_jista_r[epoch] = test_loss
    Acc_test_jista_r[epoch] =  100 * float(correct) /float(len(test_loader.dataset))
#     print("Performance at epoch " + str(int(epoch)) + ": " + str(Acc_test_ista_r[epoch]))
    
# torch.save(model_jnn.state_dict(), 'mljista_model.pt')<jupyter_output>/ |#                                                  | 0 Elapsed Time: 0:00:00<jupyter_text>Plot accuracies<jupyter_code>plt.style.use('default')
fig = plt.figure(figsize=(6,4))
plt.plot(Acc_train_0, ':b', linewidth = 2,label='ML-ISTA train')
plt.plot(Acc_test_0,'b',linewidth = 2,label='ML-ISTA test')
# plt.plot(Acc_test_ista_r, linewidth = 2,label = 'ML-ISTA')
plt.plot(Acc_train_jista_r,':r', linewidth = 2,label = 'ML-JISTA train')
plt.plot(Acc_test_jista_r,'r', linewidth = 2,label = 'ML-JISTA test')
# plt.plot(Acc_test_fista_r, linewidth = 2,label = 'ML-FISTA')
plt.grid('on')
plt.title('Test and Train Accuracy: Alpha = 1/sqrt(n), M formulation')
plt.legend()
plt.axis([0, EPOCH-1, 90, 101])
plt.show()<jupyter_output>/anaconda/envs/mlvm/lib/python3.6/site-packages/matplotlib/cbook/deprecation.py:107: MatplotlibDeprecationWarning: Passing one of 'on', 'true', 'off', 'false' as a boolean is deprecated; use an actual boolean (True/False) instead.
  warnings.warn(message, mplDeprecation, stacklevel=1)<jupyter_text>Visualise global filters of baseline<jupyter_code>cols = 10
rows = 10
indices = random.sample(range(m3), cols*rows)

dict1 = model_0.W3
atom1_dim = dict1.shape[3]
print(dict1.shape)

dict2 = F.conv_transpose2d(dict1, model_0.W2, stride=model_0.strd1, dilation=1)
atom2_dim = dict2.shape[3]
print(dict2.shape)

dict3 = F.conv_transpose2d(dict2, model_0.W1, stride=model_0.strd2, dilation=1)
atom3_dim = dict3.shape[3]
print(dict3.shape)

idx = 1
plt.figure(figsize=(10,10))
for j in range(rows):
    for i in range(cols):
        plt.subplot(cols,rows,idx)
        plt.imshow(np.reshape(dict3.cpu().data.numpy()[idx-1], (atom3_dim, atom3_dim)), cmap='gray')
        plt.axis('off')
        idx+=1

plt.show()<jupyter_output>torch.Size([512, 128, 4, 4])
torch.Size([512, 64, 12, 12])
torch.Size([512, 1, 28, 28])<jupyter_text>Visualise global filters of JISTA<jupyter_code>cols = 10
rows = 10
indices = random.sample(range(m3), cols*rows)

dict1 = model_jnn.W3
atom1_dim = dict1.shape[3]
print(dict1.shape)

dict2 = F.conv_transpose2d(dict1, model_jnn.W2, stride=model_jnn.strd1, dilation=1)
atom2_dim = dict2.shape[3]
print(dict2.shape)

dict3 = F.conv_transpose2d(dict2, model_jnn.W1, stride=model_jnn.strd2, dilation=1)
atom3_dim = dict3.shape[3]
print(dict3.shape)

idx = 1
plt.figure(figsize=(10,10))
for j in range(rows):
    for i in range(cols):
        plt.subplot(cols,rows,idx)
        plt.imshow(np.reshape(dict3.cpu().data.numpy()[idx-1], (atom3_dim, atom3_dim)), cmap='gray')
        plt.axis('off')
        idx+=1

plt.show()<jupyter_output>torch.Size([512, 128, 4, 4])
torch.Size([512, 64, 12, 12])
torch.Size([512, 1, 28, 28])
{'fraction_non_alphanumeric': 0.1480867523138555, 'fraction_numerical': 0.03605470368835474, 'mean_word_length': 3.101983002832861, 'pattern_counts': {'":': 1, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '79763', 'license': 'MIT', 'n_tokens_mistral': 2982, 'n_tokens_neox': 2809, 'n_words': 494, 'path': 'joint/dev/ML_ISTA/Shared/Run_Models_MNIST_M.ipynb', 'repo_name': 'mm5110/sparse-structures-for-classification', 'size': '7239'}