<jupyter_start><jupyter_text>Load the training data and label<jupyter_code>train_data = np.loadtxt('./data/Features.csv',delimiter=',',dtype="int")
train_label= np.loadtxt('./data/Labels.csv',delimiter=',',dtype="int")

print("shape of train_data: {}".format(train_data.shape))
print("shape of train_label: {}".format(train_label.shape))<jupyter_output>shape of train_data: (11500, 4096)
shape of train_label: (11500,)
shape of test_data: (1500, 4096)<jupyter_text>Show the training data<jupyter_code>plt.figure(figsize=(20, 20))
for index in range(100):
    image = train_data[index].reshape(64,64)
    plt.subplot(10, 10, index+1)
    plt.imshow(image)
plt.show()
train_image_num = train_data.shape[0]

print("shape of train_data: {}".format(train_data.shape))
print("shape of train_label: {}".format(train_label.shape))


print("train_image_num  is : {}".format(train_image_num))
val_image_num = 1500<jupyter_output><empty_output><jupyter_text>Convert the training labels to one hot vector<jupyter_code>label_temp = np.zeros((train_image_num, 13), dtype = np.float32)
for i in range(train_image_num):
    label_temp[i][train_label[i]] = 1
train_label_onehot = np.copy(label_temp)
print("One-hot training labels shape:",train_label_onehot.shape)<jupyter_output>One-hot training labels shape: (11500, 13)<jupyter_text>Hyperparameters<jupyter_code>EPOCH = 30
Batch_size = 32
Learning_rate = 0.001<jupyter_output><empty_output><jupyter_text>Training with SGD, without bias, learning rate scheduler learning rate initialize with 0.001. Through each epoch, learning rate plus 0.001, it is to say I try learning rate value from 0.001 to 0.03 to find out the region whose loss function value descends the most rapidly.<jupyter_code>net1 = model.Network()

train_batch_num = (train_image_num  -  val_image_num)//Batch_size
val_batch_num = (val_image_num)//Batch_size

net1_train_loss = []
net1_val_loss = []

for epoch in range(1, EPOCH+1):
    train_hit = 0
    val_hit = 0
    total_train_loss = 0
    total_val_loss = 0
    for it in range(train_batch_num):
        pred, train_loss = net1.forward(train_data[it*Batch_size:(it+1)*Batch_size].T, train_label_onehot[it*Batch_size:(it+1)*Batch_size].T)
        pred_index = np.argmax(pred, axis=0)
        train_hit += (pred_index==train_label[it*Batch_size:(it+1)*Batch_size]).sum()
        total_train_loss += train_loss
        
        net1.backward(train_data[it*Batch_size:(it+1)*Batch_size].T)
        net1.update(Learning_rate)
        
    for titt in range(val_batch_num):
        tit=train_batch_num+titt
        pred, val_loss = net1.forward(train_data[tit*Batch_size:(tit+1)*Batch_size].T, train_label_onehot[tit*Batch_size:(tit+1)*Batch_size].T)
        pred_index = np.argmax(pred, axis=0)
        val_hit += (pred_index==train_label[tit*Batch_size:(tit+1)*Batch_size]).sum()
        total_val_loss += val_loss
    
    net1_train_loss.append(total_train_loss/train_batch_num)
    net1_val_loss.append(total_val_loss/val_batch_num)
    Learning_rate += 0.001
    print('Epoch:%3d'%epoch, '|Train Loss:%8.4f'%(total_train_loss/train_batch_num), '|Train Acc:%3.4f'%(train_hit/(train_image_num-val_image_num)*100.0)
          , '|Val Loss:%8.4f'%(total_val_loss/val_batch_num), '|Val Acc:%3.4f'%(val_hit/val_image_num*100.0))
plt.figure(1)
plt.plot(range(1,EPOCH+1), net1_train_loss)
plt.title('Learning rate scheduler')
plt.ylabel('Loss function value')
plt.xlabel('epochs')
plt.show()<jupyter_output><empty_output><jupyter_text>Training with SGD, without bias, lr = 0.008<jupyter_code>Learning_rate = 0.008
EPOCH = 40
Batch_size = 32
net1 = model.Network()

train_batch_num = (train_image_num  -  val_image_num)//Batch_size
val_batch_num = (val_image_num)//Batch_size

net1_train_loss = []
net1_val_loss = []
net1_train_acc = []
net1_val_acc = []

for epoch in range(1, EPOCH+1):
    train_hit = 0
    val_hit = 0
    total_train_loss = 0
    total_val_loss = 0
    for it in range(train_batch_num):
        pred, train_loss = net1.forward(train_data[it*Batch_size:(it+1)*Batch_size].T, train_label_onehot[it*Batch_size:(it+1)*Batch_size].T)
        pred_index = np.argmax(pred, axis=0)
        train_hit += (pred_index==train_label[it*Batch_size:(it+1)*Batch_size]).sum()
        total_train_loss += train_loss
        
        net1.backward(train_data[it*Batch_size:(it+1)*Batch_size].T)
        net1.update(Learning_rate)
        
    for titt in range(val_batch_num):
        tit=train_batch_num+titt
        pred, val_loss = net1.forward(train_data[tit*Batch_size:(tit+1)*Batch_size].T, train_label_onehot[tit*Batch_size:(tit+1)*Batch_size].T)
        pred_index = np.argmax(pred, axis=0)
        val_hit += (pred_index==train_label[tit*Batch_size:(tit+1)*Batch_size]).sum()
        total_val_loss += val_loss
    
    net1_train_loss.append(total_train_loss/train_batch_num)
    net1_val_loss.append(total_val_loss/val_batch_num)
    net1_train_acc.append(train_hit/(train_image_num-val_image_num)*100.0)
    net1_val_acc.append(val_hit/val_image_num*100.0)
    if (epoch%5==0):
        print('Epoch:%3d'%epoch, '|Train Loss:%8.4f'%(total_train_loss/train_batch_num), '|Train Acc:%3.4f'%(train_hit/(train_image_num-val_image_num)*100.0)
              , '|Val Loss:%8.4f'%(total_val_loss/val_batch_num), '|Val Acc:%3.4f'%(val_hit/val_image_num*100.0))<jupyter_output>Epoch:  5 |Train Loss:  0.5026 |Train Acc:83.1500 |Val Loss:  0.8079 |Val Acc:70.6667
Epoch: 10 |Train Loss:  0.1242 |Train Acc:97.3200 |Val Loss:  0.6659 |Val Acc:80.0667
Epoch: 15 |Train Loss:  0.0032 |Train Acc:99.8300 |Val Loss:  0.7122 |Val Acc:81.6667
Epoch: 20 |Train Loss:  0.0011 |Train Acc:99.8400 |Val Loss:  0.7383 |Val Acc:82.0000
Epoch: 25 |Train Loss:  0.0007 |Train Acc:99.8400 |Val Loss:  0.7595 |Val Acc:81.9333
Epoch: 30 |Train Loss:  0.0005 |Train Acc:99.8400 |Val Loss:  0.7764 |Val Acc:82.0000
Epoch: 35 |Train Loss:  0.0004 |Train Acc:99.8400 |Val Loss:  0.7907 |Val Acc:82.0667
Epoch: 40 |Train Loss:  0.0003 |Train Acc:99.8400 |Val Loss:  0.8030 |Val Acc:81.9333<jupyter_text>Training with SGD and adding bias 0.1, lr = 0.008<jupyter_code>net2 = model.Network(bias=True)

train_batch_num = (train_image_num  -  val_image_num)//Batch_size
val_batch_num = (val_image_num)//Batch_size

net2_train_loss = []
net2_val_loss = []
net2_train_acc = []
net2_val_acc = []

for epoch in range(1, EPOCH+1):
    train_hit = 0
    val_hit = 0
    total_train_loss = 0
    total_val_loss = 0
    for it in range(train_batch_num):
        pred, train_loss = net2.forward(train_data[it*Batch_size:(it+1)*Batch_size].T, train_label_onehot[it*Batch_size:(it+1)*Batch_size].T)
        pred_index = np.argmax(pred, axis=0)
        train_hit += (pred_index==train_label[it*Batch_size:(it+1)*Batch_size]).sum()
        total_train_loss += train_loss
        
        net2.backward(train_data[it*Batch_size:(it+1)*Batch_size].T)
        net2.update(Learning_rate)
        
    for titt in range(val_batch_num):
        tit=train_batch_num+titt
        pred, val_loss = net2.forward(train_data[tit*Batch_size:(tit+1)*Batch_size].T, train_label_onehot[tit*Batch_size:(tit+1)*Batch_size].T)
        pred_index = np.argmax(pred, axis=0)
        val_hit += (pred_index==train_label[tit*Batch_size:(tit+1)*Batch_size]).sum()
        total_val_loss += val_loss
    
    net2_train_loss.append(total_train_loss/train_batch_num)
    net2_val_loss.append(total_val_loss/val_batch_num)
    net2_train_acc.append(train_hit/(train_image_num-val_image_num)*100.0)
    net2_val_acc.append(val_hit/val_image_num*100.0)
    if (epoch%5==0):
        print('Epoch:%3d'%epoch, '|Train Loss:%8.4f'%(total_train_loss/train_batch_num), '|Train Acc:%3.4f'%(train_hit/(train_image_num-val_image_num)*100.0)
              , '|Val Loss:%8.4f'%(total_val_loss/val_batch_num), '|Val Acc:%3.4f'%(val_hit/val_image_num*100.0))
plt.figure(2)
plt.plot(range(1,EPOCH+1), net1_train_loss, linestyle='-', color='blue', label='train, no bias')
plt.plot(range(1,EPOCH+1), net1_val_loss, linestyle='--', color='blue', label='val, no bias')
plt.plot(range(1,EPOCH+1), net2_train_loss, linestyle='-', color='orange', label='train with bias')
plt.plot(range(1,EPOCH+1), net2_val_loss, linestyle='--', color='orange', label='val. with bias')
plt.title('Learning curve')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend(loc='best')
plt.show()
plt.figure(3)
plt.plot(range(1,EPOCH+1), net1_train_acc, linestyle='-', color='blue', label='train, no bias')
plt.plot(range(1,EPOCH+1), net1_val_acc, linestyle='--', color='blue', label='val, no bias')
plt.plot(range(1,EPOCH+1), net2_train_acc, linestyle='-', color='orange', label='train with bias')
plt.plot(range(1,EPOCH+1), net2_val_acc, linestyle='--', color='orange', label='val. with bias')
plt.title('Accuracy')
plt.xlabel('Epochs')
plt.ylabel('accuracy')
plt.legend(loc='best')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14979117281860255, 'fraction_numerical': 0.05327915114572751, 'mean_word_length': 5.195804195804196, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '113390', 'license': 'CC0-1.0', 'n_tokens_mistral': 3767, 'n_tokens_neox': 3451, 'n_words': 542, 'path': 'DLAD_Lab01/LAB1_main.ipynb', 'repo_name': 'YanxiangDing/DLAD2021_Lab', 'size': '8859'}