<jupyter_start><jupyter_text>Model Construction<jupyter_code># Model_3, use only the very last hidden layer from Roberta.
from torch import nn
from transformers import RobertaConfig, RobertaModel

class OurRobertaCOPA(torch.nn.Module):
    def __init__(self):
        super(OurRobertaCOPA, self).__init__()
        # self.configuration = RobertaConfig()
        # self.tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
        # self.l1 = RobertaModel(self.configuration)
        self.l1 = RobertaModel.from_pretrained("roberta-base")
        self.l1.requires_grad = True
        self.softmax = nn.Softmax(dim=0)
        self.pre_classifier = torch.nn.Linear(768, 512)
        self.dropout = torch.nn.Dropout(0.3)
        # self.classifier = torch.nn.Linear(768, 5)
        # hidden_dim=32 for later trials.
        # self.lstm = nn.LSTM(768, 32, 1, bias=False)
        self.output_layer = nn.Linear(512, 2)

    def forward(self, sequence_1, sequence_2):
        # Two input here
        token_1 = tokenizer(sequence_1)
        token_2 = tokenizer(sequence_2)
        output_1 = self.l1(input_ids=torch.tensor(token_1["input_ids"]).unsqueeze(0), attention_mask=torch.tensor(token_1["attention_mask"]).unsqueeze(0))[0]
        output_2 = self.l1(input_ids=torch.tensor(token_2["input_ids"]).unsqueeze(0), attention_mask=torch.tensor(token_2["attention_mask"]).unsqueeze(0))[0]
        # RobertaModel(RobertaConfig())

        # _, (hidden_rep_1, _) = self.lstm(output_1.unsqueeze(0))
        # _, (hidden_rep_2, _) = self.lstm(output_2.unsqueeze(0))
        # _, (hidden_rep_1, _) = self.lstm(output_1)
        # _, (hidden_rep_2, _) = self.lstm(output_2)

        hidden_rep_1 = torch.nn.ReLU()(self.pre_classifier(output_1[0])).squeeze(0)
        hidden_rep_2 = torch.nn.ReLU()(self.pre_classifier(output_2[0])).squeeze(0)
        pooler_1 = hidden_rep_1[:, 0]
        pooler_2 = hidden_rep_2[:, 0]
        # hidden_rep_1 = self.pre_classifier(output_1[0]).squeeze(0)
        # hidden_rep_2 = self.pre_classifier(output_2[0]).squeeze(0)
        # print("-------hidden_rep_1:")
        # print(hidden_rep_1)
        # print(hidden_rep_1.size())
        # print("-------hidden_rep_2:")
        # print(hidden_rep_2)
        # print(hidden_rep_2.size())
        
        # hidden_rep = torch.cat((hidden_rep_1.unsqueeze(1), hidden_rep_2.unsqueeze(1)), 1)
        # hidden_rep = self.dropout(torch.cat((hidden_rep_1, hidden_rep_2), 0))
        hidden_rep = self.dropout(torch.cat((pooler_1, pooler_2), 0))

        print("-------hidden_rep:")
        # print(hidden_rep)
        print(hidden_rep.size())

        output = self.output_layer(hidden_rep.unsqueeze(0))
        print("-------output:")
        # print(output)
        print(output.size())
        print("--------------")

        output_squezzed = output.squeeze(0).squeeze(0)
        print("-------output_squezzed:")
        print(output_squezzed)
        print(output_squezzed.size())
        print("--------------")
        
        # y_hat = softmax(output_squezzed)
        # y_sum =  torch.sum(y_hat, 0)
        # col1= torch.sum(y_hat, 0)[0]
        # col2 = torch.sum(y_hat, 0)[1]
        # y_result = torch.tensor(torch.argmax(y_sum)).type(torch.FloatTensor)
        # y_result = torch.tensor(y_sum)
        
        return output_squezzed<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># Initialization
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
# model = OurRobertaCOPA()
model = RobertaForMultipleChoice.from_pretrained('roberta-base')
model.to(device)
ce = nn.CrossEntropyLoss()
softmax = nn.Softmax(dim=0)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)

epochs = 52
per_num_epoch = 1

# train_acc = np.zeros(epochs)
train_loss_by_epoch = np.zeros(epochs)
dev_acc = np.zeros(epochs)
dev_loss_by_epoch = np.zeros(epochs)

start_time = datetime.now()

for j in range(epochs):
    if j % per_num_epoch == 0:
        print('--------------Epoch: ' + str(j+1) + '--------------')
    
    if j % per_num_epoch == 0:
        print(f'Training for epoch {j + 1}.......')
    
    av_train_loss = 0
    # print("av_train_loss_original: ", av_train_loss)
    model.train()
    for i in range(0, dev_raw_data.shape[0] - 100):
        # print("av_train_loss_track: ", av_train_loss)
        prompt = dev_raw_data.iloc[i]['asks-for'] + ". " + dev_raw_data.iloc[i]['p']
        choice0 = dev_raw_data.iloc[i]['a1']
        choice1 = dev_raw_data.iloc[i]['a2']
        label = torch.tensor(dev_raw_data.iloc[i]['most-plausible-alternative'] - 1).unsqueeze(0).to(device)
        # print("label is: ", label)

        encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)
        # encoding = {(prompt+choice0), (prompt+choice1)}
        # outputs = model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)
        outputs = model(**{k: v.unsqueeze(0) for k,v in encoding.items()}, labels=label)
        # print("outputs: ", outputs)

        train_loss = outputs.loss
        train_logits = outputs.logits
        av_train_loss += train_loss

        if i == 0:
            print("train_loss: ", train_loss)
            print("train_logits: ", train_logits)
            print("label: ", label)
        if i == 1:
            print("train_loss: ", train_loss)
            print("train_logits: ", train_logits)
            print("label: ", label)

        train_loss.backward()

        optimizer.step()
        
        optimizer.zero_grad()

    train_loss_by_epoch[j] = av_train_loss / (dev_raw_data.shape[0] - 100)
    print("av_train_loss: ", train_loss_by_epoch[j])

    # validation
    # if (j + 1) % per_num_epoch == 0:
    #     print(f'.......Validating for epoch {j + 1}')
    if (j) % per_num_epoch == 0:
        print(f'.......Validating for epoch {j + 1}')
        av_dev_loss = 0
        # model.eval()
        with torch.no_grad():
            for i in range(dev_raw_data.shape[0] - 99, dev_raw_data.shape[0]):
                # print("av_dev_loss_track: ", av_dev_loss)
                prompt_val = dev_raw_data.iloc[i]['asks-for'] + ". " + dev_raw_data.iloc[i]['p']
                choice0_val = dev_raw_data.iloc[i]['a1']
                choice1_val = dev_raw_data.iloc[i]['a2']
                label_val = torch.tensor(dev_raw_data.iloc[i]['most-plausible-alternative'] - 1).unsqueeze(0).to(device)

                encoding_val = tokenizer([prompt_val, prompt_val], [choice0_val, choice1_val], return_tensors='pt', padding=True).to(device)
                # outputs = model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)
                outputs_val = model(**{k: v.unsqueeze(0) for k,v in encoding_val.items()}, labels=label_val)
                
                dev_loss = outputs_val.loss
                dev_logits = outputs_val.logits
                av_dev_loss += dev_loss
                
                if i == dev_raw_data.shape[0] - 99:
                    print("dev_loss: ", dev_loss)
                    print("dev_logits: ", dev_logits)
                    print("label: ", label_val)
                if i == dev_raw_data.shape[0] - 1:
                    print("dev_loss: ", dev_loss)
                    print("dev_logits: ", dev_logits)
                    print("label: ", label_val)

                #calculate accuracy
                y_pred = 1 if outputs_val.logits[0][1] > outputs_val.logits[0][0] else 0
                y_pred = torch.tensor(y_pred).unsqueeze(0).to(device)
                # print("y_pred: ", y_pred)
                # print("label: ", label)
                # print("y_pred =? label: ", y_pred == label)
                if y_pred == label_val:
                    dev_acc[j] += 1
                
        dev_acc[j] /= 100
        print("dev_acc[j]: ", dev_acc[j])
        dev_loss_by_epoch[j] = av_dev_loss / 100
        
    # learning rate decay
    # if j == 5:
    #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
    # elif j == 15:
    #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
    # elif j == 20:
    #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
    # elif j == 40:
    #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-6)
    # elif j == 50:
    #     optimizer = torch.optim.Adam(model.parameters(), lr=1e-7)    
    scheduler.step()

end_time = datetime.now()
print(f'Training completed in {str(end_time - start_time)}')
# plot
plt.figure(figsize=(14, 7))
plt.title("Loss VS Epoch")

plt.plot(train_loss_by_epoch, label="train_loss")
plt.plot(dev_loss_by_epoch, label="dev_loss")
plt.xlabel("epoch")
plt.ylabel("loss")
plt.legend()
plt.show()
# plot
plt.figure(figsize=(14, 7))
plt.title("acc VS Epoch")

plt.plot(dev_acc, label="dev_acc")
plt.xlabel("epoch")
plt.ylabel("acc")
plt.legend()
plt.show()
# save the model
torch.save(model, 'RoBERTa.pth')# epochs = 100, lr=1e-2 -> scheular<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>test_model = torch.load('RoBERTa.pth')
test_model.eval()
num_correct_pred = 0

with torch.no_grad():
    for i in range(0, test_raw_data.shape[0]):
        prompt = test_raw_data.iloc[i]['asks-for'] + ". " + test_raw_data.iloc[i]['p']
        choice0 = test_raw_data.iloc[i]['a1']
        choice1 = test_raw_data.iloc[i]['a2']
        label_test = torch.tensor(test_raw_data.iloc[i]['most-plausible-alternative']).unsqueeze(0).to(device)

        encoding_test = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)
        # outputs = test_model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)
        outputs_test = test_model(**{k: v.unsqueeze(0) for k,v in encoding_test.items()}, labels=label_test)
        print("outputs_test.logits: ", outputs_test.logits)
        # test_logits = outputs_test.logits

        #calculate accuracy
        y_pred_test = 1 if outputs_test.logits[0][1] > outputs_test.logits[0][0] else 0
        y_pred_test = torch.tensor(y_pred_test).unsqueeze(0).to(device)

        if y_pred_test == label_test:
            print("test_logits: ", outputs_test.logits)
            print("y_pred: ", y_pred_test)
            print("label: ", label_test)
            num_correct_pred += 1

acc = num_correct_pred / test_raw_data.shape[0]
print("test_accuracy = ", acc)
# test_model.eval()
num_correct_pred = 0

with torch.no_grad():
    for i in range(0, test_raw_data.shape[0]):
        prompt = test_raw_data.iloc[i]['question'] + ". " + test_raw_data.iloc[i]['premise']
        choice0 = test_raw_data.iloc[i]['choice1']
        choice1 = test_raw_data.iloc[i]['choice2']
        label = torch.tensor(test_raw_data.iloc[i]['label']).unsqueeze(0).to(device)

        encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)
        outputs = test_model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)

        test_logits = outputs.logits

        #calculate accuracy
        y_pred = 1 if outputs.logits[0][1] > outputs.logits[0][0] else 0
        y_pred = torch.tensor(y_pred).unsqueeze(0).to(device)

        if y_pred == label:
            print("test_logits: ", test_logits)
            print("y_pred: ", y_pred)
            print("label: ", label)
            num_correct_pred += 1

acc = num_correct_pred / test_raw_data.shape[0]
print("test_accuracy = ", acc)
# Revise
# Training
ce = nn.CrossEntropyLoss()
softmax = nn.Softmax(dim=0)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)

epochs = 100
per_num_epoch = 1

# train_acc = np.zeros(epochs)
train_loss_by_epoch = np.zeros(epochs)
dev_acc = np.zeros(epochs)
dev_loss_by_epoch = np.zeros(epochs)

start_time = datetime.now()

for j in range(epochs):
    if j % per_num_epoch == 0:
        print('--------------Epoch: ' + str(j+1) + '--------------')
    
    if j % per_num_epoch == 0:
        print(f'Training for epoch {j + 1}.......')
    av_train_loss = 0
    # print("av_train_loss_original: ", av_train_loss)
    model.train()
    for i in range(0, train_raw_data.shape[0]):
        # print("av_train_loss_track: ", av_train_loss)
        prompt = train_raw_data.iloc[i]['question'] + ". " + train_raw_data.iloc[i]['premise']
        choice0 = train_raw_data.iloc[i]['choice1']
        choice1 = train_raw_data.iloc[i]['choice2']
        label = torch.tensor(train_raw_data.iloc[i]['label']).unsqueeze(0).to(device)
        # print("label is: ", label)
        # label = torch.tensor(rawdata.iloc[i]['label']).unsqueeze(0).to(device)

        encoding = tokenizer([prompt, prompt], [choice0, choice1], return_tensors='pt', padding=True).to(device)
        # outputs = model(input_ids=encoding['input_ids'].unsqueeze(0), attention_mask=encoding['attention_mask'].unsqueeze(0), labels=label)
        
        outputs = model(**{k: v.unsqueeze(0) for k,v in encoding.items()}, labels=label)
        
        train_loss = outputs.loss
        train_logits = outputs.logits
        av_train_loss += train_loss

        if i == 0:
            print("train_loss: ", train_loss)
            print("train_logits: ", train_logits)
            print("label: ", label)
        if i == 1:
            print("train_loss: ", train_loss)
            print("train_logits: ", train_logits)
            print("label: ", label)

        train_loss.backward()

        optimizer.step()

        # learning rate decay
        if j == 25:
            optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
        elif j == 50:
            optimizer = torch.optim.Adam(model.parameters(), lr=1e-7)
        
        optimizer.zero_grad()

    train_loss_by_epoch[j] = av_train_loss / train_raw_data.shape[0]
    print("av_train_loss: ", train_loss_by_epoch[j])

end_time = datetime.now()
print(f'Training completed in {str(end_time - start_time)}')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15424855082708894, 'fraction_numerical': 0.0207832602855931, 'mean_word_length': 2.815264293419633, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '3324', 'license': 'MIT', 'n_tokens_mistral': 5148, 'n_tokens_neox': 4955, 'n_words': 748, 'path': 'copa_roberta_more_data.ipynb', 'repo_name': 'LiGhtime/NLPProject', 'size': '14146'}