<jupyter_start><jupyter_text>Test5 Data<jupyter_code>test_logprobs_file = 'finetuning-roberta/outputs_more_epochs/test5-finetuned-roberta-epochs-logprobs'
test_decision_file = 'finetuning-roberta/outputs_more_epochs/test5-finetuned-roberta-epochs-predictions'
test_index_file = 'data/fever/test5.index'

test_logprobs_fp = open(test_logprobs_file, "r")
test_decision_fp = open(test_decision_file, "r")
test_index_fp = open(test_index_file, "r")
test_logprobs_hdr = test_logprobs_fp.readline()
test_logprobs = []
for output in tqdm(test_logprobs_fp.readlines()):
    output = output.rstrip().split()
    if len(output)==4:
        probs = [float(output[1][1:]), float(output[2]), float(output[3][:-1])]
    elif len(output)==5 and output[1]=='[':
        probs = [float(output[2]), float(output[3]), float(output[4][:-1])]
    elif len(output)==5 and output[4]==']':
        probs = [float(output[1][1:]), float(output[2]), float(output[3])]
    elif len(output)==6:
        probs = [float(output[2]), float(output[3]), float(output[4])]
    test_logprobs.append(probs)
test_logprobs = pd.DataFrame(test_logprobs)
test_index = []
for output in tqdm(test_index_fp.readlines()):
    output = output.rstrip().split()
    test_index.append(output)
test_index = pd.DataFrame(test_index)
test_data = pd.concat([test_index, test_logprobs], axis=1, ignore_index=True)
test_data = test_data.drop(1, 1)
test_data = test_data.drop(2, 1)
test_data = test_data.drop(3, 1)
test_data.columns = ['ids', 'supports', 'neutral', 'refutes']
test_data['claim_ids'] = test_data.groupby('ids').cumcount()
test_result = test_data.pivot(index='ids', columns='claim_ids')
test_features = pd.concat([test_result['supports'], test_result['neutral'], test_result['refutes']], ignore_index=True, axis=1)
test_features = test_features.values
test_features = imp.transform(test_features)
pred_test = rf.predict(test_features)
pred_proba_test = rf.predict_proba(test_features)
y_test = [0, 1, 2]

from sklearn import preprocessing
lb = preprocessing.LabelBinarizer()
y_test_binarize = lb.fit_transform(y_test)

pred_test = lb.inverse_transform(pred_test)
label_inv = {0: 'SUPPORTS', 1: 'NOT ENOUGH INFO', 2: 'REFUTES'}
test_classifications = [label_inv[k] for k in pred_test]
rf_file = pd.concat([pd.DataFrame(test_result.index), pd.DataFrame(pred_test)], axis=1)
rf_file.to_csv('data/fever/test5-finetuned-roberta-epochs-rf_file', header=False, index=False)
test_classifications<jupyter_output><empty_output><jupyter_text>Using Neural Network<jupyter_code>import torch
import torchvision
import torchvision.transforms as transforms
import math
class Feedforward(torch.nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(Feedforward, self).__init__()
        self.input_size = input_size
        self.hidden_size  = hidden_size
        self.output_size = output_size
        self.dropout = torch.nn.Dropout(p=0.05)
        self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size[0])
        self.relu = torch.nn.ReLU()
        self.fc2 = torch.nn.Linear(self.hidden_size[0], self.hidden_size[1])
        self.fc3 = torch.nn.Linear(self.hidden_size[1], self.hidden_size[2])
        self.fc4 = torch.nn.Linear(self.hidden_size[2], self.output_size)
        self.sigmoid = torch.nn.Sigmoid()
        
    def forward(self, x):
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        x = self.dropout(x)
        x = self.relu(x)
        x = self.fc3(x)
        x = self.relu(x)
        x = self.fc4(x)
        output = self.sigmoid(x)
        return output
batch_size = 1000
epoch = 150
input_size = 15
hidden_size = [50, 100, 50]
output_size = 3
from sklearn.model_selection import train_test_split
x_train, x_eval, y_train, y_eval = train_test_split(features, labels, test_size = 0.2, random_state = 42)
x_test = test_features
for i in tqdm(range(len(x_train))):
    for j in range(3):
        x_train[i][j*5:(j+1)*5] = np.sort(x_train[i][j*5:(j+1)*5])
for i in tqdm(range(len(x_eval))):
    for j in range(3):
        x_eval[i][j*5:(j+1)*5] = np.sort(x_eval[i][j*5:(j+1)*5])
for i in tqdm(range(len(x_test))):
    for j in range(3):
        x_test[i][j*5:(j+1)*5] = np.sort(x_test[i][j*5:(j+1)*5])
x_train_batches = []
y_train_batches = []
x_eval_batches = []
y_eval_batches = []
x_test_batches = []
for i in tqdm(range(math.ceil(len(x_train)/batch_size))):
    x_train_batches.append(torch.FloatTensor(x_train[i*batch_size:i*batch_size+batch_size]))
    y_train_batches.append(torch.FloatTensor(y_train[i*batch_size:i*batch_size+batch_size]))
for i in tqdm(range(math.ceil(len(x_eval)/batch_size))):
    x_eval_batches.append(torch.FloatTensor(x_eval[i*batch_size:i*batch_size+batch_size]))
    y_eval_batches.append(torch.FloatTensor(y_eval[i*batch_size:i*batch_size+batch_size]))
for i in tqdm(range(math.ceil(len(x_test)/batch_size))):
    x_test_batches.append(torch.FloatTensor(x_test[i*batch_size:i*batch_size+batch_size]))
model = Feedforward(input_size, hidden_size, output_size)
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr = 0.01)
model.eval()
y_pred = model(x_eval_batches[0])
before_train = criterion(y_pred.squeeze(), y_eval_batches[0])
print('Evaluation loss before training' , before_train.item())
for param_group in optimizer.param_groups:
    param_group['lr'] = 0.001
train_loss_evolution = []
eval_loss_evolution = []
for epoch in range(250):
    model.train()
    train_loss = []
    for i in range(len(x_train_batches)):
        optimizer.zero_grad()
        # Forward pass
        y_pred = model(x_train_batches[i])
        # Compute Loss
        loss = criterion(y_pred.squeeze(), y_train_batches[i])
        
        train_loss.append(loss.item())
        # Backward pass
        loss.backward()
        optimizer.step()
    
    model.eval()
    eval_loss = []
    for i in range(len(x_eval_batches)):
        y_pred = model(x_eval_batches[i])
        after_train = criterion(y_pred.squeeze(), y_eval_batches[i])
        eval_loss.append(after_train.item())
    print('Epoch: {}, Training Loss: {}, Evalution Loss: {}'.format(epoch, np.mean(train_loss), np.mean(eval_loss)))
    train_loss_evolution.append(np.mean(train_loss))
    eval_loss_evolution.append(np.mean(eval_loss))
import matplotlib.pyplot as plt
%matplotlib inline
plt.plot(range(len(train_loss_evolution)), train_loss_evolution)
plt.plot(range(len(train_loss_evolution)), eval_loss_evolution)
plt.xlabel('Epoch')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Binary Cross Entropy Loss')
plt.legend(['Training Loss', 'Evalutaion Loss'])
model.eval()
predictions = []
for i in range(len(x_test_batches)):
    y_pred = model(x_test_batches[i])
    predictions.append(y_pred)
probs = []
labels = []
for batch in tqdm(predictions):
    values, idxs = batch.max(1)
    probs = probs + values.tolist()
    labels = labels + idxs.tolist()
ff_file = pd.concat([pd.DataFrame(test_result.index), pd.DataFrame(labels)], axis=1)
ff_file.to_csv('data/fever/test5-finetuned-roberta-epochs-ff-tuned3_file', header=False, index=False)
x_train[0]<jupyter_output><empty_output><jupyter_text>Testing with full test data  Training data<jupyter_code>import pandas as pd
from tqdm import tqdm

hypothesis_file = 'data/fever/train50-truncated.hypothesis'
premise_file = 'data/fever/train50-truncated.premise'
label_file = 'data/fever/train50-truncated.label'
index_file = 'data/fever/train50-truncated.index'
logprobs_file = 'finetuning-roberta/outputs_parameter_tuning/params1/train50-truncated-finetuned-roberta-params1-logprobs'
decision_file = 'finetuning-roberta/outputs_parameter_tuning/params1/train50-truncated-finetuned-roberta-params1-results'

hypothesis_fp = open(hypothesis_file, "r")
premise_fp = open(premise_file, "r")
label_fp = open(label_file, "r")
index_fp = open(index_file, "r")
logprobs_fp = open(logprobs_file, "r")
decision_fp = open(decision_file, "r")
logprobs_hdr = logprobs_fp.readline()
logprobs = []
for output in tqdm(logprobs_fp.readlines()):
    output = output.rstrip().split()
    if len(output)==4:
        probs = [float(output[1][1:]), float(output[2]), float(output[3][:-1])]
    elif len(output)==5 and output[1]=='[':
        probs = [float(output[2]), float(output[3]), float(output[4][:-1])]
    elif len(output)==5 and output[4]==']':
        probs = [float(output[1][1:]), float(output[2]), float(output[3])]
    elif len(output)==6:
        probs = [float(output[2]), float(output[3]), float(output[4])]
    logprobs.append(probs)
logprobs = pd.DataFrame(logprobs)
index = []
for output in tqdm(index_fp.readlines()):
    output = output.rstrip().split()
    index.append(output)
index = index[:len(logprobs)]
index = pd.DataFrame(index)
data = pd.concat([index, logprobs], axis=1, ignore_index=True)
data = data.drop(1, 1)
data = data.drop(2, 1)
data = data.drop(3, 1)
data.columns = ['ids', 'supports', 'neutral', 'refutes']
data['claim_ids'] = data.groupby('ids').cumcount()
result = data.pivot(index='ids', columns='claim_ids')
result.head(10)
supports = result['supports']
refutes = result['refutes']
neutral = result['neutral']
feature_length = int(result.shape[1])

for i in range(int(feature_length/3), result['supports'].shape[1], 1):
    supports = supports.drop(i, 1)
    refutes = refutes.drop(i, 1)
    neutral = neutral.drop(i, 1)
result = pd.concat([supports, neutral, refutes], axis=1)
data = pd.read_json('data/fever/train.sentences.p5.s5.jsonl', lines=True)
data.head(10)
labels = []
for i in tqdm(result.index):
    labels.append(data.loc[data['id']==int(i)]['label'].values[0])
label = {'SUPPORTS': 0, 'NOT ENOUGH INFO': 1, 'REFUTES': 2}
labels = [label[k] for k in labels]
labels = labels[:len(result)]
labels = pd.DataFrame(labels, columns=['label'])
results = pd.concat([result.reset_index(), labels], axis=1, ignore_index=True)
features = results.loc[:, 1:feature_length].values
labels = results[feature_length+1].values
import numpy as np
num_feat = int(feature_length/3)
for i in tqdm(range(len(features))):
    for j in range(3):
        features[i][j*num_feat:(j+1)*num_feat] = -np.sort(-features[i][j*num_feat:(j+1)*num_feat])
max_sentences = 250
new_features = []
for i in tqdm(features):
    fvector = []
    for j in range(3):
        fvector.extend(i[j*num_feat:j*num_feat+max_sentences])
    new_features.append(fvector)
features = new_features
from sklearn.impute import SimpleImputer
import numpy as np
imp = SimpleImputer(missing_values=np.nan, strategy='mean')
imp.fit(features)
from sklearn.preprocessing import label_binarize
features = imp.transform(features)
labels = label_binarize(labels, classes=[0, 1, 2])<jupyter_output><empty_output><jupyter_text>Test data<jupyter_code>test_logprobs_file = 'finetuning-roberta/outputs_parameter_tuning/params1/test-truncated-finetuned-roberta-params1-logprobs'
test_decision_file = 'finetuning-roberta/outputs_parameter_tuning/params1/test-truncated-finetuned-roberta-params1-results'
test_index_file = 'data/fever/test-truncated.index'

test_logprobs_fp = open(test_logprobs_file, "r")
test_decision_fp = open(test_decision_file, "r")
test_index_fp = open(test_index_file, "r")
test_logprobs_hdr = test_logprobs_fp.readline()
test_logprobs = []
for output in tqdm(test_logprobs_fp.readlines()):
    output = output.rstrip().split()
    if len(output)==4:
        probs = [float(output[1][1:]), float(output[2]), float(output[3][:-1])]
    elif len(output)==5 and output[1]=='[':
        probs = [float(output[2]), float(output[3]), float(output[4][:-1])]
    elif len(output)==5 and output[4]==']':
        probs = [float(output[1][1:]), float(output[2]), float(output[3])]
    elif len(output)==6:
        probs = [float(output[2]), float(output[3]), float(output[4])]
    test_logprobs.append(probs)
test_logprobs = pd.DataFrame(test_logprobs)
test_index = []
for output in tqdm(test_index_fp.readlines()):
    output = output.rstrip().split()
    test_index.append(output)
test_index = pd.DataFrame(test_index)
test_data = pd.concat([test_index, test_logprobs], axis=1, ignore_index=True)
test_data = test_data.drop(1, 1)
test_data = test_data.drop(2, 1)
test_data = test_data.drop(3, 1)
test_data.columns = ['ids', 'supports', 'neutral', 'refutes']
test_data['claim_ids'] = test_data.groupby('ids').cumcount()
test_result = test_data.pivot(index='ids', columns='claim_ids')
test_supports = test_result['supports']
test_refutes = test_result['refutes']
test_neutral = test_result['neutral']

#for i in range(int(test_result.shape[1]/3), test_result['supports'].shape[1], 1):
#    test_supports = test_supports.drop(i, 1)
#    test_refutes = test_refutes.drop(i, 1)
#    test_neutral = test_neutral.drop(i, 1)
test_features = pd.concat([test_supports, test_neutral, test_refutes], ignore_index=True, axis=1)
num_feat = int(test_result.shape[1]/3)
test_features = test_features.values
for i in tqdm(range(len(test_features))):
    for j in range(3):
        test_features[i][j*num_feat:(j+1)*num_feat] = -np.sort(-test_features[i][j*num_feat:(j+1)*num_feat])
x_test = []
for i in tqdm(test_features):
    fvector = []
    for j in range(3):
        fvector.extend(i[j*num_feat:j*num_feat+max_sentences])
    x_test.append(fvector)
x_test = imp.transform(x_test)<jupyter_output><empty_output><jupyter_text>Neural Net<jupyter_code>import torch
import torchvision
import torchvision.transforms as transforms
import math
class FeedforwardBig(torch.nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(FeedforwardBig, self).__init__()
        self.input_size = input_size
        self.hidden_size  = hidden_size
        self.output_size = output_size
        self.dropout1 = torch.nn.Dropout(p=0.05)
        self.dropout2 = torch.nn.Dropout(p=0.05)
        self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size[0])
        self.relu = torch.nn.ReLU()
        self.fc2 = torch.nn.Linear(self.hidden_size[0], self.hidden_size[1])
        self.fc3 = torch.nn.Linear(self.hidden_size[1], self.hidden_size[2])
        self.fc4 = torch.nn.Linear(self.hidden_size[2], self.hidden_size[3])
        self.fc5 = torch.nn.Linear(self.hidden_size[3], self.output_size)
        self.sigmoid = torch.nn.Sigmoid()
        
    def forward(self, x):
        x = self.dropout1(x)
        x = self.fc1(x)
        x = self.relu(x)
        x = self.dropout2(x)
        x = self.fc2(x)
        x = self.relu(x)
        x = self.fc3(x)
        x = self.relu(x)
        x = self.fc4(x)
        x = self.relu(x)
        x = self.fc5(x)
        output = self.sigmoid(x)
        return output
batch_size = 100
input_size = max_sentences*3
hidden_size = [250, 100, 25, 10]
output_size = 3
from sklearn.model_selection import train_test_split
x_train, x_eval, y_train, y_eval = train_test_split(features, labels, test_size = 0.2, random_state = 42)
x_train = train_x
y_train = train_y
x_eval = eval_x
y_eval = eval_y
x_train_batches = []
y_train_batches = []
x_eval_batches = []
y_eval_batches = []
x_test_batches = []
for i in tqdm(range(math.ceil(len(x_train)/batch_size))):
    x_train_batches.append(torch.FloatTensor(x_train[i*batch_size:i*batch_size+batch_size]))
    y_train_batches.append(torch.FloatTensor(y_train[i*batch_size:i*batch_size+batch_size]))
for i in tqdm(range(math.ceil(len(x_eval)/batch_size))):
    x_eval_batches.append(torch.FloatTensor(x_eval[i*batch_size:i*batch_size+batch_size]))
    y_eval_batches.append(torch.FloatTensor(y_eval[i*batch_size:i*batch_size+batch_size]))
for i in tqdm(range(math.ceil(len(x_test)/batch_size))):
    x_test_batches.append(torch.FloatTensor(x_test[i*batch_size:i*batch_size+batch_size]))
model = FeedforwardBig(input_size, hidden_size, output_size)
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr = 0.01)
model.eval()
y_pred = model(x_eval_batches[0])
before_train = criterion(y_pred.squeeze(), y_eval_batches[0])
print('Evaluation loss before training' , before_train.item())
for param_group in optimizer.param_groups:
    param_group['lr'] = 0.001
train_loss_evolution = []
eval_loss_evolution = []
for epoch in range(200):
    model.train()
    train_loss = []
    for i in range(len(x_train_batches)):
        optimizer.zero_grad()
        # Forward pass
        y_pred = model(x_train_batches[i])
        # Compute Los
        loss = criterion(y_pred.squeeze(), y_train_batches[i])
        
        train_loss.append(loss.item())
        # Backward pass
        loss.backward()
        optimizer.step()
    
    model.eval()
    eval_loss = []
    for i in range(len(x_eval_batches)):
        y_pred = model(x_eval_batches[i])
        after_train = criterion(y_pred.squeeze(), y_eval_batches[i])
        eval_loss.append(after_train.item())
    print('Epoch: {}, Training Loss: {}, Evalution Loss: {}'.format(epoch, np.mean(train_loss), np.mean(eval_loss)))
    train_loss_evolution.append(np.mean(train_loss))
    eval_loss_evolution.append(np.mean(eval_loss))
import matplotlib.pyplot as plt
%matplotlib inline
plt.plot(range(len(train_loss_evolution)), train_loss_evolution)
plt.plot(range(len(train_loss_evolution)), eval_loss_evolution)
plt.xlabel('Epoch')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Binary Cross Entropy Loss')
plt.legend(['Training Loss', 'Evalutaion Loss'])
plt.rcParams.update({'font.size': 20})
plt.savefig('/home/miproj/4thyr.oct2019/jp730/fever2018/figures/NN-original-training-curve.png', dpi=200, bbox_inches='tight')
model.eval()
predictions = []
for i in range(len(x_test_batches)):
    y_pred = model(x_test_batches[i])
    predictions.append(y_pred)
probs = []
labels = []
for batch in tqdm(predictions):
    values, idxs = batch.max(1)
    probs = probs + values.tolist()
    labels = labels + idxs.tolist()
ff_file = pd.concat([pd.DataFrame(test_result.index), pd.DataFrame(labels)], axis=1)
ff_file.to_csv('data/fever/old_roberta_for_figure', header=False, index=False)
model.eval()
predictions = []
for i in range(len(x_eval_batches)):
    y_pred = model(x_eval_batches[i])
    predictions.append(y_pred)
probs = []
for batch in tqdm(predictions):
    probs.extend(batch.tolist())

labels = []
for batch in tqdm(predictions):
    values, idxs = batch.max(1)
    labels = labels + idxs.tolist()
from sklearn.metrics import roc_curve, auc, accuracy_score
plt.rcParams.update({'font.size': 12})

n_classes = 3
fpr_mlp = dict()
tpr_mlp = dict()
roc_auc_mlp = dict()
for i in range(n_classes):
    fpr_mlp[i], tpr_mlp[i], _ = roc_curve(y_eval[:, i], [x[i] for x in probs])
    roc_auc_mlp[i] = auc(fpr_mlp[i], tpr_mlp[i])

eval_y = [x.tolist().index(1) for x in y_eval]
print("Accuracy", accuracy_score(eval_y, labels))
# Plot of a ROC curve for a specific class
classes = ['SUPPORTS', 'NOT ENOUGH INFO', 'REFUTES']
for i in [0, 2, 1]:
    plt.figure()
    plt.plot(fpr_mlp[i], tpr_mlp[i], label='ROC curve (area = %0.2f)' % roc_auc_mlp[i])
    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic {}'.format(classes[i]))
    plt.legend(loc="lower right")
    plt.show()
    plt.savefig('/home/miproj/4thyr.oct2019/jp730/fever2018/figures/NN-ROC-'+str(classes[i]), dpi=200, bbox_inches='tight')
pred_data = pd.read_json('data/fever/train-papelo-predictions', lines=True)
true_data = pd.read_json('data/fever-data/train.jsonl', lines=True)
def change_label(label):
    if label=='REFUTES':
        return 0
    elif label=='SUPPORTS':
        return 1
    else:
        return 2
pred_y = pred_data['predicted_label'].apply(change_label).tolist()
test_y = true_data['label'].apply(change_label).tolist()
pred_y = label_binarize(pred_y, classes=[0, 1, 2])
test_y = label_binarize(test_y, classes=[0, 1, 2])
from sklearn.metrics import roc_curve, auc

n_classes = 3
fpr_papelo = dict()
tpr_papelo = dict()
roc_auc_papelo = dict()
for i in range(n_classes):
    fpr_papelo[i], tpr_papelo[i], _ = roc_curve(test_y[:, i], pred_y[:, i])
    roc_auc_papelo[i] = auc(fpr_papelo[i], tpr_papelo[i])

print("Accuracy", metrics.accuracy_score(test_y, pred_y))
# Plot of a ROC curve for a specific class
classes = ['REFUTES', 'SUPPORTS', 'NOT ENOUGH INFO']
for i in range(n_classes):
    plt.figure()
    plt.plot(fpr_papelo[i], tpr_papelo[i], '--o', label='ROC curve (area = %0.2f)' % roc_auc_papelo[i])
    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic {}'.format(classes[i]))
    plt.show()
plt.style.use('seaborn-whitegrid')
fig, ax = plt.subplots(3, 3, figsize=(12, 10), sharex=True)
plt.rcParams.update({'font.size': 16})
ax[0][0].plot(fpr_papelo[0], tpr_papelo[0], '--o', label='ROC curve (area = %0.2f)' % roc_auc_papelo[0])
ax[0][0].plot([0, 1], [0, 1], 'k--')
ax[0][0].set_xlim([0.0, 1.0])
ax[0][0].set_ylim([-0.1, 1.05])
ax[0][1].plot(fpr_papelo[2], tpr_papelo[2], '--o', label='ROC curve (area = %0.2f)' % roc_auc_papelo[2])
ax[0][1].plot([0, 1], [0, 1], 'k--')
ax[0][1].set_xlim([0.0, 1.0])
ax[0][1].set_ylim([-0.1, 1.05])
ax[0][2].plot(fpr_papelo[1], tpr_papelo[1], '--o', label='ROC curve (area = %0.2f)' % roc_auc_papelo[1])
ax[0][2].plot([0, 1], [0, 1], 'k--')
ax[0][2].set_xlim([0.0, 1.0])
ax[0][2].set_ylim([-0.1, 1.05])

for i in range(3):
    ax[1][i].plot(fpr_rf[i], tpr_rf[i], label='AUC = %0.3f' % roc_auc_rf[i])
    ax[1][i].plot([0, 1], [0, 1], 'k--')
    ax[1][i].set_xlim([0.0, 1.0])
    ax[1][i].set_ylim([-0.1, 1.05])
    ax[1][i].legend(loc="lower right", handlelength=0, handletextpad=0, fancybox=True, frameon=True)

for i in range(3):
    ax[2][i].plot(fpr_mlp[i], tpr_mlp[i], label='AUC = %0.3f' % roc_auc_mlp[i])
    ax[2][i].plot([0, 1], [0, 1], 'k--')
    ax[2][i].set_xlim([0.0, 1.0])
    ax[2][i].set_ylim([-0.1, 1.05])
    ax[2][i].legend(loc="lower right", handlelength=0, handletextpad=0, fancybox=True, frameon=True)
    

for a in ax.flat:
    a.label_outer()
ax[0][0].set_ylabel('Rule-based')
ax[1][0].set_ylabel('Random Forest')
ax[2][0].set_ylabel('Multi-layer Perceptron')
ax[2][0].set_xlabel('SUPPORTS')
ax[2][1].set_xlabel('NOT ENOUGH INFO')
ax[2][2].set_xlabel('REFUTES')
fig.text(-0.04, 0.5, 'True Positive Rate', ha='center', va='center', rotation='vertical', size=20)
fig.text(0.5, -0.04, 'False Positive Rate', ha='center', size=20)
plt.tight_layout()
plt.savefig('test.png', bbox_inches='tight', dpi=250)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16585951037455013, 'fraction_numerical': 0.024170258141911406, 'mean_word_length': 4.9718758291323955, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '578682', 'license': 'Apache-2.0', 'n_tokens_mistral': 9075, 'n_tokens_neox': 8760, 'n_words': 1444, 'path': 'knowledge_aggregation.ipynb', 'repo_name': 'jontypage/fever2018', 'size': '22507'}