<jupyter_start><jupyter_text>Tuning the no-answer performance for SQ2 We saw the inferior performance of compound model caused by weaker classification accuracy. In the paper we follow the BERT's approach for the independent model (using score score = start_logit[0] + end_logit[0] as no-answer logit), and use score = logit\[0,0\] for joint/compound models as a no-answer logit. Note the bert's preprocessing finds optimal threshold on dev data over the difference of the lowest no-answer score and best span answer score (no_ans_score=score-best_span_score) in all windows (subparts of the input that satisfy model's input length constraint) of the example.Here, we investigate what happens if we fuse no-answer scores from independent heads (both start/end probability spaces are extended with no answer), joint heads score and best span answer score via trivial logistic regression and see whether there is any difference than when using official BERT's approach.<jupyter_code>%load_ext autoreload
%autoreload 2
import torch
import math
import os
os.chdir("../")

from IPython.display import Markdown, display
def printmd(string):
    display(Markdown(string))
    
def colorize(string,color="red"):
     return f"<span style=\"color:{color}\">{string}</span>"
print(os.getcwd())<jupyter_output>/home/anonymized_name/research/JointSpanExtraction<jupyter_text>Lets try it out for best and worst models (same code is repeated down for the worst model)<jupyter_code>import pickle
# This checkpoint: EM_74.14_F1_76.85_L_2.63
with open("squad2_scores_and_gts_best_model.pkl","rb") as f:
    data = pickle.load(f)
scores,labels = [],[]
for _id, l in data['labels'].items():
    labels.append(l)
    scores.append(data['scores'][_id])
import torch
X = torch.FloatTensor(scores)
Y = torch.BoolTensor(labels)
print(X,Y)
print(X.shape, Y.shape)
class ConstrainedLR(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = torch.nn.Linear(4,1,bias=True)
        
    def forward(self,X):
        return self.linear(X).squeeze(-1)
@torch.no_grad()
def evaluate_model(M,X,Y):
    M.eval()
    odds = M(X)
    predictions=odds>0.
    accuracy = (predictions==Y).sum()/float(len(Y))
    return accuracy
evaluate_model(ConstrainedLR(),X,Y)
import torch.nn.functional as F
from tqdm import tqdm
from transformers import get_linear_schedule_with_warmup
def run_training():
    STEPS=100
    model =  ConstrainedLR()
    model = model.train()
    opt = torch.optim.SGD(model.parameters(), lr=0.3)
    scheduler = get_linear_schedule_with_warmup(
                    opt,
                    num_warmup_steps=10,
                    num_training_steps=STEPS
                )

    best_acc= 0
    best_r = None
    iterator = tqdm(range(STEPS))
    labels=Y.float()
    for i in iterator:
        log_odds = model(X)
        l_list = F.binary_cross_entropy_with_logits(log_odds, labels, reduction='none')
        l = l_list.mean()
        l.backward()
        opt.step()
        scheduler.step()
        opt.zero_grad()
        if i % 1 == 0:
            r = { k: v.tolist() for k,v in dict(model.named_parameters()).items()}
            model.eval()
            with torch.no_grad():
                acc = evaluate_model(model, X, Y)
            model.train()
            if acc>best_acc:
                iterator.set_description(str(acc))
                best_acc = acc
                best_r = r
    return best_acc, best_r
total_best_acc, total_best_r = 0.,None
for _ in range(20):
    acc,r = run_training()
    if acc>total_best_acc:
        print(r)
        total_best_acc=acc
        total_best_r=r
print("Total best:")
print(total_best_acc)
print(total_best_r)<jupyter_output>Total best:
tensor(0.8038)
{'linear.weight': [[0.42036378383636475, -0.20511984825134277, 0.10401123017072678, -0.40278103947639465]], 'linear.bias': [0.12815922498703003]}<jupyter_text>Solution from the BERT's source get_predictions method<jupyter_code>found_model = ConstrainedLR()
found_model.linear.weight[0]=torch.FloatTensor([1.,0.,0.,-1,])
found_model.linear.bias[0] =0.

found_best_threshold=-0.752
evaluate_model(found_model,X+found_best_threshold,Y)<jupyter_output><empty_output><jupyter_text>__Conclusion 1__: For the best model, solution is approximately the same!<jupyter_code>import pickle
# This checkpoint: EM_72.62_F1_75.05_L_2.72

with open("squad2_scores_and_gts_worst_model.pkl","rb") as f:
    data = pickle.load(f)
scores,labels = [],[]
for _id, l in data['labels'].items():
    labels.append(l)
    scores.append(data['scores'][_id])
import torch
X = torch.FloatTensor(scores)
Y = torch.BoolTensor(labels)
print(X,Y)
print(X.shape, Y.shape)
class ConstrainedLR(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = torch.nn.Linear(4,1,bias=True)
        
    def forward(self,X):
        return self.linear(X).squeeze(-1)
@torch.no_grad()
def evaluate_model(M,X,Y):
    M.eval()
    odds = M(X)
    predictions=odds>0.
    accuracy = (predictions==Y).sum()/float(len(Y))
    return accuracy
evaluate_model(ConstrainedLR(),X,Y)
import torch.nn.functional as F
from tqdm import tqdm
from transformers import get_linear_schedule_with_warmup
def run_training():
    STEPS=100
    model =  ConstrainedLR()
    model = model.train()
    opt = torch.optim.SGD(model.parameters(), lr=0.3)
    scheduler = get_linear_schedule_with_warmup(
                    opt,
                    num_warmup_steps=10,
                    num_training_steps=STEPS
                )

    best_acc= 0
    best_r = None
    iterator = tqdm(range(STEPS))
    labels=Y.float()
    for i in iterator:
        log_odds = model(X)
        l_list = F.binary_cross_entropy_with_logits(log_odds, labels, reduction='none')
        l = l_list.mean()
        l.backward()
        opt.step()
        scheduler.step()
        opt.zero_grad()
        if i % 1 == 0:
            r = { k: v.tolist() for k,v in dict(model.named_parameters()).items()}
            model.eval()
            with torch.no_grad():
                acc = evaluate_model(model, X, Y)
            model.train()
            if acc>best_acc:
                iterator.set_description(str(acc))
                best_acc = acc
                best_r = r
    return best_acc, best_r
total_best_acc, total_best_r = 0.,None
for _ in range(20):
    acc,r = run_training()
    if acc>total_best_acc:
        print(r)
        total_best_acc=acc
        total_best_r=r
print("Total best:")
print(total_best_acc)
print(total_best_r)<jupyter_output>Total best:
tensor(0.7867)
{'linear.weight': [[0.3304724097251892, -0.03998826816678047, -0.004543735645711422, -0.4211843013763428]], 'linear.bias': [0.7147210836410522]}<jupyter_text>Solution from the BERT's source get_predictions method<jupyter_code>found_model = ConstrainedLR()
found_model.linear.weight[0]=torch.FloatTensor([1.,0.,0.,-1,])
found_model.linear.bias[0] =0.

found_best_threshold=-0.945
evaluate_model(found_model,X+found_best_threshold,Y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1162558769055421, 'fraction_numerical': 0.03946431115543525, 'mean_word_length': 3.3875, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '445824', 'license': 'MIT', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2359, 'n_words': 521, 'path': 'analysis/squad2_noans_tuning.ipynb', 'repo_name': 'KNOT-FIT-BUT/JointSpanExtraction', 'size': '7019'}