<jupyter_start><jupyter_text>Estimate sentence probability with BERT Calculating probability more properly:P_f, P_b: Probability forward pass, backward pass, respectivelyP_f = P(w_0) * P(w_1|w_0) * P(w_2|w_0, w_1) * ... * P(w_N)P_b = P(w_N-1|w_N) * P(w_N-2|w_N-1, w_N) * ... * P(w_0|w_1, w_2, ... ,w_N)P_f, P_b become smaller as the sentence length increases, hence, I try normalizing them by sentence length.Since we're dealing with dependent probabilities, I use a geometric mean.```mean(P_f) = (P(w_0) * P(w_1|w_0) * ... * P(w_N|w_0, w_1, ..., w_{N-1})) ^ (1/N),```where N is the sentence length.Finally, the sentence probability P(S) is the geometric mean of forward and backwards probabilities:```P(S) = (mean(P_f(S)) * mean(P_b(S))) ^ (1/2)```<jupyter_code>import numpy as np
import torch
from transformers import BertTokenizer, BertForMaskedLM

BOS_TOKEN = '[CLS]'
EOS_TOKEN = '[SEP]'
MASK_TOKEN = '[MASK]'

# Load pre-trained model (weights)
with torch.no_grad():
    model = BertForMaskedLM.from_pretrained('bert-large-uncased')
    model.eval()
    # Load pre-trained model tokenizer (vocabulary)
    tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')
def print_top_predictions(probs, k=5):
    probs = probs.detach().numpy()
    top_indexes = np.argpartition(probs, -k)[-k:]
    sorted_indexes = top_indexes[np.argsort(-probs[top_indexes])]
    top_tokens = tokenizer.convert_ids_to_tokens(sorted_indexes)
    print(f"Ordered top predicted tokens: {top_tokens}")
    print(f"Ordered top predicted values: {probs[sorted_indexes]}")
def get_sentence_prob(sentence, verbose=False):
    # Pre-process sentence, adding special tokens
    tokenized_input = tokenizer.tokenize(sentence)
    if tokenized_input[0] != BOS_TOKEN:
        tokenized_input.insert(0, BOS_TOKEN)
    if tokenized_input[-1] != EOS_TOKEN:
        tokenized_input.append(EOS_TOKEN)
    sent_len = len(tokenized_input)
    ids_input = tokenizer.convert_tokens_to_ids(tokenized_input)
    print(f"Processing sentence: {tokenized_input}\n")
    
    sm = torch.nn.Softmax(dim=0) # used to convert last hidden state to probs
    
    sent_prob_forward = 1
    sent_prob_backwards = 1
    # Mask non-special tokens in forward and backwards directions; calculate their probabilities
    for i in range(1, len(tokenized_input) - 1): # Don't loop first and last tokens
        probs_forward = get_directional_prob(sm, tokenized_input, i, 'forward', verbose=verbose)
        probs_backwards = get_directional_prob(sm, tokenized_input, i, 'backwards', verbose=verbose)
        prob_forward = probs_forward[ids_input[i]] # Prediction for masked word 
        prob_backwards = probs_backwards[ids_input[i]] # Prediction for masked word 
        sent_prob_forward *= np.power(prob_forward.detach().numpy(), 1/sent_len)
        sent_prob_backwards *= np.power(prob_backwards.detach().numpy(), 1/sent_len)

        print(f"Word: {tokenized_input[i]} \t Prob_forward: {prob_forward}; Prob_backwards: {prob_backwards}")

    print(f"Geometric-mean forward sentence probability: {sent_prob_forward}")
    print(f"Geometric-mean backward sentence probability: {sent_prob_backwards}\n")
    
    # Obtain geometric average of forward and backward probs
    geom_mean_sent_prob = np.sqrt(sent_prob_forward * sent_prob_backwards)
    print(f"Average normalized sentence prob: {geom_mean_sent_prob}\n")
    return geom_mean_sent_prob
def get_directional_prob(sm, tokenized_input, i, direction, verbose=False):
    current_tokens = tokenized_input[:]
    if direction == 'backwards':
        current_tokens[1:i+1] = [MASK_TOKEN for j in range(i)]
    elif direction == 'forward':
        current_tokens[i:-1] = [MASK_TOKEN for j in range(len(tokenized_input) - 1 - i)]
    else:
        print("Direction can only be 'forward' or 'backwards'")
        exit()
    if verbose: 
        print()
        print(current_tokens)
        
    masked_input = torch.tensor([tokenizer.convert_tokens_to_ids(current_tokens)])
    predictions = model(masked_input)
    global BERTcall_ctr
    BERTcall_ctr += 1
    predictions = predictions[0]
    probs = sm(predictions[0, i]) # Softmax to get probabilities
    if verbose: 
        print_top_predictions(probs)
    
    return probs # Model predictions
get_sentence_prob("Smurfs eat the ancient nuns ungracefully.")
get_sentence_prob("Smurfs eat ancient the nuns ungracefully.")
get_sentence_prob("eat smurfs the ancient nuns ungracefully.")
get_sentence_prob("Smurfs eat the ancient ungracefully nuns.")
get_sentence_prob("Smurfs the eat ancient nuns ungracefully.")
get_sentence_prob("kids eat the red grapes quickly.")
get_sentence_prob("quickly eat the red grapes kids.")
get_sentence_prob("the kids eat the red grapes quickly.")
get_sentence_prob("girls eat the red grapes quickly.")
get_sentence_prob("the red grapes eat kids quickly.")
get_sentence_prob("kids eat red the grapes quickly.")
get_sentence_prob("eat kids the red grapes quickly.")
get_sentence_prob("kids eat the red quickly grapes.")
get_sentence_prob("kids the eat red grapes quickly.")
get_sentence_prob("Colorless green ideas sleep furiously.")
get_sentence_prob("Confused dumb benches eat endlessly.")
get_sentence_prob("Hairless ugly men complain constantly.")
get_sentence_prob("The test was a success.")
get_sentence_prob("The was test a success.")
get_sentence_prob("The test was success a.")
get_sentence_prob("The party was a success.")
get_sentence_prob("The farewell party was definitely not a success.")
get_sentence_prob("He answered unequivocally.")
get_sentence_prob("He answered quickly.", verbose=True)
get_sentence_prob("The guy with small hands demanded a quid pro quo.")
get_sentence_prob("The guy with small hands demanded an exchange.")
get_sentence_prob("This is a sentence.")
get_sentence_prob("This is a macrame.", verbose=False)
get_sentence_prob("This is a joke.", verbose=False)
get_sentence_prob("Are you kidding me?", verbose=False)

get_sentence_prob("Rachel was wearing a lovely satin dress last night.")
get_sentence_prob("Rachel was wearing a lovely satin dress last night.")
get_sentence_prob("Grandma was wearing a lovely satin dress last night.")
get_sentence_prob("Mother was wearing a lovely satin dress last night.")
get_sentence_prob("She was wearing a lovely satin dress last night.")
get_sentence_prob("He was wearing a lovely satin dress last night.")
get_sentence_prob("I was wearing a lovely satin dress last night.")
get_sentence_prob("Angela was wearing a lovely satin dress last night.")
get_sentence_prob("Roberta was wearing a lovely satin dress last night.")
get_sentence_prob("Running was wearing a lovely satin dress last night.")
get_sentence_prob("The man ate the steak.")
get_sentence_prob("The man who arrived late ate the steak with a glass of wine.")
get_sentence_prob("The steak was eaten by the man.")
get_sentence_prob("The stake ate the man.")
get_sentence_prob("He was born in Berlin.")
get_sentence_prob("He was born in Santiago.")
get_sentence_prob("He was born in France.")
get_sentence_prob("He was born in window.")
get_sentence_prob("He was born in was.")

get_sentence_prob("I fed my cat some of it and he damn near passed out.")
get_sentence_prob("I fed my dog some of it and he damn near passed out.")
get_sentence_prob("I fed my window some of it and he damn near passed out.")
get_sentence_prob("I fed my the some of it and he damn near passed out.")
print("Should have similar/high probs\n")
get_sentence_prob("I forgot to take my medicine.")
get_sentence_prob("I forgot to take my medicines.")
get_sentence_prob("I forgot to take my medication.")
get_sentence_prob("I forgot to take my pills.")
print("Should have low probs\n")
get_sentence_prob("I forgot to take my turn.")
get_sentence_prob("I forgot to take my medical.")
get_sentence_prob("I forgot to take my medically.")
get_sentence_prob("I forgot to take my turned.")
get_sentence_prob("We will explore the elements used to construct sentences, and what parts of speech are used to expand and elaborate on them.")
get_sentence_prob("Wikipedia is a multilingual online encyclopedia created and maintained as an open collaboration project by a community of volunteer editors.")
get_sentence_prob("Once she gave her a little cap of red velvet, which suited her so well that she would never wear anything else.")
# Load pre-trained model (weights)
with torch.no_grad():
    model = BertForMaskedLM.from_pretrained('bert-base-uncased')
    model.eval()
    # Load pre-trained model tokenizer (vocabulary)
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
#get_sentence_prob("I fed my cat some of it and he damn near passed out")
get_sentence_prob("He was born in Berlin.")
get_sentence_prob("He was born in Santiago.")
get_sentence_prob("He was born in France.")
get_sentence_prob("He was born in window.")
get_sentence_prob("He was born in was.")
get_sentence_prob("I fed my cat some of it and he damn near passed out.")
get_sentence_prob("I fed my dog some of it and he damn near passed out.")
get_sentence_prob("I fed my window some of it and he damn near passed out.")
get_sentence_prob("I fed my the some of it and he damn near passed out.")
print("Should have similar/high probs\n")
get_sentence_prob("I forgot to take my medicine.")
get_sentence_prob("I forgot to take my medicines.")
get_sentence_prob("I forgot to take my medication.")
get_sentence_prob("I forgot to take my pills.")
print("Should have low probs\n")
get_sentence_prob("I forgot to take my turn.")
get_sentence_prob("I forgot to take my medical.")
get_sentence_prob("I forgot to take my medically.")
get_sentence_prob("I forgot to take my turned.")<jupyter_output>Should have similar/high probs

Processing sentence: ['[CLS]', 'i', 'forgot', 'to', 'take', 'my', 'medicine', '.', '[SEP]']
Word: i 	 Prob_forward: 0.009378484450280666; Prob_backwards: 0.9732086658477783

Word: forgot 	 Prob_forward: 0.00021250064310152084; Prob_backwards: 0.015700489282608032

Word: to 	 Prob_forward: 0.03624541684985161; Prob_backwards: 0.055403366684913635

Word: take 	 Prob_forward: 0.017103174701333046; Prob_backwards: 0.02183195948600769

Word: my 	 Prob_forward: 0.00037714111385867; Prob_backwards: 2.8737551474478096e-05

Word: medicine 	 Prob_forward: 0.01692848652601242; Prob_backwards: 1.0145240594283678e-06

Word: . 	 Prob_forward: 0.9715754985809326; Prob_backwards: 0.9818249344825745


Geometric-mean forward sentence probability: 0.027014838021123058


Geometric-mean backward sentence probability: 0.02007291597720789


Average normalized sentence prob: 0.023286617911063946

Processing sentence: ['[CLS]', 'i', 'forgot', 'to', 'take', 'my', 'medicines', '.'[...]
{'fraction_non_alphanumeric': 0.10170603674540682, 'fraction_numerical': 0.033089613798275214, 'mean_word_length': 5.257478005865103, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 9, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '331496', 'license': 'Apache-2.0', 'n_tokens_mistral': 3679, 'n_tokens_neox': 3313, 'n_words': 1056, 'path': 'notebooks/Bert_as_LM_unidirectional.ipynb', 'repo_name': 'glicerico/wordcat_transformer', 'size': '10668'}