<jupyter_start><jupyter_text>Hw2 Experimentation: Language Models**Author**:<NAME>**Date**:2021-04-13<jupyter_code>%load_ext autoreload
%autoreload 2

import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Module and Package Imports<jupyter_code>from lm import *
from data import *
from generator import Sampler
from experiments import *<jupyter_output><empty_output><jupyter_text>Hyperparameter tuning experiments<jupyter_code># For bigram
ks = [1,0.5,0.1,0.05,0.01,0.005,0.001, 0.0005, 0.0001,0.00005,0.00001]
r = addk_ablation(2,ks)
# For trigram
ks = [1,0.5,0.1,0.05,0.01,0.005,0.001, 0.0005, 0.0001,0.00005,0.00001]
r = addk_ablation(3,ks)
#Backoff ablation with no add-k
ns = [2,3,4,5,6,7]
r = backoff_ablation(ns,None, r)<jupyter_output>brown  read. train: 39802 dev: 8437 test: 8533
reuters  read. train: 38183 dev: 8083 test: 8199<jupyter_text>Plots of Hyperparameter tuning  Add-k bigram<jupyter_code>addk2 = None
with open("data/results/addk_ablation_2.pkl", "rb") as file:
    addk2 = pkl.load(file)
    
addk3 = None
with open("data/results/addk_ablation_3.pkl", "rb") as file:
    addk3 = pkl.load(file)
dnames = ['brown','reuters','gutenberg']
fig,ax = plt.subplots(1,3, figsize = (24,6),sharex = True, sharey = True)

for i,d in enumerate(dnames):
    bdev_perp = [addk2[d][2][i]['perplexity']['dev'] for i in ks]
    tdev_perp = [addk3[d][3][i]['perplexity']['dev'] for i in ks]
    train_perp = [addk2[d][2][i]['perplexity']['train'] for i in ks]
    unigram_train = addk2[d][1]['perplexity']['train']
    unigram_dev = addk2[d][1]['perplexity']['dev']
    if i == 0:
        ax[i].set_ylabel("Perplexity", fontsize = 20)
    ax[i].semilogx(ks, train_perp, label = "Train")
    ax[i].semilogx(ks, bdev_perp, label = "Bigram Dev")
    ax[i].semilogx(ks, tdev_perp, label = "Trigram Dev")
#     ax[i].axhline(unigram_train, label = "Unigram train",linestyle = 'dashed',linewidth = 0.9
#             , c = 'blue')
    ax[i].axhline(unigram_dev, label = "Unigram Dev",linestyle = 'dashed',linewidth = 0.9
            , c = 'red')
    ax[i].set_title(d, fontsize = 20)
plt.tick_params(axis = 'both',labelsize = 15)

plt.suptitle("Perplexity with Add-k Smoothing for Bi-Trigram Models",fontsize = 25)
fig.supxlabel("Value of K",fontsize = 20)
plt.legend(fontsize = 15)<jupyter_output><empty_output><jupyter_text>Add-k Trigram<jupyter_code>addk = None
with open("data/results/addk_ablation_3.pkl", "rb") as file:
    addk = pkl.load(file)
dnames = ['brown','reuters','gutenberg']
fig,ax = plt.subplots(1,3, figsize = (24,6),sharex = True, sharey = True)

for i,d in enumerate(dnames):
    dev_perp = [addk[d][3][i]['perplexity']['dev'] for i in ks]
    train_perp = [addk[d][3][i]['perplexity']['train'] for i in ks]
    unigram_train = addk[d][1]['perplexity']['train']
    unigram_dev = addk[d][1]['perplexity']['dev']
    if i == 0:
        ax[i].set_ylabel("Perplexity", fontsize = 20)
    ax[i].semilogx(ks, train_perp, label = "Train")
    ax[i].semilogx(ks, dev_perp, label = "Dev")
#     ax[i].axhline(unigram_train, label = "Unigram train",linestyle = 'dashed',linewidth = 0.9
#             , c = 'blue')
    ax[i].axhline(unigram_dev, label = "Unigram dev",linestyle = 'dashed',linewidth = 0.9
            , c = 'orange')
    ax[i].set_title(d, fontsize = 20)
plt.tick_params(axis = 'both',labelsize = 15)

plt.suptitle("Perplexity with Add-k Smoothing for Trigram Model",fontsize = 25)
fig.supxlabel("Value of K",fontsize = 20)
plt.legend(fontsize = 15)<jupyter_output><empty_output><jupyter_text>Backoff Plots<jupyter_code>r = None
with open("data/results/backoff_ablation.pkl", "rb") as file:
    r = pkl.load(file)
import matplotlib
matplotlib.rc('xtick', labelsize=15) 
matplotlib.rc('ytick', labelsize=15) 
r
train = []
test = []
dev = []
for i in range(2,6):
    data = r[i]['backoff']
    train.append(data['perplexity']['train'])
    dev.append(data['perplexity']['dev'])
    test.append(data['perplexity']['test'])

    
add_k_best_perf = [1715.467, 384.85, 657.5382256991614]
fig,ax = plt.subplots(1,3, figsize = (24,6),sharex = True, sharey = True)

for i,d in enumerate(dnames):
    backoff_dev_perp = [r[d][i]['backoff']['perplexity']['dev'] for i in ns if i <=5]
    backoff_train_perp = [r[d][i]['backoff']['perplexity']['train'] for i in ns if i <=5]
    if i == 0:
        ax[i].set_ylabel("Perplexity", fontsize = 20)
    ax[i].plot(ns[:4], backoff_dev_perp, label = "Dev")
    ax[i].plot(ns[:4], backoff_train_perp, label = "Train")
    ax[i].axhline(add_k_best_perf[i], label = "Best add-k",linestyle = 'dashed',linewidth = 0.9
            , c = 'blue')
    ax[i].set_title(d, fontsize = 20)
plt.tick_params(axis = 'both',labelsize = 15)

plt.suptitle("Perplexity with Backoff Conditional Probabilities",fontsize = 25)
fig.supxlabel("N-gram Size",fontsize = 20)
plt.legend(fontsize = 15)<jupyter_output><empty_output><jupyter_text>Sampling information<jupyter_code># !streamlit run demo.py
import pickle as pkl
with open("data/models/brown_best.pkl", "rb") as file:
    blm = pkl.load(file)
    
with open("data/models/reuters_best.pkl", "rb") as file:
    rlm = pkl.load(file)
    
with open("data/models/gutenberg_best.pkl", "rb") as file:
    glm = pkl.load(file)

blmb = Sampler(blm)
rlmb = Sampler(rlm)
glmb = Sampler(glm)
with open("data/models/brown_uni.pkl", "rb") as file:
    blmu = pkl.load(file)
    
with open("data/models/reuters_uni.pkl", "rb") as file:
    rlmu = pkl.load(file)
    
with open("data/models/gutenberg_uni.pkl", "rb") as file:
    glmu = pkl.load(file)

blmu = Sampler(blmu)
rlmu = Sampler(rlmu)
glmu = Sampler(glmu)
def sample_sentence_pref(model, sentence):
    s = sentence.split(" ")
    return " ".join(model.sample_sentence(prefix = s))
res_dict = {}
model_dict = {}
dnames = ['brown','reuters','gutenberg']
b_models = [blmb, rlmb, glmb]
u_models = [blmu, rlmu, glmu]

for i,d in enumerate(dnames):
    model_dict[d] = {}
    model_dict[d]['best'] = b_models[i]
    model_dict[d]['unigram'] = u_models[i]
prefixes = ['I am','The contract','Who hath','It is','The forecast']
res_dict = sampling_ablation(model_dict,prefixes)
with open("data/results/sentences.pkl","wb") as file:
    pkl.dump(res_dict, file)
res_dict<jupyter_output><empty_output><jupyter_text>Analysis on Out-of-Domain Text: Perplexity<jupyter_code>sentences = {
   0: "This third quarter fiscal forecast for DOW Chemical is bearish according to financial analysts.",
   1: "Who hath such scuples as to remain untrodden by the perils of temptation.",
   2: "Hey did hear about that fight the football team got in to after practice?", 
   3: "Walked alone did she, on to tomorrow.",
    # Generated samples that score higher with model that generated them
    #Gutenberg
   4: "It was like fire sent them and new psychological influences set in stone brought the Holy Ghost",
    #Reuters
   5: "It was like the workforce body",
   6: "It was like a loss for Bahrain Oman and current residents are packing said company spokesman"
    
}

perp = perplexity_ablation(model_dict, sentences)
with open('data/results/perplexities.pkl','wb') as file:
    pkl.dump(perp, file)
perp
from tabulate import tabulate
rows = []
for s in perp:
    row = [s] 
    for d in dnames:
        row.append("{} / {}".format(round(perp[s][d]['best']),
                                 round(perp[s][d]['unigram'])))
    rows.append(row)
print(rows) 
latex_file = "data/results/perp.tex"
latex_str = tabulate(rows, headers = [""] +  dnames, tablefmt="latex") 
with open(latex_file, 'w') as f:  
    f.write(latex_str)
    f.close()<jupyter_output><empty_output><jupyter_text>Table for in-domain<jupyter_code>1513.8  &  17443.5  &    2459.17  \\
 reuters   & 7216.01 &   1466.87 &   12016.8   \\
 gutenberg & 4130.09 &  44906    &     981.369 \\
add_k_best_train = [127.561,72.4294,123.74]
add_k_best_test = [1822.19,378.492,649.978 ]
backoff_best_train = [74.6185,51.223,91.8805]
backoff_best_test = [558.59,180.312,285.342]
unigram_best_test = [1758.25,1576.85,1035.78]
unigram_best_train = [1513.8,1466.87,981.369]
id_dict = {}
trows = ['unigram','add-k','backoff']
id_dict['unigram'] = list(zip(unigram_best_train, unigram_best_test))
id_dict['add-k'] = list(zip(add_k_best_train, add_k_best_test))
id_dict['backoff'] = list(zip(backoff_best_train, backoff_best_test))
id_dict
rows = []

for t in trows:
    row = [t] + ['{} / {}'.format(round(id_dict[t][i][0]),
                                  round(id_dict[t][i][1])) for i in range(3)]
    rows.append(row)
latex_file = "data/results/in-domain.tex"
latex_str = tabulate(rows, headers = [""] +  dnames, tablefmt="latex") 
with open(latex_file, 'w') as f:  
    f.write(latex_str)
    f.close()<jupyter_output><empty_output><jupyter_text>Table for appendix sentences<jupyter_code>rows = []
for i,s in sentences.items():
    rows.append([i,s])

latex_file = "data/results/sent_appendix.tex"
latex_str = tabulate(rows, headers = ["index","sentence"], tablefmt="latex") 
with open(latex_file, 'w') as f:  
    f.write(latex_str)
    f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16512549537648613, 'fraction_numerical': 0.04568472038749449, 'mean_word_length': 4.522796352583587, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '272325', 'license': 'Apache-2.0', 'n_tokens_mistral': 3860, 'n_tokens_neox': 3565, 'n_words': 723, 'path': 'hw2/hw2_experiments.ipynb', 'repo_name': 'SamShowalter/CS272-NLP', 'size': '9091'}