<jupyter_start><jupyter_text>Add error-correction based on model eval results<jupyter_code>from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
%load_ext autoreload
%autoreload 2
%matplotlib inline
import matplotlib.pylab as plt
import sys
sys.path.append('/home/catskills/Desktop/openasr20/end2end_asr_pytorch')
import os
os.environ['IN_JUPYTER']='True'
from glob import glob
import json
import numpy as np
language='amharic'
stage='NIST'
with open(f'analysis/{language}/eval.json', 'r') as f:
    R=json.load(f)
R[0:2]
len(R)
E=[x for x in R if x[-1] != 'SUCCESS']
len(E)
len(E)/len(R)
from tokenizer import tokenizer
R1=[(tokenizer(gold), tokenizer(pred), success=='SUCCESS') for x,y,gold,pred,success in R]
len(R1), len(R)
R1[0]
vocab=[]
for gold,pred,x in R1:
    vocab.extend(gold)
    vocab.extend(pred)
vocab=list(set(vocab))
len(vocab)
vocab[0:10]
M={}
for gold,pred,x in R1:
    for y in gold:
        M[y]=y
len(M)
for gold,pred,x in R1:
    if len(gold)==len(pred):
        for wgold,wpred in zip(gold,pred):
            if wpred not in M:
                M[wpred]=wgold
len(M)
Z=[x for x in vocab if x not in M]
len(Z)
Z[0:10]
for x in Z:
    M[x]=''
len(M)
from error_correct import error_correct
# Do no harm
for gold,pred,success in R1:
    if not success:
        continue
    repred=error_correct(pred, M)
    if repred != pred:
        print('whoops', repred)
        break
# First pass
fixed=[]
harder=[]
for gold,pred,success in R1:
    repred=error_correct(pred, M)
    if repred == gold:
        fixed.append((gold,pred))
    else:
        harder.append((gold,pred,repred))
len(fixed), len(harder), len(R1)
len(R)
len(E)-len(harder)
with open(f'analysis/{language}/afterburner.json', 'w') as f:
    json.dump(M, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1512834516657564, 'fraction_numerical': 0.014199890770071, 'mean_word_length': 4.189801699716714, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '389846', 'license': 'Apache-2.0', 'n_tokens_mistral': 730, 'n_tokens_neox': 726, 'n_words': 182, 'path': 'notebooks/afterburner.ipynb', 'repo_name': 'catskillsresearch/openasr20', 'size': '1831'}