<jupyter_start><jupyter_text>greedy decoding<jupyter_code>def ctc_best_path(out,labels):
    "implements best path decoding as shown by Graves"
    out = [labels[i] for i in np.argmax(out, axis=1) if i!=labels[-1]]
    o = ""
    for i,j in groupby(out):
        o = o + i
    return o.replace("_","")<jupyter_output><empty_output><jupyter_text>beam search decoding<jupyter_code>lm_w = kenlm.LanguageModel('/home/hemant/asr_wm/data/ner/pp_without_misc/without_space/4_gram.arpa')
def sort_beam(ptot,k):
    if len(ptot) < k:
        return [i for i in ptot.keys()]
    else:
        dict_ = sorted(dict((v,k) for k,v in ptot.items()).items(),reverse=True)[:k]
        return [i[1] for i in dict_]

    NEG_INF = -float("inf")
def make_new_beam():
    fn = lambda : (NEG_INF, NEG_INF)
    return defaultdict(fn)
def ctc_beam_search(out,labels, prune=0.0001, k=20,alpha=0.01,beta=0,lm=None):
    
    bc_i = 0 # blank/special charatcter index 
    F = out.shape[1]
    dummy_ = np.vstack((np.zeros(F), out))
    out = np.log(np.vstack((np.zeros(F), out)))
    steps = out.shape[0]
#     print(out)
    pb, pnb = defaultdict(Counter), defaultdict(Counter)
    pb[0][''], pnb[0][''] = 0, -float("inf")
    prev_beams = ['']
    for t in tqdm(range(1,steps)):
        pruned_alphabet = [labels[i] for i in np.where(dummy_[t] > prune)[0]]
        for b in prev_beams:
            for c_t in pruned_alphabet:
                index = labels.index(c_t)
                if c_t == "_": #Extending with a blank
                    if pb[t][b] == 0:
                        pb[t][b] = -float('inf')
                    pb[t][b] = lse(pb[t][b],out[t][index]+pb[t-1][b], out[t][index]+pnb[t-1][b])
                else:
                    i_plus = b + c_t
                    if len(b.replace(' ', '')) > 0 and c_t == b[-1]: #Extending with the same character as the last one
                        pnb[t][b] = lse(pnb[t][b],out[t][index]+pnb[t-1][b])
                        pnb[t][i_plus] = lse(pnb[t][i_plus],out[t][index]+pb[t-1][b])
                    else:
                        pnb[t][i_plus] = lse(out[t][index]+pb[t-1][b], out[t][index]+pnb[t-1][b])
                    if i_plus not in prev_beams:
                        pb[t][i_plus] = lse(pb[t][i_plus],out[t][labels.index("_")]+pb[t - 1][i_plus], out[t][labels.index("_")]+ pnb[t - 1][i_plus])
                        pnb[t][i_plus] = lse(pnb[t][i_plus],out[t][index] + pnb[t - 1][i_plus])


        keys = set(pb[t].keys()).union(set(pnb[t].keys()))
        ptot = {i:pb[t][i]+pnb[t][i] for i in keys }
        print(ptot)
#         break
    
        prev_beams = sort_beam(ptot,k)
        
    return prev_beams[0]

ctc_beam_search(dev[0][0],labels,0.01,k=1,lm=lm_w)
dev[0][1]
def error(dummy):
    total_cer, total_wer, num_tokens, num_chars = 0, 0, 0, 0
    for i in tqdm(dummy):
        transcript, reference = i[0], i[1]
        wer_inst = wer_(transcript,reference)
        total_wer += wer_inst
        num_tokens += len(reference.split(' '))\


    wer = (float(total_wer) / num_tokens)*100
    print('Test Summary \t'
        'Average WER {wer:.3f}\t'.format(wer=wer))
def greedy(dummy,p=False):
    d = []
    for i in tqdm(dummy):
        out, reference = i[0], i[1]
        gred_txt = ctc_best_path(out,labels)
        if p: 
            print('transcript: '.upper(),gred_txt)
            print('reference: '.upper(),reference,'\n')
        d.append([gred_txt,reference])
    error(d)
def beam(dummy,labels,prune,beam_width,p=False):
    d = []
    for i in tqdm(dummy):
        out, reference = i[0], i[1]
        beam_txt = ctc_beam_search(out,labels,prune,beam_width)
        if p: 
            print('transcript: '.upper(),beam_txt)
            print('reference: '.upper(),reference,'\n')
        d.append([beam_txt,reference])
    error(d)
pr = True
dev = np.load('/home/hemant/asr_wm/data/ner/pp_without_misc/without_space/dev.npy',allow_pickle=True)
greedy(dev[:1],pr)
beam(dev[:1],labels,0.001,10,pr)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17773339990014977, 'fraction_numerical': 0.016475287069395907, 'mean_word_length': 2.5523049645390072, 'pattern_counts': {'":': 1, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '194819', 'license': 'Apache-2.0', 'n_tokens_mistral': 1609, 'n_tokens_neox': 1531, 'n_words': 270, 'path': '.ipynb_checkpoints/test-checkpoint.ipynb', 'repo_name': 'raotnameh/ctc', 'size': '4006'}