<jupyter_start><jupyter_text>Write the functions$$P(T_i|T_{i-1})$$$$P(W_i|T_i)$$<jupyter_code>pos = list(set(train.pos))
smooth = 0.00001

pos
pos_ppos = {}
pos_ppos_denom = {}
for col_name in set(train.ppos):
    sub_frame = train[train.ppos == col_name]
    counted = ctr(sub_frame.pos)
    pos_ppos[col_name] = counted
    pos_ppos_denom[col_name] = len(sub_frame)
    

# conditional 
def Ptt(Ti,Tprev):
    if Ti not in pos_ppos[Tprev]: return smooth
    return pos_ppos[Tprev][Ti] / pos_ppos_denom[Tprev]

word_pos = {}
word_pos_denom = {}
for col_name in set(train.pos):
    sub_frame = train[train.pos == col_name]
    counted = ctr(sub_frame.word)
    word_pos[col_name] = counted
    word_pos_denom[col_name] = len(sub_frame)
    

# conditional 
def Pwt(W, T):
    if W not in word_pos[T]: return smooth
    return word_pos[T][W] / word_pos_denom[T]
Pwt('the','DT')<jupyter_output><empty_output><jupyter_text>Evaluate: build the trellis- example trellis with an example sequence<jupyter_code>sequence = ['the', 'government']
emissions = []
transitions = []
for word in sequence:
    emission = [(t,Pwt(word, t)) for t in pos]
    transition = [(tprev,Ptt(ti,tprev)) for ti in pos for tprev,prob in emission]
    emissions.append(emission)
    transitions.append(transition)<jupyter_output><empty_output><jupyter_text>Evaluate: decode<jupyter_code>hyp = []

for e,t in zip(emissions, transitions):
    # argmax of e
    e = sorted(e, key = itemgetter(1))[-1]
    hyp.append(e)<jupyter_output><empty_output><jupyter_text>Evaluate: check accuracy- load test data- pass all word sequences through your hmm- decode- find the most probable sequence- compare to the "gold"<jupyter_code>hyp<jupyter_output><empty_output><jupyter_text>Methods to help with determining accuracy<jupyter_code>def emiss(word = ''):
    return [(t,Pwt(word,t)) for t in pos]

def trans(c):
    return [[t, Ptt(t,c)] for t in pos]

#New Trellis
def evaluate(word, rest):
    list = emiss(word)
    max(list)
    sTags = []
    tag = max(list, key = itemgetter(1))[0]
    sTags.append(tag)
    i = 1
    while len(rest) > i:
        tList = trans(tag)
        nWord = rest[i]
        i = i + 1
        eList = emiss(nWord)
        cList = [(t[0], t[1] * e[1]) for t, e in zip(tList, eList)]
        tag = max(cList, key = itemgetter(1))[0]
        sTags.append(tag)
    return sTags<jupyter_output><empty_output><jupyter_text>Creating the test set<jupyter_code>test = pd.read_csv('test.txt', delimiter=' ', names=['word', 'pos', 'other'])
test['word'] = test.word.str.lower()
test.describe()
test<jupyter_output><empty_output><jupyter_text>Evaluating accuracy of test set<jupyter_code>test['hyp'] = evaluate(test.word[0], test.word,)
test['hyp']
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix

accuracy_score(test.pos, test.hyp)<jupyter_output><empty_output><jupyter_text>Making the confusion matrix<jupyter_code>act = test.word
hyp = test.hyp
confusion = pd.crosstab(test.pos, hyp, rownames = ['actual'], colnames = ['predicted'], margins = True)
confusion<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14032829095590602, 'fraction_numerical': 0.006115223688445446, 'mean_word_length': 4.723756906077348, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '615286', 'license': 'AFL-3.0', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1166, 'n_words': 248, 'path': 'HW3/A3-hmm.ipynb', 'repo_name': 'Colmine/NLP_Stuff', 'size': '3107'}