<jupyter_start><jupyter_text>POS tagging using modified Viterbi  Data Preparation<jupyter_code>#Importing libraries
import nltk, re, pprint, random,time
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
from nltk.tokenize import word_tokenize


# reading the Treebank tagged sentences
nltk_data = list(nltk.corpus.treebank.tagged_sents(tagset='universal'))
# first few tagged sentences
print(nltk_data[:40])
# Splitting into train and test
random.seed(1234)
train_set, test_set = train_test_split(nltk_data,train_size=0.95, test_size=0.05, random_state=42)

print(len(train_set))
print(len(test_set))
print(train_set[:40])
# Getting list of tagged words
train_tagged_words = [tup for sent in train_set for tup in sent]
len(train_tagged_words)
# tokens 
tokens = [pair[0] for pair in train_tagged_words]
tokens[:10]
# vocabulary
V = set(tokens)
print(len(V))
# number of tags
T = set([pair[1] for pair in train_tagged_words])
print(len(T))
print(T)<jupyter_output>12
{'X', '.', 'VERB', 'ADJ', 'DET', 'NOUN', 'PRON', 'NUM', 'PRT', 'CONJ', 'ADP', 'ADV'}<jupyter_text>12 coarse tag classes as follows:  PRT - particles or other function words NUM - cardinal numbers ADP - adpositions CONJ - conjunctions NOUN - nouns X - other: foreign words, typos, abbreviations ADJ - adjectives DET - determiners VERB - verbs     PRON - pronouns . - punctuation ADV - adverbs<jupyter_code># computing P(w/t) and storing in T x V matrix
t = len(T)
v = len(V)
w_given_t = np.zeros((t, v))
# compute word given tag: Emission Probability
def word_given_tag(word, tag, train_bag = train_tagged_words):
    tag_list = [pair for pair in train_bag if pair[1]==tag]
    count_tag = len(tag_list)
    w_given_tag_list = [pair[0] for pair in tag_list if pair[0]==word]
    count_w_given_tag = len(w_given_tag_list)
    
    return (count_w_given_tag, count_tag)

print("\n", "style")
print(word_given_tag('style','NOUN'))
print(word_given_tag('style','VERB'))
print(word_given_tag('style','ADJ'))

print("\n", "reported")
print(word_given_tag('reported','NOUN'))
print(word_given_tag('reported','VERB'))
print(word_given_tag('reported','ADJ'))
# compute tag given tag: tag2(t2) given tag1 (t1), i.e. Transition Probability

def t2_given_t1(t2, t1, train_bag = train_tagged_words):
    tags = [pair[1] for pair in train_bag]
    count_t1 = len([t for t in tags if t==t1])
    count_t2_t1 = 0
    for index in range(len(tags)-1):
        if tags[index]==t1 and tags[index+1] == t2:
            count_t2_t1 += 1
    return (count_t2_t1, count_t1)
# examples
print(t2_given_t1(t2='NOUN', t1='ADJ'))
print(t2_given_t1('VERB', 'NOUN'))
print(t2_given_t1('PRT', 'NUM'))
#Please note P(tag|start) is same as P(tag|'.')
print(t2_given_t1('NOUN', '.'))
print(t2_given_t1('VERB', '.'))
print(t2_given_t1('PRT', '.'))

# creating t x t transition matrix of tags
# each column is t2, each row is t1
# thus M(i, j) represents P(tj given ti)

tags_matrix = np.zeros((len(T), len(T)), dtype='float32')
for i, t1 in enumerate(list(T)):
    for j, t2 in enumerate(list(T)): 
        tags_matrix[i, j] = t2_given_t1(t2, t1)[0]/t2_given_t1(t2, t1)[1]
tags_matrix
# convert the matrix to a df for better readability
tags_df = pd.DataFrame(tags_matrix, columns = list(T), index=list(T))
tags_df
tags_df.loc['.', :]
# heatmap of tags matrix
# T(i, j) means P(tag j given tag i)
plt.figure(figsize=(18, 12))
heat=sns.heatmap(tags_df,annot = True)
#below 2 lines are for temporary fix for matplotlib regression feature which broke heatmap  between 3.1.0 and 3.1.1
bottom, top = heat.get_ylim()
heat.set_ylim(bottom + 0.5, top - 0.5)
plt.show()

# frequent tags
# filter the df to get P(t2, t1) > 0.5
tags_frequent = tags_df[tags_df>0.5]
plt.figure(figsize=(18, 12))
tag_freq_heat=sns.heatmap(tags_frequent,annot = True)
#below 2 lines are for temporary fix for matplotlib regression feature which broke heatmap  between 3.1.0 and 3.1.1
bottom, top = tag_freq_heat.get_ylim()
tag_freq_heat.set_ylim(bottom + 0.5, top - 0.5)
plt.show()<jupyter_output><empty_output><jupyter_text>We observe that transition probabilities are between Noun and other parts of speech such as Adjective and Determinant are highest  Build the vanilla Viterbi based POS tagger<jupyter_code>len(train_tagged_words)
# Vanilla Viterbi
def Viterbi(words, train_bag = train_tagged_words):
    state = []
    T = list(set([pair[1] for pair in train_bag]))
    
    for key, word in enumerate(words):
        #initialise list of probability column for a given observation
        p = [] 
        for tag in T:
            if key == 0:
                transition_p = tags_df.loc['.', tag]
            else:
                transition_p = tags_df.loc[state[-1], tag]
                
            # compute emission and state probabilities
            emission_p = word_given_tag(words[key], tag)[0]/word_given_tag(words[key], tag)[1]
            state_probability = emission_p * transition_p    
            p.append(state_probability)
            
        pmax = max(p)
        # getting state for which probability is maximum
        state_max = T[p.index(pmax)] 
        state.append(state_max)
    return list(zip(words, state))


# Running on entire test dataset would take more than 3-4hrs. 
# Let's test our Viterbi algorithm on a few sample sentences of test dataset

random.seed(1234)

# choose random 5 sents
rndom = [random.randint(1,len(test_set)) for x in range(5)]

# list of sents
test_run = [test_set[i] for i in rndom]

# list of tagged words
test_run_base = [tup for sent in test_run for tup in sent]

# list of untagged words
test_tagged_words = [tup[0] for sent in test_run for tup in sent]
test_run
# tagging the test sentences
start = time.time()
tagged_seq = Viterbi(test_tagged_words)
end = time.time()
difference = end-start
print("Time taken in seconds: ", difference)
print("\n")
print(tagged_seq)
# accuracy
check = [i for i, j in zip(tagged_seq, test_run_base) if i == j] 
vanilla_viterbi_accuracy = len(check)/len(tagged_seq)
print('Accuracy of Vanilla Viterbi Algo:',vanilla_viterbi_accuracy)
#checking incorrectly tagged words
incorrect_tagged_cases = [[test_run_base[i-1],j] for i, j in enumerate(zip(tagged_seq, test_run_base)) if j[0]!=j[1]]
incorrect_tagged_cases<jupyter_output><empty_output><jupyter_text>The incorrect tagged cases are due to fact that when the algorithm hits an unknown word (i.e. not present in the training set hence emission probability is 0), it will assign the first tag to that word  Solve the problem of unknown words<jupyter_code>#Lets see how many unknown words we have (words that are present in test but not in train)

training_vocabulary_set = V
# building the test vocabulary to a set
test_vocabulary_set = set([t for t in test_tagged_words])

unknown_words = list(test_vocabulary_set - training_vocabulary_set)
print("Total Unknown words :", len(unknown_words))
print("\n")
print("Unknown Words :", unknown_words)<jupyter_output>Total Unknown words : 4


Unknown Words : ['pre-1917', 'low-cost', 'develops', 'copied']<jupyter_text>Modified Viterbi 1 Use transition probability of tags when emission probability is zero (in case of unknown words)<jupyter_code>def Modified_viterbi_trans_prob(words, train_bag = train_tagged_words):
    state = []
    T = list(set([pair[1] for pair in train_bag]))
    
    for key, word in enumerate(words):
        #initialise list of probability column for a given observation
        p = [] 
        p_transition =[] # list for storing transition probabilities
        for tag in T:
            if key == 0:
                transition_p = tags_df.loc['.', tag]
            else:
                transition_p = tags_df.loc[state[-1], tag]
                
            # compute emission and state probabilities
            emission_p = word_given_tag(words[key], tag)[0]/word_given_tag(words[key], tag)[1]
            state_probability = emission_p * transition_p    
            p.append(state_probability)
            p_transition.append(transition_p)
            
        pmax = max(p)
        state_max = T[p.index(pmax)] 
        
      
        # if probability is zero (unknown word) then use transition probability
        if(pmax==0):
            pmax = max(p_transition)
            state_max = T[p_transition.index(pmax)]
                           
        else:
            state_max = T[p.index(pmax)] 
        
        state.append(state_max)
    return list(zip(words, state))
# tagging the test sentences
start = time.time()
tagged_seq = Modified_viterbi_trans_prob(test_tagged_words)
end = time.time()
difference = end-start

print("Time taken in seconds: ", difference)

# accuracy
check = [i for i, j in zip(tagged_seq, test_run_base) if i == j] 
accuracy_viterbi_modified = len(check)/len(tagged_seq)
print('Modified_viterbi_trans_prob Accuracy: ',accuracy_viterbi_modified*100)<jupyter_output>Time taken in seconds:  21.624194145202637
Modified_viterbi_trans_prob Accuracy:  93.81443298969072<jupyter_text>We can see that our modified viterbi algoroth has increased accuracy of 93.81% as compared to 92.78% in Vanilla Viterbi Algorithm  Lexicons and rule based tagger<jupyter_code># Lexicon (or unigram tagger)

unigram_tagger = nltk.UnigramTagger(train_set)
accuracy_unigram_tagger = unigram_tagger.evaluate(test_set)
print("The accuracy of the Unigram Tagger is -", accuracy_unigram_tagger)<jupyter_output>The accuracy of the Unigram Tagger is - 0.911735797129939<jupyter_text>Combining unigram tagger with a rule based regex tagger<jupyter_code># patterns for tagging using a rule based tagger
patterns = [
    (r'^([0-9]|[aA-zZ])+\-([0-9]|[aA-zZ])*$','ADJ'), # words such as '10-lap','30-day','York-based'
    (r'.*able$', 'ADJ'),                     # words ending with 'able' such as 'questionable'
    (r'.*ful$', 'ADJ'),                      # words ending with 'ful' such as 'useful'
    (r'.*ous$', 'ADJ'),                      # words ending with 'ous' such as 'Previous'
    
    (r'.*\'s$', 'NOUN'),                     # possessive nouns
    (r'.*s$', 'NOUN'),                       # plural nouns
    (r'^[aA-zZ].*[0-9]+','NOUN'),            # Alpha Numeric such as Door Number, Street Number etc
    (r'.*ers$', 'NOUN'),                     # words ending with 'ers' such as 'filters','workers'
    (r'.*ment$', 'NOUN'),                    # words ending with 'ment' such as 'reinvestment' 
    (r'.*town$', 'NOUN'),                    # words ending with 'town' such as 'town','downtown'  
    (r'.*ness$', 'NOUN'),                    # words ending with 'ness' such as 'sluggishness' 
    (r'^[A-Z]+([a-z]{1,2})?\.?$','NOUN'),    # words such as 'Nov.','Mr.','Inc.'
    
    (r'[aA-zZ]+(ed|ing|es)$', 'VERB'),       # words ending with 'ed' or 'ing' or 'es'    
    (r'.*ly$', 'ADV'),                       # words ending with 'ly'
    
    (r'^[0-9]+(.[0-9]+)?$', 'NUM'),          # cardinal numbers such as 61, 1956, 9.8, 8.45, 352.7        
    (r'^(0|([*|-|$].*))','X'),               # words such as '*', '0', *-1', '*T*-1', '*ICH*-1', '*?*'   
    
    (r'(The|the|A|a|An|an|That|that|This|this|Those|those|These|these)$', 'DET'), # determinants     
    (r'.*', 'NOUN')  
    ]
    
# Rule based tagger
rule_based_tagger = nltk.RegexpTagger(patterns)

# unigram tagger backed up by the rule-based tagger
rule_based_unigram_tagger = nltk.UnigramTagger(train_set, backoff = rule_based_tagger)
accuracy_rule_based_unigram_tagger=rule_based_unigram_tagger.evaluate(test_set)
print("The accuracy of the Unigram Tagger with rule based backoff is: ",+accuracy_rule_based_unigram_tagger)
# Bigram tagger backed up by the rule-based-unigram tagger
bigram_tagger = nltk.BigramTagger(train_set, backoff = rule_based_unigram_tagger)
accuracy_bigram_tagger=bigram_tagger.evaluate(test_set)
print("The accuracy of the Bigram Tagger with rule based unigram backoff is: ",+accuracy_bigram_tagger)
#trigram tagger
trigram_tagger = nltk.TrigramTagger(train_set, backoff = bigram_tagger)
accuracy_trigram_tagger=trigram_tagger.evaluate(test_set)
print("The accuracy of the trigram Tagger with Bigram tagger as backoff is: ",+accuracy_trigram_tagger)<jupyter_output>The accuracy of the trigram Tagger with Bigram tagger as backoff is:  0.9630430509140947<jupyter_text>Lets modify Viterbi Algo to include Trigram tagger as backoff<jupyter_code># A trigram tagger backed off by a rule based tagger.

def trigram_tagger(word, train_set = train_set):
    
    # specify patterns for tagging. I have identified most of the patterns from the first 100 sentences in universal dataset
    patterns = [
    (r'^([0-9]|[aA-zZ])+\-([0-9]|[aA-zZ])*$','ADJ'), # words such as '10-lap','30-day','York-based'
    (r'.*able$', 'ADJ'),                     # words ending with 'able' such as 'questionable'
    (r'.*ful$', 'ADJ'),                      # words ending with 'ful' such as 'useful'
    (r'.*ous$', 'ADJ'),                      # words ending with 'ous' such as 'Previous'
    
    (r'.*\'s$', 'NOUN'),                     # possessive nouns
    (r'.*s$', 'NOUN'),                       # plural nouns
    (r'^[aA-zZ].*[0-9]+','NOUN'),            # Alpha Numeric such as Door Number, Street Number etc
    (r'.*ers$', 'NOUN'),                     # words ending with 'ers' such as 'filters','workers'
    (r'.*ment$', 'NOUN'),                    # words ending with 'ment' such as 'reinvestment' 
    (r'.*town$', 'NOUN'),                    # words ending with 'town' such as 'town','downtown'  
    (r'.*ness$', 'NOUN'),                    # words ending with 'ness' such as 'sluggishness' 
    (r'^[A-Z]+([a-z]{1,2})?\.?$','NOUN'),    # words such as 'Nov.','Mr.','Inc.'
    
    (r'[aA-zZ]+(ed|ing|es)$', 'VERB'),       # words ending with 'ed' or 'ing' or 'es'    
    (r'.*ly$', 'ADV'),                       # words ending with 'ly'
    
    (r'^[0-9]+(.[0-9]+)?$', 'NUM'),          # cardinal numbers such as 61, 1956, 9.8, 8.45, 352.7        
    (r'^(0|([*|-|$].*))','X'),               # words such as '*', '0', *-1', '*T*-1', '*ICH*-1', '*?*'   
    
    (r'(The|the|A|a|An|an|That|that|This|this|Those|those|These|these)$', 'DET'), # determinants     
    (r'.*', 'NOUN')  
    ]
    
    rule_based_tagger = nltk.RegexpTagger(patterns)

    # trigram backed up by the regex tagger
    trigram_regex_tagger = nltk.TrigramTagger(train_set, backoff = rule_based_tagger)
    return trigram_regex_tagger.tag_sents([[(word)]])    
# Modified Viterbi Algo- Technique II - Backoff trigram rule based tagger
def Viterbi_tri_backoff(words, train_bag = train_tagged_words):
    state = []
    T = sorted(list(set([pair[1] for pair in train_bag])))
    V = [i[0] for i in train_bag]
    
    # use the trigram tagger backed up by the rule based tagger for unknown words.
    for key, word in enumerate(words):
        if word not in V:
            unknown_word_tag = trigram_tagger(word)
            for sent in unknown_word_tag:
                for tup in sent:
                    state.append(tup[1])
        else:            
            p = [] 
            for tag in T:
                if key == 0:
                    transition_p = tags_df.loc['.', tag]
                else:
                    transition_p = tags_df.loc[state[-1], tag]
                
                # compute emission and state probabilities
                emission_p = word_given_tag(words[key], tag)[0]/word_given_tag(words[key], tag)[1]
                state_probability = emission_p * transition_p    
                p.append(state_probability)
            
            pmax = max(p)
            # getting state for which probability is maximum
            state_max = T[p.index(pmax)] 
            state.append(state_max)
    return list(zip(words, state))<jupyter_output><empty_output><jupyter_text>Evaluating tagging accuracy<jupyter_code># tagging the test sentences.
start = time.time()
viterbi_trigram_tagged_seq = Viterbi_tri_backoff(test_tagged_words)
end = time.time()
difference = end-start

print("Time taken in seconds: ", difference)

# accuracy
viterbi_trigram_word_check = [i for i, j in zip(viterbi_trigram_tagged_seq, test_run_base) if i == j]

viterbi_trigram_rule_accuracy = len(viterbi_trigram_word_check)/len(viterbi_trigram_tagged_seq)
print("The accuracy of Viterbi Algo with rule based Trigram tagger as backoff is:: ",+viterbi_trigram_rule_accuracy)
viterbi_trigram_incorrect_tagged_cases = [j for i, j in enumerate(zip(viterbi_trigram_tagged_seq, test_run_base)) if j[0]!=j[1]]
viterbi_trigram_incorrect_tagged_cases<jupyter_output><empty_output><jupyter_text>Compare the tagging accuracies of the modifications with the vanilla Viterbi algorithm<jupyter_code>evalute_data = [['Vanilla Viterbi', vanilla_viterbi_accuracy * 100], 
                  ['Vanilla Viterbi Modified', accuracy_viterbi_modified * 100], 
                  ['Unigram Tagger', accuracy_unigram_tagger * 100],
                  ['Unigram + RegexpTagger', accuracy_rule_based_unigram_tagger * 100],
                  ['Bigram Tagger + Unigram_tagger', accuracy_bigram_tagger*100],
                  ['Trigram Tagger + Bigram_tagger', accuracy_trigram_tagger*100],
                  ['Viterbi + Trigram_tagger', viterbi_trigram_rule_accuracy * 100]]

evalute_data_acccuracy_df = pd.DataFrame(evalute_data, columns = ['Algorithm', 'Accuracy'])
evalute_data_acccuracy_df.reset_index()
evalute_data_acccuracy_df
fig, ax1 = plt.subplots(figsize=(14,8),dpi=80)
bar = sns.barplot(x='Algorithm', y='Accuracy', data = evalute_data_acccuracy_df)
bar.set_xticklabels(bar.get_xticklabels(), rotation=45)<jupyter_output><empty_output><jupyter_text>List down cases which were incorrectly tagged by original POS tagger and got corrected by your modifications<jupyter_code>#function to compare taggers
def compareAlgos(sentence):
    words = word_tokenize(sentence)
    tagged_seq = Viterbi(words)
    print("\033[1m" + 'Vanilla Viterbi Tagged Sentence:' + "\033[0m")
    print(tagged_seq)
    print('\n')

    tagged_seq_modified1 = Modified_viterbi_trans_prob(words)
    print("\033[1m" + 'Modified Veterbi techniques 1 Sentence:' + "\033[0m")
    print(tagged_seq_modified1)
    print('\n')
    tagged_seq_modified2 = Viterbi_tri_backoff(words)
    print("\033[1m" + 'Modified Veterbi techniques 2 (trigram backoff tagged) Sentence:' + "\033[0m")
    print(tagged_seq_modified2)

    
compareAlgos("Android is a mobile operating system developed by Google.")<jupyter_output>[1mVanilla Viterbi Tagged Sentence:[0m
[('Android', 'X'), ('is', 'VERB'), ('a', 'DET'), ('mobile', 'ADJ'), ('operating', 'NOUN'), ('system', 'NOUN'), ('developed', 'VERB'), ('by', 'ADP'), ('Google', 'X'), ('.', '.')]


[1mModified Veterbi techniques 1 Sentence:[0m
[('Android', 'NOUN'), ('is', 'VERB'), ('a', 'DET'), ('mobile', 'ADJ'), ('operating', 'NOUN'), ('system', 'NOUN'), ('developed', 'VERB'), ('by', 'ADP'), ('Google', 'DET'), ('.', '.')]


[1mModified Veterbi techniques 2 (trigram backoff tagged) Sentence:[0m
[('Android', 'NOUN'), ('is', 'VERB'), ('a', 'DET'), ('mobile', 'ADJ'), ('operating', 'NOUN'), ('system', 'NOUN'), ('developed', 'VERB'), ('by', 'ADP'), ('Google', 'NOUN'), ('.', '.')]<jupyter_text>Android was tagged as X initally which got correctly recognized as NOUN later<jupyter_code>compareAlgos("Google and Twitter made a deal in 2015 that gave Google access to Twitter's firehose.")<jupyter_output>[1mVanilla Viterbi Tagged Sentence:[0m
[('Google', 'X'), ('and', 'CONJ'), ('Twitter', 'X'), ('made', 'VERB'), ('a', 'DET'), ('deal', 'NOUN'), ('in', 'ADP'), ('2015', 'X'), ('that', 'ADP'), ('gave', 'VERB'), ('Google', 'X'), ('access', 'NOUN'), ('to', 'PRT'), ('Twitter', 'X'), ("'s", 'PRT'), ('firehose', 'X'), ('.', '.')]


[1mModified Veterbi techniques 1 Sentence:[0m
[('Google', 'NOUN'), ('and', 'CONJ'), ('Twitter', 'NOUN'), ('made', 'VERB'), ('a', 'DET'), ('deal', 'NOUN'), ('in', 'ADP'), ('2015', 'DET'), ('that', 'ADP'), ('gave', 'VERB'), ('Google', 'X'), ('access', 'NOUN'), ('to', 'PRT'), ('Twitter', 'VERB'), ("'s", 'PRT'), ('firehose', 'VERB'), ('.', '.')]


[1mModified Veterbi techniques 2 (trigram backoff tagged) Sentence:[0m
[('Google', 'NOUN'), ('and', 'CONJ'), ('Twitter', 'NOUN'), ('made', 'VERB'), ('a', 'DET'), ('deal', 'NOUN'), ('in', 'ADP'), ('2015', 'NUM'), ('that', 'ADP'), ('gave', 'VERB'), ('Google', 'NOUN'), ('access', 'NOUN'), ('to', 'PRT'), ('Twitter', 'NOUN'), [...]<jupyter_text>Words like Google,Twitter,firehose which were tagged as X initially are getting correctly identified as Noun in the trigram approach<jupyter_code>compareAlgos("The 2018 FIFA World Cup is the 21st FIFA World Cup, an international football tournament contested once every four years.")<jupyter_output>[1mVanilla Viterbi Tagged Sentence:[0m
[('The', 'DET'), ('2018', 'X'), ('FIFA', 'X'), ('World', 'NOUN'), ('Cup', 'X'), ('is', 'VERB'), ('the', 'DET'), ('21st', 'X'), ('FIFA', 'X'), ('World', 'NOUN'), ('Cup', 'X'), (',', '.'), ('an', 'DET'), ('international', 'ADJ'), ('football', 'NOUN'), ('tournament', 'X'), ('contested', 'X'), ('once', 'ADV'), ('every', 'DET'), ('four', 'NUM'), ('years', 'NOUN'), ('.', '.')]


[1mModified Veterbi techniques 1 Sentence:[0m
[('The', 'DET'), ('2018', 'NOUN'), ('FIFA', 'NOUN'), ('World', 'NOUN'), ('Cup', 'NOUN'), ('is', 'VERB'), ('the', 'DET'), ('21st', 'NOUN'), ('FIFA', 'NOUN'), ('World', 'NOUN'), ('Cup', 'NOUN'), (',', '.'), ('an', 'DET'), ('international', 'ADJ'), ('football', 'NOUN'), ('tournament', 'NOUN'), ('contested', 'NOUN'), ('once', 'ADV'), ('every', 'DET'), ('four', 'NUM'), ('years', 'NOUN'), ('.', '.')]


[1mModified Veterbi techniques 2 (trigram backoff tagged) Sentence:[0m
[('The', 'DET'), ('2018', 'NUM'), ('FIFA', 'NOUN'), ('World', [...]<jupyter_text>FIFA, CUP got corrected to NOUN. 'contested' was tagged as X initially and was tagged and VERB correctly later.<jupyter_code>compareAlgos("NASA invited social media users to experience the launch of ICESAT-2 Satellite.")<jupyter_output>[1mVanilla Viterbi Tagged Sentence:[0m
[('NASA', 'X'), ('invited', 'X'), ('social', 'ADJ'), ('media', 'NOUN'), ('users', 'NOUN'), ('to', 'PRT'), ('experience', 'NOUN'), ('the', 'DET'), ('launch', 'NOUN'), ('of', 'ADP'), ('ICESAT-2', 'X'), ('Satellite', 'X'), ('.', '.')]


[1mModified Veterbi techniques 1 Sentence:[0m
[('NASA', 'NOUN'), ('invited', 'NOUN'), ('social', 'ADJ'), ('media', 'NOUN'), ('users', 'NOUN'), ('to', 'PRT'), ('experience', 'NOUN'), ('the', 'DET'), ('launch', 'NOUN'), ('of', 'ADP'), ('ICESAT-2', 'DET'), ('Satellite', 'NOUN'), ('.', '.')]


[1mModified Veterbi techniques 2 (trigram backoff tagged) Sentence:[0m
[('NASA', 'NOUN'), ('invited', 'VERB'), ('social', 'ADJ'), ('media', 'NOUN'), ('users', 'NOUN'), ('to', 'PRT'), ('experience', 'NOUN'), ('the', 'DET'), ('launch', 'NOUN'), ('of', 'ADP'), ('ICESAT-2', 'ADJ'), ('Satellite', 'NOUN'), ('.', '.')]
{'fraction_non_alphanumeric': 0.16600341102899374, 'fraction_numerical': 0.019329164297896533, 'mean_word_length': 3.8863247863247863, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '585105', 'license': 'MIT', 'n_tokens_mistral': 8521, 'n_tokens_neox': 7874, 'n_words': 2098, 'path': 'HMM+-based+POS+tagging-+Assignment.ipynb', 'repo_name': 'aniketbangar/NLP-HMMs-and-Viterbi-algorithm-for-POS-tagging', 'size': '22867'}