<jupyter_start><jupyter_text><NAME> - A53213478Code for 250A - HW110-10-2017  Hangman<jupyter_code>__author__ = "<NAME>"
import string
import operator
from collections import defaultdict<jupyter_output><empty_output><jupyter_text>Reads data<jupyter_code>r = open('hw1_word_counts_05.txt','r')
r.seek(0)
corpus = r.readlines()<jupyter_output><empty_output><jupyter_text>Computes prior probabilities<jupyter_code>cosplit = [i.strip().split() for i in corpus]
word_count = defaultdict(int)
for i,j in cosplit:
    word_count[i]=int(j)
word_probabilities = word_count.copy()
sumcounts = sum(word_count.values())
for i in word_probabilities:
    word_probabilities[i]=word_probabilities[i]*1.0/sumcounts
print "Total #unique words : ",len(word_count),"\nTotal words : ",sumcounts
sorted_dict = sorted(word_count.items(), key=operator.itemgetter(1))
print "14 least frequent 5-letter words along with their counts:"
for i in sorted_dict[:14]:
    print i[0],i[1]
print "\n15 most frequent 5-letter words along with their counts :"
for i in sorted_dict[-15:]:
    print i[0],i[1]<jupyter_output>14 least frequent 5-letter words along with their counts:
TROUP 6
MAPCO 6
CAIXA 6
OTTIS 6
BOSAK 6
NIAID 7
YALOM 7
SERNA 7
CLEFT 7
CCAIR 7
FOAMY 7
PAXON 7
TOCOR 7
FABRI 7

15 most frequent 5-letter words along with their counts :
SIXTY 73086
THERE 86502
YEARS 88900
FORTY 94951
OTHER 106052
FIFTY 106869
FIRST 109957
AFTER 110102
WHICH 142146
THEIR 145434
ABOUT 157448
WOULD 159875
EIGHT 165764
SEVEN 178842
THREE 273077<jupyter_text>The above results do make sense, since we see that the least frequent words are mostly typos or proper nouns, and the most frequent words are numbers and pronouns, which is exactly what we can expect in a large corpus of words.  Creates place-holder for included and excluded words<jupyter_code>def create_given():
    for word in word_count :
        for i in range(5) :
            if ((correct[i]!= word[i]) and (correct[i]!=' ')):
                exclude[word] = word_count[word]
                break

    for word in word_count :
        dele = 0
        for a in range(5) :
            for b in range(len(inc[a])) :
                if inc[a][b]==word[a] :
                    exclude[word] = word_count[word]
                    dele = 1
                    break
            if dele == 1:
                dele = 0
                break
            
    for z in word_count :
        if z not in exclude :
            given[z] = word_count[z]<jupyter_output><empty_output><jupyter_text>Filtering out unwanted words for ease in probability calculation<jupyter_code>def create_incorrect():
    for x in range(len(incorrect)):
        for y in range(5) :
            if y in inc:
                inc[y].append(incorrect[x])
            else :
                inc[y] = [incorrect[x]]
    for m in range(len(correct)) :
        for n in range(5) :
            if m!=n and correct[m]!=correct[n] :
                if n in inc:
                    if correct[m]!= ' ':
                        inc[n].append(correct[m])
                else :
                    if correct[m]!= ' ':
                        inc[n] = [correct[m]]<jupyter_output><empty_output><jupyter_text>Next guess probability computation<jupyter_code>def best_next_guess():
    prob_evi_given_word = {}
    for word in word_count :
        if word in given :
            prob_evi_given_word[word] = 1
        else :
            prob_evi_given_word[word] = 0

    denominator_1 = sum((prob_evi_given_word[word] * word_probabilities[word]) for word in word_count)
    prob_word_given_evi = {}
    for word in word_count :
        prob_word_given_evi[word] = float((prob_evi_given_word[word]*word_probabilities[word])/denominator_1)

    alphabet = {'A':0, 'B':0, 'C':0, 'D':0, 'E':0, 'F':0, 'G':0, 'H':0, 'I':0, 'J':0, 'K':0, 'L':0, 'M':0, 'N':0, 'O':0, 'P':0, 'Q':0, 'R':0, 'S':0, 'T':0, 'U':0, 'V':0, 'W':0, 'X':0, 'Y':0, 'Z':0}
    for word in word_count:
        for letter in alphabet :
            if letter in word :
                alphabet[letter]+=prob_word_given_evi[word]    

    for x in correct :
        if x in alphabet :
            del alphabet[x]

    alphabet_sort = sorted(alphabet.items(), key=operator.itemgetter(1))

    print("Next Best Guess is ",alphabet_sort[-1][0])
    print("Probability of that is ",float(alphabet_sort[-1][1]))<jupyter_output><empty_output><jupyter_text>Results for 1.9(b)<jupyter_code>for i in range(0,9):
    inc = {0:[],1:[],2:[],3:[],4:[]}
    exclude,given = {},{}

    correct = []
    incorrect=[]
    while len(correct)!=5 :
        correct = raw_input ('\n Enter correct characters (Enter 5 characters and use Space to denote if character not filled till now :)')
        if len(correct)!=5 :
            print("Correct Character Length not 5, Please try again")
    incorrect = raw_input('Enter incorrect characters : ')

    create_incorrect()
    create_given()
    best_next_guess()<jupyter_output>Enter correct characters (Enter 5 characters and use Space to denote if character not filled till now :)     
Enter incorrect characters : 
('Next Best Guess is ', 'E')
('Probability of that is ', 0.5394172389647942)

 Enter correct characters (Enter 5 characters and use Space to denote if character not filled till now :)     
Enter incorrect characters : EA
('Next Best Guess is ', 'O')
('Probability of that is ', 0.5340315651557651)

 Enter correct characters (Enter 5 characters and use Space to denote if character not filled till now :)A   S
Enter incorrect characters : 
('Next Best Guess is ', 'E')
('Probability of that is ', 0.7715371621621622)

 Enter correct characters (Enter 5 characters and use Space to denote if character not filled till now :)A   S
Enter incorrect characters : I
('Next Best Guess is ', 'E')
('Probability of that is ', 0.7127008416220354)

 Enter correct characters (Enter 5 characters and use Space to denote if character not filled till now :)  O  
Enter inc[...]
{'fraction_non_alphanumeric': 0.11273521505376344, 'fraction_numerical': 0.046370967741935484, 'mean_word_length': 3.088598901098901, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '510033', 'license': 'Apache-2.0', 'n_tokens_mistral': 2160, 'n_tokens_neox': 1943, 'n_words': 601, 'path': 'CSE 250A/hw1/Code/CSE 250A - HW1.ipynb', 'repo_name': 'vnnsrk/UCSD', 'size': '5974'}