<jupyter_start><jupyter_text>Homework 5  Homework 5.1 A bigram is the combination of words.  Find the 10 most common bigrams from the text.  Order counts in the bigram combination for example "in the" is not the same bigram as "the in"<jupyter_code>%%writefile code\orderedbigram.py
import mrjob
from mrjob.job import MRJob
from mrjob.step import MRStep
import re
from itertools import islice, izip
import itertools
import heapq

WORD_RE = re.compile(r'[a-zA-Z]+')
TOPN = 10
class BigramCount(MRJob):
    def steps(self):
        return [
            MRStep(mapper=self.mapper_get_words,
                   combiner=self.combiner_count_words,
                   reducer=self.reducer_count_words),
            MRStep(mapper=self.topN_mapper,
                   reducer=self.topN_reducer)
        ]
    
    def mapper_get_words(self, _, line):
        words = WORD_RE.findall(line)
        for i in izip(words, islice(words, 1, None)):
            bigram =str(i[0]+" " +i[1])
            yield bigram, 1
    def combiner_count_words(self, bigram, counts):
        yield bigram, sum(counts)
    def reducer_count_words(self, bigram, counts):
        yield bigram,sum(counts)
    def topN_mapper(self, bigram, counts):
        yield "Top " + str(TOPN), (counts, bigram)
    def topN_reducer(self, _, countsAndBigrams):
        for countAndbigram in heapq.nlargest(TOPN, countsAndBigrams):
            yield _, countAndbigram
        



if __name__ == '__main__':
    BigramCount.run()
    
    
%run code/orderedbigram.py data/bible+shakes.nopunc.txt<jupyter_output>No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
Running step 1 of 2...
Running step 1 of 2...
Running step 1 of 2...
Creating temp directory c:\users\akhila~1\appdata\local\temp\orderedbigram.Akhila Sharon.20181005.171322.996000
Creating temp directory c:\users\akhila~1\appdata\local\temp\orderedbigram.Akhila Sharon.20181005.171322.996000
Creating temp directory c:\users\akhila~1\appdata\local\temp\orderedbigram.Akhila Sharon.20181005.171322.996000
Running step 2 of 2...
Running step 2 of 2...
Running step 2 of 2...
job output is in c:\users\akhila~1\appdata\local\temp\orderedbigram.Akhila Sharon.20181005.171322.996000\output
job output is in c:\users\akhila~1\appdata\local\temp\orderedbigram.Akhila Sharon.20181005.171322.996000\output
job output is in c:\users\[...]<jupyter_text>Homework 5.2Now do the same analysis but make the word order not count "in the" == "the in".  Find the 10 most common ordered bigrams from the alice text.<jupyter_code>%%writefile code\unorderedbigram.py
import mrjob
from mrjob.job import MRJob
from mrjob.step import MRStep
import re
from itertools import islice, izip
import itertools
import heapq

WORD_RE = re.compile(r'[a-zA-Z]+')
TOPN = 10
class BigramCount(MRJob):
    def steps(self):
        return [
            MRStep(mapper=self.mapper_get_words,
                   combiner=self.combiner_count_words,
                   reducer=self.reducer_count_words),
            MRStep(mapper=self.topN_mapper,
                   reducer=self.topN_reducer)
        ]
    
    def mapper_get_words(self, _, line):
        words = WORD_RE.findall(line)
        for i in izip(words, islice(words, 1, None)):
            bigram=(i[0],i[1])
            s_bigram=sorted(bigram)
            yield s_bigram,1
    def combiner_count_words(self, bigram, counts):
        yield bigram, sum(counts)
    def reducer_count_words(self, bigram, counts):
        yield bigram,sum(counts)
    def topN_mapper(self, bigram, counts):
        yield "Top " + str(TOPN), (counts, bigram)
    def topN_reducer(self, _, countsAndBigrams):
        for countAndbigram in heapq.nlargest(TOPN, countsAndBigrams):
            yield _, countAndbigram
        



if __name__ == '__main__':
    BigramCount.run()
    
    
%run code/unorderedbigram.py data/bible+shakes.nopunc.txt<jupyter_output>No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
Running step 1 of 2...
Running step 1 of 2...
Running step 1 of 2...
Running step 1 of 2...
Creating temp directory c:\users\akhila~1\appdata\local\temp\unorderedbigram.Akhila Sharon.20181005.171429.603000
Creating temp directory c:\users\akhila~1\appdata\local\temp\unorderedbigram.Akhila Sharon.20181005.171429.603000
Creating temp directory c:\users\akhila~1\appdata\local\temp\unorderedbigram.Akhila Sharon.20181005.171429.603000
Creating temp directory c:\users\akhila~1\appdata\local\temp\unorderedbigram.Akhila Sharon.20181005.171429.603000
Running step 2 of 2...
Running step 2 of 2...
Running step 2 of 2...
Running step 2 [...]<jupyter_text>Homework 5.3A trigram are three word combintation.  Find the 10 most common ordered trigrams from the alice text.  Make it so that the order of the words do not count in the trigram combination for example "in the air" is the same trigram as "the air in" or "air in the"...<jupyter_code>%%writefile code\unorderedtrigram.py
import mrjob
from mrjob.job import MRJob
from mrjob.step import MRStep
import re
from itertools import islice, izip
import itertools
import heapq

WORD_RE = re.compile(r'[a-zA-Z]+')
TOPN = 10
class BigramCount(MRJob):
    def steps(self):
        return [
            MRStep(mapper=self.mapper_get_words,
                   combiner=self.combiner_count_words,
                   reducer=self.reducer_count_words),
            MRStep(mapper=self.topN_mapper,
                   reducer=self.topN_reducer)
        ]
    
    def mapper_get_words(self, _, line):
        words = WORD_RE.findall(line)
        for i in izip(words, islice(words, 1, None),islice(words,2,None)):
            bigram=(i[0],i[1],i[2])
            s_bigram=sorted(bigram)
            yield s_bigram,1
    def combiner_count_words(self, bigram, counts):
        yield bigram, sum(counts)
    def reducer_count_words(self, bigram, counts):
        yield bigram,sum(counts)
    def topN_mapper(self, bigram, counts):
        yield "Top " + str(TOPN), (counts, bigram)
    def topN_reducer(self, _, countsAndBigrams):
        for countAndbigram in heapq.nlargest(TOPN, countsAndBigrams):
            yield _, countAndbigram
        



if __name__ == '__main__':
    BigramCount.run()
    
    
%run code/unorderedtrigram.py data/bible+shakes.nopunc.txt<jupyter_output>No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
Running step 1 of 2...
Running step 1 of 2...
Running step 1 of 2...
Running step 1 of 2...
Running step 1 of 2...
Running step 1 of 2...
Creating temp directory c:\users\akhila~1\appdata\local\temp\unorderedtrigram.Akhila Sharon.20181005.171710.912000
Creating temp directory c:\users\akhila~1\appdata\local\temp\unorderedtrigram.Akhila Sharon.20181005.171710.912000
Creating temp directory c:\users\akhila~1\appdata\local\temp\unorderedtrigram.Ak[...]<jupyter_text>Homework 5.4Create graphs to explain the relationship of the frequency of monograms ( words ) to bigrams and trigam frequencies<jupyter_code>wordfile=pd.read_csv("./data/wordcountfreq.csv")
bigramfiletwo=pd.read_csv("./data/bigramcounttwo.csv")
trigramfile=pd.read_csv("./data/trigramcount.csv")<jupyter_output><empty_output><jupyter_text>Show the relationship between frequency of monogram to frequency of each word in a bigram<jupyter_code>fig_size = plt.rcParams["figure.figsize"]
fig_size[0] = 13
fig_size[1] = 4
plt.rcParams["figure.figsize"] = fig_size

plt.bar(bigramfiletwo.bigramone,bigramfiletwo.frequency,label="bigramone",color="orange")
plt.bar(wordfile.word,wordfile.frequency,label="word",color="green")
plt.bar(bigramfiletwo.bigramtwo,bigramfiletwo.frequency,label="bigramtwo",color="orange")
#green represents monogram
#orange represnts words in bigram<jupyter_output><empty_output><jupyter_text>Show the relationship between frequency of monogram to frequency of each word in a trigram<jupyter_code>plt.bar(trigramfile.trigramone,trigramfile.frequency,label="trigramone",color="blue")
plt.bar(trigramfile.trigramtwo,trigramfile.frequency,label="trigramtwo",color="blue")
plt.bar(trigramfile.trigramthree,trigramfile.frequency,label="trigramthree",color="blue")
plt.bar(wordfile.word,wordfile.frequency,label="word",color="green")
#green represents monogram
#blue represnts words in bigram
#it is observed that as the number of n increases in the n-gram,
#m its frequency relative to the monogram is very less.<jupyter_output><empty_output><jupyter_text>For greater than a score of 3   Create a TD - IDF implementation and  Analyze the following Sherlock Holmes book from Project Gutenberg text versions of :The Adventures of Sherlock Holmes-  http://www.gutenberg.org/ebooks/1661.txt.utf-8A Study in Scarlet - http://www.gutenberg.org/files/244/244-0.txtThe Hound of the Baskervilles - http://www.gutenberg.org/files/2852/2852-0.txtThe Return of Sherlock Holmes - http://www.gutenberg.org/files/108/108-0.txtThe Sign of the Four - http://www.gutenberg.org/ebooks/2097.txt.utf-8 Display the scores for the top 20 highest frequencty terms and the relationship to the books<jupyter_code>#named books as 01.txt,02.txt.. so on
#was'nt able to get the exact result , but got the top 20 highest frequency in each book.
# used https://github.com/Jasonnor/tf-idf-python/blob/master/src/tf_idf.py as reference code
import re

import os
from math import log
from operator import itemgetter


class tf_idf:
   
    global list_words
    list_words=[]
 
    def __init__(self):
        self.files = {}
        self.corpus = {}
        self.stop_words = set(())
        content = open('stopwords.txt', 'rb').read().decode('utf-8')
        for line in content.splitlines():
            self.stop_words.add(line)
            
    def add_file(self, file_name):
        # Load data and cut
        content = open(file_name, 'rb').read() if file_name[0] == '/' or file_name[0] == 'C' else open('./data/' + file_name, 'rb').read()
        words = re.findall('[a-z]+', content.lower())
         # Build dictionary
        dictionary = {}
        for w in words:
            if len(w.strip()) < 2 or w.lower() in self.stop_words:
                continue
            dictionary[w] = dictionary.get(w, 0.0) + 1.0
            self.corpus[w] = self.corpus.get(w, 0.0) + 1.0

        # Get term frequency
        total = sum(dictionary.values())
        for k in dictionary:
            dictionary[k] /= total

        # Add tf to the corpus
        self.files[file_name] = dictionary
        
    def get_tf_idf(self, file_name, top_k):
        # Get inverse document frequency
        tf_idf_of_file = {}
        for w in self.corpus.keys():
            w_in_f = 1.0
            for f in self.files:
                if w in self.files[f]:
                    w_in_f += 1.0
            # Get tf-idf
            if w in self.files[file_name]:
                tf_idf_of_file[w] = log(len(self.files) / w_in_f) * self.files[file_name][w]
               
                
        # Top-K result of tf-idf
        tags = sorted(tf_idf_of_file.items(), key=itemgetter(1), reverse=True)
       
        return tags[:top_k]

       
if __name__ == "__main__":
    table = tf_idf()
    folder_name = 'Books'
    folder = os.path.join('./data/' ,folder_name)
    path, dirs, files = next(os.walk("./data/Books"))
    num_of_files = len(files)
    
    for x in range(1, num_of_files):
        file_name = folder_name + '/' + str(x).zfill(2) + '.txt'
        table.add_file(file_name)

    top_k = 20
    for x in range(1, num_of_files):
        target_file = folder_name + '/' + str(x).zfill(2) + '.txt'
        my_data=[]
        
        print('Top ' + str(top_k) + ' of tf-idf in ' + target_file + ' : ')
        print(table.get_tf_idf(target_file, top_k))
        print("\n")<jupyter_output>Top 20 of tf-idf in Books/01.txt : 
[('simon', 0.00027027233243064595), ('rucastle', 0.0002567587158091136), ('mccarthy', 0.0002500019074983475), ('coronet', 0.00018919063270145215), ('goose', 0.000182433824390686), ('clair', 0.000182433824390686), ('hosmer', 0.0001554065911476214), ('turner', 0.00014864978283685526), ('stepfather', 0.0001418929745260891), ('roylott', 0.0001418929745260891), ('windibank', 0.00013513616621532297), ('neville', 0.0001283793579045568), ('stoner', 0.0001283793579045568), ('hatherley', 0.00012162254959379067), ('geese', 0.00011486574128302451), ('irene', 0.00011486574128302451), ('boscombe', 0.00010810893297225837), ('openshaw', 0.00010810893297225837), ('adler', 0.00010810893297225837), ('toller', 0.00010135212466149222)]


Top 20 of tf-idf in Books/02.txt : 
[('drebber', 0.0009515771045285108), ('ferrier', 0.0009362290867135349), ('stangerson', 0.0007213568373038711), ('gregson', 0.0006906608016739191), ('jefferson', 0.0005678766591541113), ('footnote', 0.[...]
{'fraction_non_alphanumeric': 0.10868294109048385, 'fraction_numerical': 0.06534132436778249, 'mean_word_length': 3.4599087353324642, 'pattern_counts': {'":': 1, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '288332', 'license': 'MIT', 'n_tokens_mistral': 5180, 'n_tokens_neox': 4521, 'n_words': 1161, 'path': 'Homeworks/Homework5/VALLABHANENI-AKHILA-HW5-mapreduce-8-OCTOBER-2018.ipynb', 'repo_name': 'akhilaSharon/big-data-python-class', 'size': '13682'}