<jupyter_start><jupyter_text>POS Distribution Chisheng LiTokenize the SOTU speech collections using the NLTK package and then write a program that compute the frequency of words and POS. Compare the speeches based on the following properties:- vocabulary size (number of unique words);- frequency of stopwords;- number of capital letters;- average number of characters per word;- number of nouns, adjectives, verbs, adverbs, and pronouns;- the top 10 nouns, top 10 verbs, and top 10 adjectives.<jupyter_code>import csv, re
from nltk.tokenize import word_tokenize
from nltk.tag import pos_tag
from itertools import groupby
from math import log<jupyter_output><empty_output><jupyter_text>POS Distribution for the entire SOTU speech<jupyter_code>def pos_stats(filename):
    pattern = re.compile("[A-Z]{1}")
    sum_stats= dict();
    
    with open ('stopwords.txt') as stop:
        stop_words = [w.strip().lower() for w in stop]
        
        with open(filename) as text:            
            text_string = text.read()
            text_words = word_tokenize(text_string)
            text_words.sort()
            
            # output: (word, frequency)
            # Word frequency count with upper case words
            word_freq_cap = [(key, len(list(group))) for key, group in groupby(text_words, lambda x: x)]
            
            target = open('POS output/' + filename + '_withCap.txt', 'w')
            target.write(str(word_freq_cap))
            
            # Word frequency count without upper case words
            word_freq = [(key, len(list(group))) for key, group in groupby(text_words, lambda x: x.lower())]
            
            target = open('POS output/' + filename + '_noCap.txt', 'w')
            target.write(str(word_freq))
            
            # filter out stop words
            word_filtered = [f for f in word_freq if f[0] not in stop_words]
            
            # vocabulary size with and without stop words
            sum_stats["vocabulary size"] = [len(word_freq), len(word_filtered)]
            
            # stopword frequency
            stop_freq = 0;
            # total word frequency
            total_freq = 0;
            
            for w, frq in word_freq:
                if w in stop_words:
                    stop_freq += frq
                total_freq += frq
            
            sum_stats["stop word frequency"] = stop_freq/float(total_freq)
            
            # number of capital letters            
            # average number of character per word
            total_chars = 0;
            total_words = 0;
            total_capital = 0;
            
            for word, freq in word_freq_cap:
                total_chars += len(word)*freq
                total_words += freq
                total_capital += freq * len(pattern.findall(word))
            
            if total_chars != 0 and total_words != 0:
                sum_stats["avg chars per word"] = total_chars/float(total_words)
                sum_stats["number of capital letters"] = total_capital
                
            # POS tagging, without filtering stop words 
            pos_info = None
            pos_info = pos_tag(word_tokenize(text_string.lower()))
            
            target = open('POS output/' + filename + 'pos_noCap.txt', 'w')
            target.write(str(pos_info))
                
            pos_counts = dict();
            cross_walk = {'NN':'noun', 'JJ':'adjectives', 
                          'VB':'verbs', 'RB':'adverbs', 'PRP':'pronouns'}
            
            for word, pos in pos_info:
                pos_type = cross_walk.get(pos);
                if pos:
                    pos_counts[pos_type] = pos_counts.get(pos_type, 0) + 1
                else:
                    None
                    
            sum_stats["POS counts"] = pos_counts
            
            pos_info_sort = sorted (pos_info, key= lambda x: (x[1], x[0]))
            pos_word_counts = [(k, len(list(group))) for k, group in groupby(pos_info_sort, lambda x: x )]

            # desired output: ((word, pos) counts)  
            # but the problem is that pos does not combine        
            tmp1 = sorted(pos_word_counts, key = lambda x: x[0][1])
            tmp2 = [(k, sorted(group, key = lambda x: x[1], reverse=True)) for k, group in groupby(tmp1, lambda x: x[0][1])]
            
            pos_list , word_count = zip(*tmp2)
            
            topwords_by_pos = dict()
            
            for pos_type in pos_list:
                ts1 = word_count[pos_list.index(pos_type)]
                cnt = 0
                ts2 = list();
                for word_tup in ts1:
                    if not (cnt < 10):
                        break
                    ts2.append((word_tup[0][0], word_tup[1]))
                    cnt += 1
                topwords_by_pos[cross_walk.get(pos_type)] = ts2
            
            sum_stats["top words by pos"] = topwords_by_pos
            
            # output: (frequency, # of unique word)
            word_sorted = sorted(word_filtered, key = lambda x: x[1])
            cnt_word = [(key, len(list(group))) for key, group in groupby(word_sorted, lambda x: x[1])]
            
            log_word = [(log(x), log(y)) for x, y in cnt_word]

            target = open('POS output/' + filename + 'summary.txt', 'w')
            target.write(str(sum_stats))

        return sum_stats
pos_stats('sotu1790-2015.txt')<jupyter_output><empty_output><jupyter_text>By Presidents  <NAME> (1790-1796)<jupyter_code>pos_stats('1GW.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1797-1800)<jupyter_code>pos_stats('2JA.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1801-1808)<jupyter_code>pos_stats('3TJ.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1809-1816)<jupyter_code>pos_stats('4JM.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1817-1824)<jupyter_code>pos_stats('5JM.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1825-1828)<jupyter_code>pos_stats('6JQA.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1829-1836)<jupyter_code>pos_stats('7AJ.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1837-1840)<jupyter_code>pos_stats('8MVB.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1841-1844)<jupyter_code>pos_stats('10JT.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1845-1848)<jupyter_code>pos_stats('11JP.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1849)<jupyter_code>pos_stats('12ZT.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1850-1852)<jupyter_code>pos_stats('13MF.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1853-1856)<jupyter_code>pos_stats('14FP.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1857-1860)<jupyter_code>pos_stats('15JB.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1861-1864)<jupyter_code>pos_stats('16AL.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1865-1868)<jupyter_code>pos_stats('17AJ.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1869-1876)<jupyter_code>pos_stats('18USG.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1877-1880)<jupyter_code>pos_stats('19RBH.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1881-1884)<jupyter_code>pos_stats('21CA.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1885-1888, 1893-1896)<jupyter_code>pos_stats('22GC.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1889-1892)<jupyter_code>pos_stats('23BH.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1897-1900)<jupyter_code>pos_stats('25WM.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1901-1908)<jupyter_code>pos_stats('26TR.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1909-1912)<jupyter_code>pos_stats('27WT.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1913-1920)<jupyter_code>pos_stats('28WW.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1921-1922)<jupyter_code>pos_stats('29WH.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1923-1928)<jupyter_code>pos_stats('30CC.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1929-1932)<jupyter_code>pos_stats('31HH.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1934-1945)<jupyter_code>pos_stats('32FDR.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1946-1953)<jupyter_code>pos_stats('33HT.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1953-1961)<jupyter_code>pos_stats('34DE.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1961-1963)<jupyter_code>pos_stats('35JFK.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1964-1969)<jupyter_code>pos_stats('36LBJ.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1970-1974)<jupyter_code>pos_stats('37RN.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1975-1977)<jupyter_code>pos_stats('38GRF.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1978-1981)<jupyter_code>pos_stats('39JC.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1982-1988)<jupyter_code>pos_stats('40RR.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1989-1992)<jupyter_code>pos_stats('41GHB.txt')<jupyter_output><empty_output><jupyter_text><NAME> (1993-2000)<jupyter_code>pos_stats('42WC.txt')<jupyter_output><empty_output><jupyter_text><NAME> (2001-2008)<jupyter_code>pos_stats('43GWB.txt')<jupyter_output><empty_output><jupyter_text><NAME> (2009-2015)<jupyter_code>pos_stats('44OB.txt')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13333333333333333, 'fraction_numerical': 0.04810126582278481, 'mean_word_length': 3.3812384473197783, 'pattern_counts': {'":': 0, '<': 215, '<?xml version=': 0, '>': 214, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '43', 'dir': 'jupyter-structured-clean-dedup', 'id': '567682', 'license': 'MIT', 'n_tokens_mistral': 3683, 'n_tokens_neox': 3312, 'n_words': 486, 'path': 'POS distribution.ipynb', 'repo_name': 'shngli/SOTU-mining', 'size': '9848'}