<jupyter_start><jupyter_text>Keyness Analysis (R v. D)  Setup<jupyter_code>import os

from collections import Counter

%matplotlib inline

import os
import re
import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from nltk import tokenize
from nltk.sentiment.vader import SentimentIntensityAnalyzer
## Parameters
to_strip = ',.\xa0:-()\';$"/?][!`Ą@Ś§¨’–“”…ï‘>&\\%˝˘*'
## Open speeches
all_speeches_r = open('data/republican_all.txt').read()
all_speeches_d = open('data/democrat_all.txt').read()<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>%run functions.ipynb<jupyter_output><empty_output><jupyter_text>Get n-Gram Distributions<jupyter_code>%run n-gram_frequency_analysis.ipynb<jupyter_output>98055 tokens in the Republican convention nominee speeches
91071 tokens in the Democratic convention nominee speeches<jupyter_text>Keyness Analysis<jupyter_code>calculate_keyness(word_freq_d, word_freq_r)
calculate_keyness(bigram_freq_d,  bigram_freq_r)
calculate_keyness(trigram_freq_d, trigram_freq_r)<jupyter_output>WORD                     Corpus A Freq.Corpus B Freq.Keyness
============================================================
the democratic party     47        8         33.590
we can do                24        7         11.160
the people of            40        18        10.268
i want to                46        23        9.615
we're going to           41        20        9.020
of this country          21        8         7.044
{'fraction_non_alphanumeric': 0.12257200267916946, 'fraction_numerical': 0.03951774949765573, 'mean_word_length': 3.5828220858895707, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '118335', 'license': 'MIT', 'n_tokens_mistral': 544, 'n_tokens_neox': 487, 'n_words': 114, 'path': 'previous_final_project_examples/katepoole26/keyness_analysis.ipynb', 'repo_name': 'mbod/comm313_spring2021', 'size': '1493'}