<jupyter_start><jupyter_text>Applies politicization and does calculation<jupyter_code># import required modules and set up environment
import os

# replace file path below with your own local convokit
os.chdir('../../')

import convokit

from convokit import Corpus, Parser, PolTracker, Transformer
import nltk
# load corpus, this takes a long time so you can also use a different corpus as a test
corpus = convokit.Corpus(filename='/Users/calebchiam/politics-filtered-labelled')
#corpus = convokit.Corpus(filename=convokit.download("iq2-corpus"))
corpus.print_summary_stats()
pt = PolTracker()
corpus = pt.transform(corpus)
shooting_timestamps = {
 'Fort Hood shooting': (1257397200, 1258088400),
 'Binghamton shootings': (1238731200, 1239422400),
 'Geneva County massacre': (1236657600, 1237348800),
 'Sandy Hook Elementary School shooting': (1355461200, 1356152400),
 'Aurora theater shooting': (1342756800, 1343448000),
 'Washington Navy Yard shooting': (1379304000, 1379995200),
 'San Bernardino attack': (1449032400, 1449723600),
 'Orlando nightclub shooting': (1465704000, 1466395200),
 'Las Vegas shooting': (1506830400, 1507521600),
 'Sutherland Springs church shooting': (1509854400, 1510545600),
 '<NAME> High School shooting': (1518584400, 1519275600),
 'Santa Fe High School shooting': (1526616000, 1527307200)}
def identify_bin_by_timestamp(bins, timestamp):
    for time_bin in bins:
        if time_bin[0] <= timestamp <= time_bin[1]:
            return time_bin
    return None
from collections import defaultdict
import seaborn as sns
import matplotlib.pyplot as plt
def get_counts_over_time(corpus, event, duration, bin_size):
    utts = [utt for utt in corpus.iter_utterances() if utt.meta['event'] == event and utt.meta['valid']]
    min_timestamp = min([utt.timestamp for utt in utts])
    num_bins = duration // bin_size
    bins = [(min_timestamp, min_timestamp + ((i+1)*bin_size) - 1) for i in range(num_bins)]
    
    bin_counts = defaultdict(int)
    
    for utt in utts:
        time_bin = identify_bin_by_timestamp(bins, utt.timestamp)
        if time_bin is not None:
            bin_counts[time_bin] += 1
    
    y = list(bin_counts.values())
    x = [i+1 for i in range(num_bins)]
    return y
import numpy as np
def get_scores_over_time(corpus, event, duration, bin_size):
    utts = [utt for utt in corpus.iter_utterances() if utt.meta['event'] == event and utt.meta['valid']]
    min_timestamp = min([utt.timestamp for utt in utts])
    num_bins = duration // bin_size
    bins = [(min_timestamp, min_timestamp + ((i+1)*bin_size) - 1) for i in range(num_bins)]
    
    bin_values = defaultdict(list)
    
    for utt in utts:
        time_bin = identify_bin_by_timestamp(bins, utt.timestamp)
        if time_bin is not None:
            bin_values[time_bin].append(utt.meta['political'])
    
    for time_bin, values in bin_values.items():
        bin_values[time_bin] = np.mean(values)
    
    y = list(bin_values.values())
    x = [i+1 for i in range(num_bins)]
    return y
hour = 60*60
threehour = 3*hour
sixhour = 6*hour
halfday = 12*hour
day = 24*hour
week = 7*day
import pandas as pd
from itertools import product
counts_over_time_all = [(event, get_counts_over_time(corpus, event, week, day)) for event in shooting_timestamps]
scores_over_time_all = [(event, get_scores_over_time(corpus, event, week, day)) for event in shooting_timestamps]
df = pd.DataFrame()

counts_data = []
scores_data = []

for counts_over_time in counts_over_time_all:
    event = counts_over_time[0]
    counts = counts_over_time[1]
    for idx, count in enumerate(counts):
        counts_data.append([event, idx+1, count])

for scores_over_time in scores_over_time_all:
    event = scores_over_time[0]
    scores = scores_over_time[1]
    for idx, score in enumerate(scores):
        scores_data.append([event, idx+1, score])
import seaborn as sns; sns.set(style="ticks", color_codes=True)

g = sns.FacetGrid(data=df, col="event", hue='event', col_wrap=4)
g = g.map(plt.plot, "days", "posts")

plt.subplots_adjust(top=0.92)
g = g.fig.suptitle('Number of posts per day 7 days after each mass shooting')
 
plt.show()

from nltk.tokenize import word_tokenize

## function takes in utterance and counts total words
def count_total_words(utt):
    if utt.text != None:
        tokenized = word_tokenize(utt.text.lower())
        return len(tokenized)
##check above function works as expected
print(count_total_words(corpus.get_utterance(utter_ids[400080])))
print(corpus.get_utterance(utter_ids[400080]))
## next, I want the incidence rate per day of political words

def plotPoliticization(name, start_date, end_date):
    with open('../'+name+'_utterance_ids.csv', 'r') as f:
    #with open('../auroratheater_utterance_ids.csv', 'r') as f:
        reader = csv.reader(f)
        sh_list = list(reader)
        
    inc_rate_sh = [0]*ps
    total_pol_words_sh = [0]*ps
    total_words_sh = [0]*ps
    
    times = pd.date_range(start=start_date,end=end_date,periods=ps+1)
    times = np.array(times)
    bin_times = times[:-1]
    
    # convert to datetime object
    times_temp = []
    for i,x in enumerate(times):
        times_temp.append(pd.to_datetime(x))
    times = times_temp   
    
    for i,x in enumerate(sh_list):
        utt = corpus.get_utterance(x[0])
        posted_time = datetime.fromtimestamp(utt.timestamp)
        y = 0
        while (posted_time > times[y]):

            y = y + 1

        ## this gives us the timeframe to mark it as
        total_pol_words_sh[y-1] = total_pol_words_sh[y-1] + utt.meta['num_pol_refs']
        total_words_sh[y-1] = total_words_sh[y-1] + count_total_words(utt)
        if y == 0:
            print(count_total_words(utt))

    print(total_pol_words_sh)
    print(total_words_sh)

    for i in range(0, ps):
        if total_words_sh[i] != 0:
            inc_rate_sh[i] = total_pol_words_sh[i]/total_words_sh[i]

    print(inc_rate_sh)
    return inc_rate_sh
plt.plot(bin_times,inc_rate_sh)
plt.xticks(rotation='vertical')
plt.ylabel('Political words/total words')
plt.title('Incidence rate of political words in utterances')
plt.show()
incidence_rate = []

print(events)
for i in range(len(events)):
    incidence_rate.append(plotPoliticization(events[i][3],events[i][1],events[i][2]))
    
print(incidence_rate)

#create df for facet plot

incidence_df = pd.DataFrame()

df_events_list = np.repeat(events_list, 7)
all_incidences = []

for incidence in incidence_rate:
    all_incidences+=incidence

incidence_df["event"] = df_events_list
incidence_df["days"] = list(range(1, 8))*11
incidence_df["incidences_rate"] = all_incidences

incidence_df
g = sns.FacetGrid(data=incidence_df, col="event", hue='event', col_wrap=4)
g = g.map(plt.plot, "days", "incidences_rate")

plt.subplots_adjust(top=0.92)
g = g.fig.suptitle('Number of incidences per day 7 days after each mass shooting')
 
plt.show()
print(incidence_rate)
incidence_rate_trend = [sum(incidence)/len(incidence) for incidence in incidence_rate]
print(incidence_rate_trend)
plt.plot(events_list,incidence_rate_trend)
plt.xticks(rotation='vertical')
plt.ylabel('Political words/total words')
plt.title('Incidence rate of political words in utterances after each event')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1157560706401766, 'fraction_numerical': 0.04304635761589404, 'mean_word_length': 4.302852962692026, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '35022', 'license': 'MIT', 'n_tokens_mistral': 2772, 'n_tokens_neox': 2493, 'n_words': 615, 'path': 'examples/politicization/.ipynb_checkpoints/plotter-checkpoint.ipynb', 'repo_name': 'calebchiam/cs6742-fork', 'size': '7258'}