<jupyter_start><jupyter_text>Analyze Entropy results<jupyter_code>from tqdm import tqdm
import os
# replace file path below with your own local convokit
os.chdir('/Users/marianneaubin/Documents/Classes/CS6742/cs6742-fork')
#os.chdir('/Users/grace/cs6742-fork/')
import convokit
from convokit import Corpus, User, Utterance
from convokit import PolTracker, EmoTracker
import nltk
import numpy as np
from collections import defaultdict
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
df = pd.read_csv("./datasets/politicization-corpus/csv_forum_entropy/neutral_forum_full_entropy.csv")
df.head()
print(len(df))
shooting_timestamps = {
 'Fort Hood shooting': (1257397200, 1258088400),
 'Binghamton shootings': (1238731200, 1239422400),
 'Geneva County massacre': (1236657600, 1237348800),
 'Sandy Hook Elementary School shooting': (1355461200, 1356152400),
 'Aurora theater shooting': (1342756800, 1343448000),
 'Washington Navy Yard shooting': (1379304000, 1379995200),
 'San Bernardino attack': (1449032400, 1449723600),
 'Orlando nightclub shooting': (1465704000, 1466395200),
 'Las Vegas shooting': (1506830400, 1507521600),
 'Sutherland Springs church shooting': (1509854400, 1510545600),
 '<NAME> High School shooting': (1518584400, 1519275600),
 'Santa Fe High School shooting': (1526616000, 1527307200)}
df2 = df.loc[(df['subreddit'].isin(['politics','worldnews','news']))]
df2.head()
print(len(df2))
df3 = df.loc[(df['antigun_entropy'] > 4000)]
df3['text'][534]
#print(len(df3))
49      &gt; Third, you didn't answer my question eith...
109     &gt;Saying you don't understand why a person n...
111     Great, gunman enters school with intention of ...
114     "Islamo-Mom stockpiles military-grade weapons ...
137     A guy is arrested for carrying a board with "H...
143     Your understanding of the 2000 report is highl...
157     I've heard Sean Hannity raise a similar point....
159     Well lets look at the most common 3 you see so...
168     http://www.cnn.com/2012/07/20/justice/colorado...
174     Well, I read the entire thing. Very nice socio...
216     DCPS announced a lock down of the following sc...
238     There's a difference between non-profit and be...
241     http://www.washingtonpost.com/world/asia_pacif...
242     Allegedly, this is [him](https://www.twitter.c...
243     Simple solution everyone. 1. Culture rhino ste...
252     Palestinians cannot vote in Israeli elections,...
282     “He said he was the Joker, enemy of Batman,” P...
304     This seems extremely unlikely IMO. This was a ...
317     &gt; The boys pleaded guilty on June 26 to fir...
326     Firearm deaths are significantly lower in citi...
359     From page 11:\n&gt; 12:27: /u/shankee: [Legal ...
385     And yet widespread prescription drug abuse is ...
519     The crowd loved her because she was mediocre. ...
520     But that has no bearing on his reasoning or hi...
534     Look at London and all the 'knife violence'. \...
545     Hmph!\n\n&gt; Riot police attack peaceful prot...
554     From a chess perspective: \n1. R<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11427646487536419, 'fraction_numerical': 0.11913240530916154, 'mean_word_length': 3.881516587677725, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '19171', 'license': 'MIT', 'n_tokens_mistral': 1238, 'n_tokens_neox': 972, 'n_words': 356, 'path': 'examples/politicization/analayse_entropy_results.ipynb', 'repo_name': 'calebchiam/cs6742-fork', 'size': '3099'}