<jupyter_start><jupyter_text>Initial Set Up<jupyter_code>import tweepy
import configparser
import os
import json
import GetOldTweets3 as got
import datetime
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime
import nltk
from nltk.corpus import stopwords 
from nltk.tokenize import word_tokenize 
import string
import random
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import re
import csv
import math

from collections import Counter
%run functions.ipynb<jupyter_output><empty_output><jupyter_text>Notes Within my 'data' folder:- 1 refers to tweets from January- 2 refers to tweets from February- 3 refers to tweets from March- 4 refers to tweets from April- all_time refers to tweets from the start of January to the end of April- text1 and corpus_index1.json refer to news articles from January- text2 and corpus_index2.json refer to news articles from February- text3 and corpus_index3.json refer to news articles from March- text4 and corpus_index4.json refer to news articles from April- text_all refers to news articles from the start of January to the end of April  Twitter Data<jupyter_code>twitter_cred = {}
config = configparser.ConfigParser()
config.read('config.ini')
for item,value in config['TWITTER'].items():
    twitter_cred[item]=value
auth = tweepy.OAuthHandler(twitter_cred['consumer_key'], twitter_cred['consumer_secret'])
auth.set_access_token(twitter_cred['access_key'], twitter_cred['access_secret'])
api = tweepy.API(auth)
!pip install --user GetOldTweets3
queries = ['"asian american" OR "asian-american" OR #asianamerican AND \
            racism OR racist OR xenophobia OR #racism OR #racist OR #xenophobia AND \
            coronavirus OR "corona virus" OR covid19 OR covid-19 OR pandemic OR virus OR "chinese virus" OR "china virus" OR \
            #coronavirus OR #covid19 OR #pandemic OR #chinavirus OR #chinesevirus']<jupyter_output><empty_output><jupyter_text>**January**<jupyter_code>DATA_DIR = 'data/1'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-01-01"
until = "2020-02-01"

for query in queries:
    tweet_list = download_query_tweets(query, since, until)
    
    outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, 'tweets', since, until)
    
    print('Downloaded {} tweets...\n'.format(len(tweet_list)))
    with open(outfilename,'w') as out:
        for tweet in tweet_list:
            out.write(json.dumps(tweet, default=jsonconverter) + '\n')<jupyter_output>Downloading tweets for query: '"asian american" OR "asian-american" OR #asianamerican AND             racism OR racist OR xenophobia OR #racism OR #racist OR #xenophobia AND             coronavirus OR "corona virus" OR covid19 OR covid-19 OR pandemic OR virus OR "chinese virus" OR "china virus" OR             #coronavirus OR #covid19 OR #pandemic OR #chinavirus OR #chinesevirus' from 2020-01-01 to 2020-02-01
Downloaded 7 tweets...<jupyter_text>**February**<jupyter_code>DATA_DIR = 'data/2'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-02-01"
until = "2020-03-01"

for query in queries:
    tweet_list = download_query_tweets(query, since, until)
    
    outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, 'tweets', since, until)
    
    print('Downloaded {} tweets...\n'.format(len(tweet_list)))
    with open(outfilename,'w') as out:
        for tweet in tweet_list:
            out.write(json.dumps(tweet, default=jsonconverter) + '\n')<jupyter_output>Downloading tweets for query: '"asian american" OR "asian-american" OR #asianamerican AND             racism OR racist OR xenophobia OR #racism OR #racist OR #xenophobia AND             coronavirus OR "corona virus" OR covid19 OR covid-19 OR pandemic OR virus OR "chinese virus" OR "china virus" OR             #coronavirus OR #covid19 OR #pandemic OR #chinavirus OR #chinesevirus' from 2020-02-01 to 2020-03-01
Downloaded 131 tweets...<jupyter_text>**March**<jupyter_code>DATA_DIR = 'data/3'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-03-01"
until = "2020-04-01"

for query in queries:
    tweet_list = download_query_tweets(query, since, until)
    
    outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, 'tweets', since, until)
    
    print('Downloaded {} tweets...\n'.format(len(tweet_list)))
    with open(outfilename,'w') as out:
        for tweet in tweet_list:
            out.write(json.dumps(tweet, default=jsonconverter) + '\n')<jupyter_output>Downloading tweets for query: '"asian american" OR "asian-american" OR #asianamerican AND             racism OR racist OR xenophobia OR #racism OR #racist OR #xenophobia AND             coronavirus OR "corona virus" OR covid19 OR covid-19 OR pandemic OR virus OR "chinese virus" OR "china virus" OR             #coronavirus OR #covid19 OR #pandemic OR #chinavirus OR #chinesevirus' from 2020-03-01 to 2020-04-01
Downloaded 1702 tweets...<jupyter_text>**April**<jupyter_code>DATA_DIR = 'data/4'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-04-01"
until = "2020-05-01"

for query in queries:
    tweet_list = download_query_tweets(query, since, until)
    
    outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, 'tweets', since, until)
    
    print('Downloaded {} tweets...\n'.format(len(tweet_list)))
    with open(outfilename,'w') as out:
        for tweet in tweet_list:
            out.write(json.dumps(tweet, default=jsonconverter) + '\n')<jupyter_output>Downloading tweets for query: '"asian american" OR "asian-american" OR #asianamerican AND             racism OR racist OR xenophobia OR #racism OR #racist OR #xenophobia AND             coronavirus OR "corona virus" OR covid19 OR covid-19 OR pandemic OR virus OR "chinese virus" OR "china virus" OR             #coronavirus OR #covid19 OR #pandemic OR #chinavirus OR #chinesevirus' from 2020-04-01 to 2020-05-01
Downloaded 1260 tweets...<jupyter_text>**All-time**<jupyter_code>DATA_DIR = 'data/all_time'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-01-01"
until = "2020-05-01"

for query in queries:
    tweet_list = download_query_tweets(query, since, until)
    
    outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, 'tweets', since, until)
    
    print('Downloaded {} tweets...\n'.format(len(tweet_list)))
    with open(outfilename,'w') as out:
        for tweet in tweet_list:
            out.write(json.dumps(tweet, default=jsonconverter) + '\n')
racist_queries = ['"ching chong" OR chink OR chingchong OR #chingchong OR #chink\
                    AND "kung flu" OR "kung fu flu" OR "ching chong virus" OR #kungflu OR #kungfuflu OR #chingchongvirus']
import datetime
DATA_DIR = 'data/all_time'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-01-01"
until = "2020-05-01"

for query in racist_queries:
    tweet_list = download_query_tweets(query, since, until)
    
    outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, 'racist_tweets', since, until)
    
    print('Downloaded {} tweets...\n'.format(len(tweet_list)))
    with open(outfilename,'w') as out:
        for tweet in tweet_list:
            out.write(json.dumps(tweet, default=jsonconverter) + '\n')<jupyter_output>Downloading tweets for query: '"ching chong" OR chink OR chingchong OR #chingchong OR #chink                    AND "kung flu" OR "kung fu flu" OR "ching chong virus" OR #kungflu OR #kungfuflu OR #chingchongvirus' from 2020-01-01 to 2020-05-01
Downloaded 1840 tweets...<jupyter_text>**Donald Trump's tweets**<jupyter_code>username = 'realdonaldtrump'
query = 'Chinese Virus'

DATA_DIR='data/all_time'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-01-01"
until = "2020-05-01"

tweet_list = download_user_tweets(username, query, since, until)
outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, username, since, until)
print('Downloaded {} tweets...\n'.format(len(tweet_list)))
with open(outfilename,'w') as out:
    for tweet in tweet_list:
        out.write(json.dumps(tweet, default=jsonconverter) + '\n')<jupyter_output>Downloading for realdonaldtrump and Chinese Virus
Downloaded 8 tweets...<jupyter_text>**<NAME>'s Tweets**<jupyter_code>username = 'secpompeo'
query = 'Wuhan Virus OR #WuhanVirus'

DATA_DIR='data/all_time'

if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)

since = "2020-01-01"
until = "2020-05-01"

tweet_list = download_user_tweets(username, query, since, until)
outfilename = "{}/{}_{}_to_{}.json".format(DATA_DIR, username, since, until)
print('Downloaded {} tweets...\n'.format(len(tweet_list)))
with open(outfilename,'w') as out:
    for tweet in tweet_list:
        out.write(json.dumps(tweet, default=jsonconverter) + '\n')<jupyter_output>Downloading for secpompeo and Wuhan Virus
Downloaded 8 tweets...<jupyter_text>News Articles January<jupyter_code>ZIPFILE_NAME1='christyq-b862-s2020-04-20.zip'

if not os.path.exists('data'):
    os.makedirs('data/text1')
    
manifest_data=[]

text_file_cnt=0

with zipfile.ZipFile(ZIPFILE_NAME1) as zf:
    for f in zf.filelist:
        if f.filename.count('plaintext')>0:
            fn=os.path.basename(f.filename)
            print('Extracting', fn)
            with open(os.path.join('data','text1',fn),'wb') as out:
                zipfile.shutil.copyfileobj(zf.open(f),out)
                text_file_cnt+=1
                
        if f.filename.endswith('.csv'):
            mdf = pd.read_csv(zf.open(f))
            manifest_data.append(mdf)
            
mdf=pd.concat(manifest_data)
mdf.to_json('data/corpus_index1.json', orient='records')
fn<jupyter_output><empty_output><jupyter_text>February<jupyter_code>ZIPFILE_NAME2='christyq-b863-s2020-04-20.zip'

if not os.path.exists('data'):
    os.makedirs('data/text2')
    
manifest_data=[]

text_file_cnt=0

with zipfile.ZipFile(ZIPFILE_NAME2) as zf:
    for f in zf.filelist:
        if f.filename.count('plaintext')>0:
            fn=os.path.basename(f.filename)
            print('Extracting', fn)
            with open(os.path.join('data','text2',fn),'wb') as out:
                zipfile.shutil.copyfileobj(zf.open(f),out)
                text_file_cnt+=1
                
        if f.filename.endswith('.csv'):
            mdf = pd.read_csv(zf.open(f))
            manifest_data.append(mdf)
            
mdf=pd.concat(manifest_data)
mdf.to_json('data/corpus_index2.json', orient='records')<jupyter_output>Extracting ktvu-coronavirus-scare-affecting-7762fc06-8629-11ea-8212-0242ac160002.txt
Extracting newsy-coronavirus-outbreak-reportedly-775c0838-8629-11ea-8212-0242ac160002.txt
Extracting forbes.com-how-covid-19-coronavirus-77e7ba04-8629-11ea-8212-0242ac160002.txt
Extracting laist-rage-tweets_-jokes-and-77c79e0e-8629-11ea-8212-0242ac160002.txt
Extracting china.org.cn-xinhua-headlines_-xenophobia-77d16a1a-8629-11ea-8212-0242ac160002.txt
Extracting msn-south-africa-fear-of-coronavirus-777100b2-8629-11ea-8212-0242ac160002.txt
Extracting abc7-la-leaders-speak-77bc0210-8629-11ea-8212-0242ac160002.txt
Extracting nbc-los-angeles-restaurants-in-la’s-774cc364-8629-11ea-8212-0242ac160002.txt
Extracting abc7-coronavirus_-local-boy-77c4708a-8629-11ea-8212-0242ac160002.txt
Extracting sina-xenophobia-against-asians-776c5620-8629-11ea-8212-0242ac160002.txt
Extracting business-insider-india-the-wuhan-coronavirus-778412c4-8629-11ea-8212-0242ac160002.txt
Extracting southwest-times-record-commentary_-i-did[...]<jupyter_text>March<jupyter_code>ZIPFILE_NAME3='christyq-b864-s2020-04-20.zip'

if not os.path.exists('data'):
    os.makedirs('data/text3')
    
manifest_data=[]

text_file_cnt=0

with zipfile.ZipFile(ZIPFILE_NAME3) as zf:
    for f in zf.filelist:
        if f.filename.count('plaintext')>0:
            fn=os.path.basename(f.filename)
            print('Extracting', fn)
            with open(os.path.join('data','text3',fn),'wb') as out:
                zipfile.shutil.copyfileobj(zf.open(f),out)
                text_file_cnt+=1
                
        if f.filename.endswith('.csv'):
            mdf = pd.read_csv(zf.open(f))
            manifest_data.append(mdf)
            
mdf=pd.concat(manifest_data)
mdf.to_json('data/corpus_index3.json', orient='records')<jupyter_output>Extracting east-bay-times-coronavirus-cartoons_-trump-4003aba2-8629-11ea-8212-0242ac160002.txt
Extracting ecns.cn-washington-under-fire-3fe900f4-8629-11ea-8212-0242ac160002.txt
Extracting xinhua-news-agency-spotlight_-u.s.-politicians_-40ac6e54-8629-11ea-8212-0242ac160002.txt
Extracting msn-south-africa-i’m-chinese.-that-3eeb605c-8629-11ea-8212-0242ac160002.txt
Extracting xinhua-news-agency-los-angeles-mayor-409f1b82-8629-11ea-8212-0242ac160002.txt
Extracting hr-magazine-coronavirus-and-racism_-41ceba08-8629-11ea-8212-0242ac160002.txt
Extracting click2houston.com-as-coronavirus-spreads_-4173f83e-8629-11ea-8212-0242ac160002.txt
Extracting alternet.org-xenophobia-and-racism-3eb3ec8a-8629-11ea-8212-0242ac160002.txt
Extracting latino-rebels-nahj-stands-with-40b051fe-8629-11ea-8212-0242ac160002.txt
Extracting salt-lake-tribune-3-news-live-4047e24a-8629-11ea-8212-0242ac160002.txt
Extracting newsr.in-coronavirus-increases-racist-41833eac-8629-11ea-8212-0242ac160002.txt
Extracting newsbusters-[...]<jupyter_text>April<jupyter_code>ZIPFILE_NAME4='christyq-b902-s2020-05-08.zip'

if not os.path.exists('data'):
    os.makedirs('data/text4')
    
manifest_data=[]

text_file_cnt=0

with zipfile.ZipFile(ZIPFILE_NAME4) as zf:
    for f in zf.filelist:
        if f.filename.count('plaintext')>0:
            fn=os.path.basename(f.filename)
            print('Extracting', fn)
            with open(os.path.join('data','text4',fn),'wb') as out:
                zipfile.shutil.copyfileobj(zf.open(f),out)
                text_file_cnt+=1
                
        if f.filename.endswith('.csv'):
            mdf = pd.read_csv(zf.open(f))
            manifest_data.append(mdf)
            
mdf=pd.concat(manifest_data)
mdf.to_json('data/corpus_index4.json', orient='records')<jupyter_output>Extracting public-radio-international-xenophobia-‘takes-its-d382778c-9105-11ea-8d39-0242ac160002.txt
Extracting wbez-91.5-chicago-coronavirus-in-illinois_-d29c7a2a-9105-11ea-8d39-0242ac160002.txt
Extracting outlook-india-coronavirus_-former-nba-d1f18142-9105-11ea-8d39-0242ac160002.txt
Extracting southwest-times-record-lululemon-apologizes-after-d2cbaf5c-9105-11ea-8d39-0242ac160002.txt
Extracting abs-cbn-news-jeremy-lin-pledges-d394f754-9105-11ea-8d39-0242ac160002.txt
Extracting scpr-_a-perfect-storm__-d2b0667a-9105-11ea-8d39-0242ac160002.txt
Extracting business-standard-india-jeremy-lin-pledges-d28f7442-9105-11ea-8d39-0242ac160002.txt
Extracting npr-q&a-with-leonardo-d202f8aa-9105-11ea-8d39-0242ac160002.txt
Extracting colorado-springs-gazette-reports_-pandemic-giving-d228a3a2-9105-11ea-8d39-0242ac160002.txt
Extracting miami-herald-tzi-ma-is-d366c3ac-9105-11ea-8d39-0242ac160002.txt
Extracting jakarta-post-young-people-share-d3aec99a-9105-11ea-8d39-0242ac160002.txt
Extracting wbez-91.5-c[...]<jupyter_text>**All-time**<jupyter_code>ZIPFILE_NAMES= ['christyq-b862-s2020-04-20.zip','christyq-b863-s2020-04-20.zip','christyq-b864-s2020-04-20.zip','christyq-b902-s2020-05-08.zip']

if not os.path.exists('data'):
    os.makedirs('data/text_all')
    
manifest_data=[]

text_file_cnt=0

for file in ZIPFILE_NAMES:
    with zipfile.ZipFile(file) as zf:
        for f in zf.filelist:
            if f.filename.count('plaintext')>0:
                fn=os.path.basename(f.filename)
                print('Extracting', fn)
                with open(os.path.join('data','text_all',fn),'wb') as out:
                    zipfile.shutil.copyfileobj(zf.open(f),out)
                    text_file_cnt+=1
                
            if f.filename.endswith('.csv'):
                mdf = pd.read_csv(zf.open(f))
                manifest_data.append(mdf)
            
    mdf=pd.concat(manifest_data)
    mdf.to_json('data/corpus_index_all.json', orient='records')<jupyter_output>Extracting cbs-news-uc-berkeley-deletes-770ecfdc-8629-11ea-8212-0242ac160002.txt
Extracting miami-herald-uc-berkeley-apologizes-77182bea-8629-11ea-8212-0242ac160002.txt
Extracting greenville-news-coronavirus-spreads-racism_-7713f034-8629-11ea-8212-0242ac160002.txt
Extracting newstex-blogs-the-nation-blogs-this-lunar-new-772bdfa0-8629-11ea-8212-0242ac160002.txt
Extracting newstex-blogs-the-nation-blogs-this-lunar-new-77261a02-8629-11ea-8212-0242ac160002.txt
Extracting newstex-blogs-the-nation-blogs-a-fire-may-77204be0-8629-11ea-8212-0242ac160002.txt
Extracting mailonline-jeff-lewis-apologizes-7736b4f2-8629-11ea-8212-0242ac160002.txt
Extracting mailonline-university-of-california-773310fe-8629-11ea-8212-0242ac160002.txt
Extracting ktvu-coronavirus-scare-affecting-7762fc06-8629-11ea-8212-0242ac160002.txt
Extracting newsy-coronavirus-outbreak-reportedly-775c0838-8629-11ea-8212-0242ac160002.txt
Extracting forbes.com-how-covid-19-coronavirus-77e7ba04-8629-11ea-8212-0242ac160002.txt
Extractin[...]
{'fraction_non_alphanumeric': 0.12668002829521338, 'fraction_numerical': 0.08924781891063428, 'mean_word_length': 3.91027496382055, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '232498', 'license': 'MIT', 'n_tokens_mistral': 7317, 'n_tokens_neox': 6310, 'n_words': 1087, 'path': 'previous_final_project_examples/christy-qiu/building_and_organizing_corpus.ipynb', 'repo_name': 'mbod/comm313_spring2021', 'size': '16969'}