<jupyter_start><jupyter_text>Reddit Data Step 1. Read in Data<jupyter_code># standard imports
import pandas as pd
import requests # Pushshift accesses Reddit via a url so this is needed
import json # JSON manipulation
# set URL
url = 'https://api.pushshift.io/reddit/search/submission/'

# point to subreddit
params_sr1 = {'subreddit': 'C_S_T'}
params_sr2 = {'subreddit': 'conspiracy'}

response_sr1 = requests.get(url, params_sr1)
response_sr2 = requests.get(url, params_sr2)

#check for errors in request -- we're all good
#response_sr2.status_code
# get the json data from both
data_sr1 = response_sr1.json()
data_sr2 = response_sr2.json()

# turn json dict into posts DataFrame
posts_sr1 = pd.DataFrame(data_sr1['data'])
posts_sr2 = pd.DataFrame(data_sr2['data'])

#inspect
posts_sr2.info()
# write a function to do this

# Function is variation on Katie Sylvia's BreakFast Hour Instruction
def get_pushshift_data(year, subreddit):
    year = str(year)
    
    start = year + '-01-01'
    if year == '2021':
        end = year + '-10-08'
    else:
        end = year + '-12-31'
    #print(start, end)    
    
    # URL
    url = 'https://api.pushshift.io/reddit/search/submission/'
    # Add params
    params = {'subreddit': subreddit,
              'size': 100,
              'after': start,
              'before': end,
              'filter': ['subreddit', 'title', 'selftext', 'num_comments', 'score'],
              #'num_comments': '>0',
              #'score': '>10'
              'selftext': True}
     
    # Get the data
    res = requests.get(url, params)
    # Convert the request into a list of dict objects
    data = res.json()
    # Go deeper
    posts = data['data']
    # Turn into DataFrame
    df = pd.DataFrame(posts)

    return df
cst_2021 = get_pushshift_data(2021, 'C_S_T')
cst_2020 = get_pushshift_data(2020, 'C_S_T')
cst_2019 = get_pushshift_data(2019, 'C_S_T')
cst_2018 = get_pushshift_data(2018, 'C_S_T')
cst_2017 = get_pushshift_data(2017, 'C_S_T')

cst_df = pd.concat([cst_2017, cst_2018, cst_2019, cst_2020, cst_2021])
cst_df.info()
#get reddit data from r/conspiracy
con_2021 = get_pushshift_data(2021, 'conspiracy')
con_2020 = get_pushshift_data(2020, 'conspiracy')
con_2019 = get_pushshift_data(2019, 'conspiracy')
con_2018 = get_pushshift_data(2018, 'conspiracy')
con_2017 = get_pushshift_data(2017, 'conspiracy')


#convert to DF
con_df = pd.concat([con_2017, con_2018, con_2019, con_2020, con_2021])
#ls_df.head()
con_df.info()
# 
subreddits = pd.concat([cst_df, con_df])

subreddits.to_csv('./data/subreddits.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1279293123319247, 'fraction_numerical': 0.06108336534767576, 'mean_word_length': 3.786764705882353, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '524559', 'license': 'CC0-1.0', 'n_tokens_mistral': 1055, 'n_tokens_neox': 946, 'n_words': 226, 'path': '01_code/01_DataCollection.ipynb', 'repo_name': 'khiggins115/reddit_nlp', 'size': '2603'}