<jupyter_start><jupyter_text>Data Collection & Cleaning<jupyter_code>import pandas as pd
import numpy as np

import requests<jupyter_output><empty_output><jupyter_text>Keto Subreddit: Submissions<jupyter_code>def submission(subreddit, before):
    """
    This function pulls the submissions of the desired subreddit before a specified time range.
    """
    params = {'subreddit' : subreddit, 'size' : 500, 'before' : before}
    res = requests.get('https://api.pushshift.io/reddit/search/submission', params)
    reddit_submissions = res.json()
    posts = reddit_submissions['data']
    return posts<jupyter_output><empty_output><jupyter_text>Pulling 1st Submission- To establish a datframe to append the rest of our automated pulled rows<jupyter_code>keto_post = submission('keto', None)
keto_subs = pd.DataFrame(keto_post, columns = ['subreddit', 'title', 'selftext'])
keto_subs
# grab 'created_utc' from the last post in our first submissions pull to utilize in loop

keto_post[-1]['created_utc']<jupyter_output><empty_output><jupyter_text>Creating a loop for data collection:<jupyter_code># goal was to run enough times to gather over 5k rows of data

# this is the 'created_utc' from the last post in the first submissions pull
before = 1619132798

for i in range (2,81):
    # subtracted utc by a large enough number each time in hopes to grab unique rows each time
    before -= 200_000
    
    # append new rows to the already created dataframe from the first submissions pull
    keto_subs = keto_subs.append(pd.DataFrame(submission('keto', before))[['subreddit', 'title', 'selftext']])
# check what was collected

keto_subs
# filter out for '[removed]'

keto_subs = keto_subs[keto_subs['selftext'] != '[removed]']
# filter out for '[deleted]'

keto_subs = keto_subs[keto_subs['selftext'] != '[deleted]']
# check for null values

keto_subs.isna().sum()
# drop null values

keto_subs.dropna(inplace = True)
# ensure null values were dropped

keto_subs.isna().sum()
# check 'title' column

keto_subs['title'].value_counts(ascending = False)
# referenced https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.drop_duplicates.html
# dropping dupicate titles

keto_subs.drop_duplicates(subset = ['title'], inplace = True)
keto_subs['selftext'].value_counts(ascending = False)
# also drop duplicates of 'selftext'

keto_subs.drop_duplicates(subset = ['selftext'], inplace = True)
# check shape to see how many rows are left

keto_subs.shape
# reset index after all the dropping that was done

keto_subs.reset_index(drop = True, inplace = True)
# final review of the keto df before writing to csv

keto_subs
# write cleaned keto Submissions to CSV

keto_subs.to_csv('keto_subs.csv', index = False)<jupyter_output><empty_output><jupyter_text>---  Nutrition Subreddit: Submissions<jupyter_code>def submission(subreddit, before):
    """
    This function pulls the submissions of the desired subreddit before a specified time range.
    """
    params = {'subreddit' : subreddit, 'size' : 500, 'before' : before}
    res = requests.get('https://api.pushshift.io/reddit/search/submission', params)
    
    if res.status_code == 200:
        reddit_submissions = res.json()
        return reddit_submissions['data']
    else:
        return -1<jupyter_output><empty_output><jupyter_text>Pulling 1st Submission- To establish a datframe to append the rest of our automated pulled rows<jupyter_code>nutrition_post = submission('nutrition', None)
nutrition_subs = pd.DataFrame(nutrition_post, columns = ['subreddit', 'title', 'selftext'])
nutrition_subs
# grab 'created_utc' from the last post in our first submissions pull to utilize in loop

nutrition_post[-1]['created_utc']<jupyter_output><empty_output><jupyter_text>Creating a loop for data collection:<jupyter_code># goal was to run enough times to gather over 5k rows of data

# this is the 'created_utc' from the last post in the first submissions pull
before = 1619331270

for i in range (2,81):
    # subtracted utc by a large enough number each time in hopes to grab unique rows each time
    before -= 200_000
    
    submission_res = submission('nutrition', before)
    
    # if submission_res returned -1 (res.status_code != 200), then skip over instance with 'continue'
    if submission_res == -1:
        continue
    else:                
        # append new rows to the already created dataframe from the first submissions pull
        nutrition_subs = nutrition_subs.append(pd.DataFrame(submission_res)[['subreddit', 'title', 'selftext']])
# check what was collected

nutrition_subs
# filter out for '[removed]'

nutrition_subs = nutrition_subs[nutrition_subs['selftext'] != '[removed]']
# filter out for '[deleted]'

nutrition_subs = nutrition_subs[nutrition_subs['selftext'] != '[deleted]']
# check for null values

nutrition_subs.isna().sum()
# drop null values

nutrition_subs.dropna(inplace = True)
# ensure null values were dropped

nutrition_subs.isna().sum()
# check 'title' column

nutrition_subs['title'].value_counts(ascending = False)
# referenced https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.drop_duplicates.html
# dropping dupicate titles

nutrition_subs.drop_duplicates(subset = ['title'], inplace = True)
# check 'selftext' column

nutrition_subs['selftext'].value_counts(ascending = False)
# also drop duplicates of 'selftext'

nutrition_subs.drop_duplicates(subset = ['selftext'], inplace = True)
# check shape to see how many rows are left

nutrition_subs.shape
# reset index after dropping all the rows from cleaning

nutrition_subs.reset_index(drop = True, inplace = True)
# final review of the nutrition df before writing to csv

nutrition_subs
# write cleaned keto Submissions to CSV

nutrition_subs.to_csv('nutrition_subs.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10182947877114257, 'fraction_numerical': 0.010182947877114256, 'mean_word_length': 5.1, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '131897', 'license': 'MIT', 'n_tokens_mistral': 1910, 'n_tokens_neox': 1833, 'n_words': 543, 'path': 'Data Collection & Cleaning.ipynb', 'repo_name': 'vleong1/NLP-Reddit-Classification', 'size': '5794'}