<jupyter_start><jupyter_text>Data Collection  Imported Libraries<jupyter_code>import time
import requests
import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>The first step in this project was collecting reddit data from reddit.com. Pushshift.io reddit API facilitated the data collection by collecting 100 posts per request and restricting the search to a specific subreddit. Additionally, pushshift has a "before" parameter that allowed me to specify the date and time of the desired posts.<jupyter_code>def get_data(subreddit, size, before_time=1611893764):
   
    # The base_url is provided in the pushshift documentation
    base_url = 'https://api.pushshift.io/reddit/search/submission'
    # A list to capture each request
    posts = [] 
    posts_length = 0 
    
    #The while loops runs until the desired number of posts have been accumulated
    while posts_length < size:
        res = requests.get(base_url, params = {"subreddit": subreddit, "size": 100, "before": before_time}).json()
        data = res['data']
        df = pd.DataFrame(data)
        
        # Captures the before time so the next request pulls older posts
        before_time = int(df[['created_utc']].sort_values('created_utc').values[0])
        posts.append(df)
        posts_length += len(data)
    
        # A print statement to show progress (larger requests will take more time)
        if (posts_length) % 2000 == 0:
            print(f'Post {posts_length} of {size}')
        
        # Best practice - slow the number of requests to lessen the demand on the website
        time.sleep(3)
    
    return pd.concat(posts)

# Code Help:

# push api -- https://github.com/pushshift/api
# tutorial --> https://www.youtube.com/watch?v=AcrjEWsMi_E&feature=youtu.be
# time --> https://stackoverflow.com/questions/52004801/how-to-slow-down-asynchrounous-api-calls-to-match-api-limits
# utc time converter --> https://www.epochconverter.com
# web scaping loop --> https://medium.com/better-programming/how-to-scrape-multiple-pages-of-a-website-using-a-python-web-scraper-4e2c641cff8
# web scaping loop --> https://levelup.gitconnected.com/make-your-python-web-scraper-smarter-6233f2d10c3f<jupyter_output><empty_output><jupyter_text>Bodyweight Fitness Data<jupyter_code>bodyweight_fitness = get_data('bodyweightfitness', 10_000)
bodyweight_fitness.to_csv('../datasets/bodyweight_fitness', index=False)<jupyter_output><empty_output><jupyter_text>Powerlifting Data<jupyter_code>#Gathering data from PS4 subreddit 
powerlifting = get_data('powerlifting', 10000)
powerlifting.to_csv('../datasets/powerlifting', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10498687664041995, 'fraction_numerical': 0.021372328458942633, 'mean_word_length': 4.130769230769231, 'pattern_counts': {'":': 3, '<': 18, '<?xml version=': 0, '>': 22, 'https://': 7, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '391564', 'license': 'CC0-1.0', 'n_tokens_mistral': 878, 'n_tokens_neox': 834, 'n_words': 207, 'path': 'code/Data_Collection.ipynb', 'repo_name': 'btwmendes/natural-language-processing', 'size': '2667'}