<jupyter_start><jupyter_text>Get The Top Subreddits By Subscriber CountThe top Subreddits does not include any that are private/quarantined. This does include NSFW subreddits.Data:* From: 2021-11-19* Source: https://frontpagemetrics.com/list-all-subredditsNew Data Stored In: top_subreddits.json<jupyter_code>import gzip
import praw
from config import *
from csv import reader
from collections import defaultdict
from prawcore.exceptions import Forbidden
# Will map a subreddit to its popularity
subreddit_popularity = defaultdict(int)

# The total amount of subreddits that will be returned
n = 500
# Open the subreddit popularity data
with gzip.open("../data/2021-11-19.csv.gz", "rt", errors="ignore") as read_obj:
    # pass the file object to reader() to get the reader object
    csv_reader = reader(read_obj)
    
    # Skip the header
    next(csv_reader)
    
    # Iterate over each row in the csv using reader object
    for row in csv_reader:
        subreddit = row[0]
        subscribers = row[-1]
        
        subreddit_popularity[subreddit] = subscribers
# Sort the subreddits by their subscriber count
subreddit_popularity = sorted(subreddit_popularity.items(), key=lambda x: int(x[1]), reverse=True)
# Initialize PRAW API
reddit = praw.Reddit(
    client_id=CLIENT_ID,
    client_secret=CLIENT_SECRET,
    user_agent=USER_AGENT,
    username=REDDIT_USERNAME,
    password=<PASSWORD>,
)
# Remove any private subreddits from the list
remove_list = []

# Only remove subreddits until n public subreddits are in the top
m = 0

for i in range(len(subreddit_popularity)):
    # Edge Case: Stop once n amount of subreddits are in the top
    if m == n:
        break

    subreddit = subreddit_popularity[i][0]
    
    # Edge Case: Remove r/announcements due to barely 5 posts this year
    if subreddit == "announcements":
        remove_list.append(subreddit_popularity[i])
        continue
    
    # Try to find subreddits that are locked/quarantined
    try:
        for submission in reddit.subreddit(subreddit).top('year', limit=5):
            pass
        m += 1
    except Forbidden:
        print("Locked/Quarantined Subreddit: {}".format(subreddit))
        remove_list.append(subreddit_popularity[i])
for subreddit in remove_list:
    subreddit_popularity.remove(subreddit)
file = open("../data/top_subreddits.json", "w")
file.write("{\n")

for i, subreddit in enumerate(subreddit_popularity[:n]):
    if i < (n - 1):
        file.write("\t\"{}\":{},\n".format(subreddit[0], subreddit[1]))
    else:
        file.write("\t\"{}\":{}\n".format(subreddit[0], subreddit[1]))
        
file.write("}\n")
file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10344827586206896, 'fraction_numerical': 0.01199400299850075, 'mean_word_length': 3.6579406631762654, 'pattern_counts': {'":': 3, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '55970', 'license': 'MIT', 'n_tokens_mistral': 887, 'n_tokens_neox': 875, 'n_words': 244, 'path': 'scripts/top_subreddits.ipynb', 'repo_name': 'DonaldWolfson/Reddit-Prediction-Model', 'size': '2673'}