<jupyter_start><jupyter_text>What do we need from each submission?For training- The text- The flair (if it exists)- The top n comments (set to 5 in this case)For Metadata- The title- The url- The scoreAllowed 1 rq/s so we will add a 1s sleep after each request<jupyter_code>import time
from tqdm import tqdm

n_comments = 5
raw_submissions = []
failed = []

for submission in tqdm(submissions):
    try:
        # check if it was already removed
        if not submission['selftext'] == '[removed]':
            submission_obj = Submission(reddit, url=submission['url'])
            submission_obj.comment_sort = 'best'
            submission_obj.comment_limit = 5
            comments = []
            for top_level_comment in submission_obj.comments:
                if isinstance(top_level_comment, MoreComments):
                    continue
                comments.append(top_level_comment.body)
            if submission_obj.selftext == '[deleted]' or submission_obj.selftext == '[removed]':
                desired_text = submission['selftext']
            else:
                desired_text = submission_obj.selftext
            
            raw_submission = {
                'text': desired_text,
                'flair': submission_obj.link_flair_text if submission_obj.link_flair_text else '',
                'comments': comments,
                'title': submission_obj.title,
                'url': submission_obj.url,
                'score': submission_obj.score
            }
            raw_submissions.append(raw_submission)
            
            time.sleep(2)
            if reddit.auth.limits['remaining'] < 2:
                time.sleep(int(reddit.auth.limits['reset_timestamp'] - time.time()))
        else:
            failed.append(submission)
    except Exception as e:
        failed.append(submission)
with open('../data/submissions-2016-2020-raw-reddit-api-data.json', 'w') as json_f:
    json.dump(
        {
            'submissions': raw_submissions
        },
        json_f
    )
with open('../data/FAILED-submissions-2016-2020-raw-reddit-api-data.json', 'w') as json_f:
    json.dump(
        {
            'submissions': failed
        },
        json_f
    )
len(raw_submissions)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09995557529986672, 'fraction_numerical': 0.010217681030653044, 'mean_word_length': 2.055630936227951, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '635803', 'license': 'MIT', 'n_tokens_mistral': 665, 'n_tokens_neox': 637, 'n_words': 145, 'path': 'research/data_retrieval/Pull-Up-To-Date-Reddit-Submission-Data.ipynb', 'repo_name': 'declanatkins/AITA-Prediction-Bot', 'size': '2251'}