<jupyter_start><jupyter_text>1. Dependencies<jupyter_code>import warnings
warnings.filterwarnings('ignore')
import os
import shutil
import numpy as np
import pandas as pd
import pickle
import cv2 as cv
import matplotlib.pyplot as plt
import praw
import requests
import json
import csv
import time
import datetime<jupyter_output><empty_output><jupyter_text>2. Data Acquisation  List of Properties of a submission, I am manually eliminating few of them, bacuase flair of the submission is least (or not) dependent on them ie: Flair is rarely dependent on author name, or subreddit etc All Properties: dict_keys(['all_awardings', 'allow_live_comments', 'author', 'author_flair_css_class', 'author_flair_richtext', 'author_flair_text', 'author_flair_type', 'author_fullname', 'author_patreon_flair', 'author_premium', 'awarders', 'can_mod_post', 'contest_mode', 'created_utc', 'domain', 'full_link', 'gildings', 'id', 'is_crosspostable', 'is_meta', 'is_original_content', 'is_reddit_media_domain', 'is_robot_indexable', 'is_self', 'is_video', 'link_flair_background_color', 'link_flair_css_class', 'link_flair_richtext', 'link_flair_template_id', 'link_flair_text', 'link_flair_text_color', 'link_flair_type', 'locked', 'media_only', 'no_follow', 'num_comments', 'num_crossposts', 'over_18', 'parent_whitelist_status', 'permalink', 'pinned', 'pwls', 'removed_by_category', 'retrieved_on', 'score', 'selftext', 'send_replies', 'spoiler', 'steward_reports', 'stickied', 'subreddit', 'subreddit_id', 'subreddit_subscribers', 'subreddit_type', 'thumbnail', 'title', 'total_awards_received', 'url', 'whitelist_status', 'wls'])  Reference: - https://praw.readthedocs.io/en/latest/index.html- https://github.com/pushshift/apia<jupyter_code># creating a reddit app instance to collect post data from r/india submreddit
reddit = praw.Reddit(client_id='xVeb5-ej49aBDg', client_secret='<KEY>', user_agent='reddit-scrap', username='macabdul9', password='<PASSWORD>')
# We can get the submission either by id or url
subm = reddit.submission(id="fxqifi")
subm.full_link<jupyter_output><empty_output><jupyter_text>Turn Around Time(TAT) for getting all comments of a post is slightly higher than using praw with submission_id/url than pushshift.io api comment search below code snippets illustrate the same<jupyter_code># start_time = time.time()
# comment_ids = []
# count = 0
# for comment in subm.comments.list():
#     comment_ids.append(comment.id)
#     print(comment.body)
#     count+=1
# print("--- %s seconds ---" % (time.time() - start_time))
# print(count)
# start_time = time.time()
# comments = getPushshiftSubData(','.join(comment_ids))
# count = 0
# for comment in comments:
#     print(comment['body'])
#     count +=1
# print("--- %s seconds ---" % (time.time() - start_time))
# print(count)
# This function will receive url of a submission, num_commnents as arg and it will return the top 10 comment
# and mean comment score(upvote - downvote)

def get_comments(url, num_comments):
    # call api using api wrapper
    subm = reddit.submission(url=url)
    comments_body = []
    sum_score = 0
    count = 0
    if num_comments > 0:
        for i, comment in enumerate(subm.comments.list()):

            # There may be some comments which has no body
            try : 
                comments_body.append(comment.body)
            except:
                comments_body.append('')

            # There exist some comments which has not given a score
            try:
                sum_score += comment.score
            except:
                sum_score += 0            
            count += 1
            # We only need 10 comments
            if (i+1)%10==0:
                break
    try:
        mean = sum_score/count
    except:
        mean = 0.0

    return " ".join(comments_body), mean
# Function to call and retrive the data using pushshift api
def getPushshiftSubData(query, after, before, sub):
    url = 'https://api.pushshift.io/reddit/search/submission/?title='+str(query)+'&size=2000&after='+str(after)+'&before='+str(before)+'&subreddit='+str(sub)
    print(url)
    r = requests.get(url)
    data = json.loads(r.text)
    return data['data']
# Properties to extract about a submission
features = [
    'allow_live_comments', 'author', 'author_premium', 'can_mod_post', 'contest_mode', 'created_utc',
    'full_link', 'id', 'is_crosspostable', 'is_meta', 'is_original_content', 'is_self', 'is_video',
    'link_flair_text', 'locked', 'media_only', 'no_follow', 'num_comments', 'num_crossposts', 'over_18',
    'parent_whitelist_status', 'permalink', 'pinned', 'score', 'selftext', 'send_replies', 'spoiler',
    'stickied', 'title', 'total_awards_received', 'url'
]
custom_features = [
    'comments', 'mean_comment_score'
]
# function to extract the features
def collectSubData(subm):
    # extracting all information about a submission
    feature_values = [subm.get(key) for key in features]    
    comments, mean_comment_score = get_comments(subm.get('full_link'), subm.get('num_comments')) 
    feature_values.append(comments)
    feature_values.append(mean_comment_score)
    subStats[subm.get('id')] = feature_values
# Parameters to extract the data
#Subreddit to query
sub='india'

#before and after dates
after = "1583020800"  # Sun, 01 Mar 2020 00:00:00 
before = "1586563199" # Fri, 10 Apr 2020 23:59:59  
query = "" # title should have either null string more 
subCount = 0
subStats = {}
data = getPushshiftSubData(query, after, before, sub)# Will run until all posts have been gathered 
# from the 'after' date up until before date
while len(data) > 0:
    for submission in data:
        collectSubData(submission)
        subCount+=1
    # Calls getPushshiftData() with the created date of the last submission
    # print(len(data))
    if subCount % 1000 == 0:
        print(f'{subCount} submission collected')
    print(str(datetime.datetime.fromtimestamp(data[-1]['created_utc'])))
    after = data[-1]['created_utc']
    data = getPushshiftSubData(query, after, before, sub)

print(len(data))
subCount
# Write extracted json data into pickle file
with open("Submissions-March-2020.pkl", "wb") as f:
    pickle.dump(subStats, f)
# convert it to pandas datafram
df = pd.DataFrame(data=list(subStats.values()), columns = features + custom_features, index=None)
# write the dataframe into csv file
df.to_csv("Submissions-March-2020.csv", index=False)
####<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12690750545001558, 'fraction_numerical': 0.01494861413889754, 'mean_word_length': 4.508576329331047, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '664752', 'license': 'MIT', 'n_tokens_mistral': 2142, 'n_tokens_neox': 2068, 'n_words': 554, 'path': 'src/data-acquisation-notebooks/reddit-flair-detector.ipynb', 'repo_name': 'macabdul9/midas-internship-task', 'size': '6442'}