<jupyter_start><jupyter_text>Initialising the columns<jupyter_code>post_headline = []
flare = []
from urllib.parse import urljoin
from bs4 import BeautifulSoup
import requests
import urllib.request as url
from selenium import webdriver
!pip install selenium
BASE_URL = "https://www.reddit.com/r/india/"
page = url.urlopen(BASE_URL)
soup = BeautifulSoup(page, 'html.parser')
soup.title.string
rposts = []
import pandas as pd
import numpy as np
import urllib.request, json
count = 1
URL1 = "https://gateway.reddit.com/desktopapi/v1/subreddits/india?rtj=only&redditWebClient=web2x&app=web2x-client-production&allow_over18=&include=identity&after="
token = "<PASSWORD>"
URL2 = "&layout=card&sort=new&geo_filter=IN"
# URL = URL1 + str(token) + "&dict=" + str(count%10) + URL2
while(len(rposts) < 300000):
    i = 10
    flag = 0
    while(flag != 1):
        if(i < 2):
            break
        URL =  URL1 + str(token) + "&dist=" + str(i) + URL2 
        i = i - 1
        try:
            with urllib.request.urlopen(URL) as url:
                data = json.loads(url.read().decode())
        #         print(type(data['posts']))
                jsonData = data['posts']
                for post in jsonData:
                    if(len(post) == 9):
                        #print(post)
                        if(len(jsonData[post]['flair']) != 0 ):
                            count+=1
                            rposts.append([jsonData[post]['title'],jsonData[post]['flair'][1]['text'], jsonData[post]['score'],  jsonData[post]['numComments'], jsonData[post]['created']])
                        else:
                            count+=1
                            rposts.append([jsonData[post]['title'],'others', jsonData[post]['score'],  jsonData[post]['numComments'], jsonData[post]['created']])
                token = data['token']
                flag = -1
        except:
            pass
    count = count - 1;
#     URL =  URL1 + token + "&dict=" + str(count%11) + URL2 
rposts = pd.DataFrame(rposts,columns=['title','flair', 'score',   'num_comments',  'created'])
rposts
rposts.to_csv("1000000_RedditIndiaData.csv")
rposts = []
import pandas as pd
import numpy as np
import urllib.request, json
BASE_URL = "https://www.reddit.com/r/india/new/"
URL1 = "https://gateway.reddit.com/desktopapi/v1/subreddits/india?rtj=only&redditWebClient=web2x&app=web2x-client-production&allow_over18=&include=identity&after="
token = "<PASSWORD>"
dist = 6
URL2 = "&layout=card&sort=new&geo_filter=IN"
driver = webdriver.Chrome('/Users/sidhartha7/Downloads/chromedriver')
driver.get(BASE_URL)

# URL = URL1 + str(token) + "&dict=" + str(count%10) + URL2
while(len(rposts) < 100000):
    print(len(rposts))
    URL =  URL1 + str(token) + "&dist=" + str(dist) + URL2 

    try:
        with urllib.request.urlopen(URL) as url:
            data = json.loads(url.read().decode())
            #print(type(data['posts']))
            jsonData = data['posts']
            for post in jsonData:
                if(len(post) == 9):
                    #print(post)
                    if(len(jsonData[post]['flair']) != 0 ):
                        count+=1
                        rposts.append([jsonData[post]['title'],jsonData[post]['flair'][1]['text'], jsonData[post]['score'],  jsonData[post]['numComments'], jsonData[post]['created']])
                    else:
                        count+=1
                        rposts.append([jsonData[post]['title'],'others', jsonData[post]['score'],  jsonData[post]['numComments'], jsonData[post]['created']])
            token = data['token']
            dist = data['dist']
            
    except:
        #SELENIUM WEB DRIVER TO LOAD CONTINUOUS PAGE
        driver.execute_script("window.scrollTo(0, window.scrollY + 80000)")
        
        
rposts = pd.DataFrame(rposts,columns=['title','flair', 'score',   'num_comments',  'created'])
rposts
rposts = pd.DataFrame(rposts,columns=['title','flair', 'score',   'num_comments',  'created'])
rposts
rposts.to_csv('40000_data.csv')
rposts.drop_duplicates()<jupyter_output><empty_output><jupyter_text>Scraping data based on date parameter<jupyter_code>subreddit = reddit.subreddit('india')

topics_dict = { "title":[], \
                "score":[], \
                "id":[], "url":[], \
                "comms_num": [], \
                "created": [], \
                "body":[]}

for submission in subreddit.submissions(1533081600, 1535760000):
    topics_dict["title"].append(submission.title)
    topics_dict["score"].append(submission.score)
    topics_dict["id"].append(submission.id)
    topics_dict["url"].append(submission.url)
    topics_dict["comms_num"].append(submission.num_comments)
    topics_dict["created"].append(submission.created)
    topics_dict["body"].append(submission.selftext)

topics_data = pd.DataFrame(topics_dict)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1602221308103661, 'fraction_numerical': 0.02097902097902098, 'mean_word_length': 2.772692009309542, 'pattern_counts': {'":': 7, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '382705', 'license': 'Apache-2.0', 'n_tokens_mistral': 1641, 'n_tokens_neox': 1557, 'n_words': 258, 'path': 'SCRAPPING/ScrapperScript.ipynb', 'repo_name': 'ahtrahdis7/Reddit-Flair-Predictor-', 'size': '4860'}