<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My\ Drive/Insight/Project/Reddit\ images
!pip install praw
##Upload reddit credentials so sensitive info is not visible 

from google.colab import files
src = list(files.upload().values())[0]
open('redcreds.py','wb').write(src)
import redcreds
import praw 
reddit = praw.Reddit(client_id = redcreds.my_client_id,client_secret=redcreds.my_client_secret, password=<PASSWORD>, user_agent=redcreds.user_agent, username=redcreds.username)
import requests, re, urllib
import pandas as pd

counter=0
hot_posts = reddit.subreddit('malefashion').top('all', limit = None)

post_list1=[]
counter=0
for post in hot_posts:
  if post.link_flair_text == 'WIWT':
    
    url = (post.url)
    
    list1=[]

    #print(post.url)
    list1.append(post.url)
    #print(post.upvote_ratio)
    list1.append(post.upvote_ratio)
    #print(post.score)
    list1.append(post.score)
    #print(post.id)
    list1.append(post.id)

    counter+=1
    print('This is post #'+ str(counter))
    post_list1.append(list1)

    
post_list1=pd.DataFrame(post_list1)
post_list1


    
    
## image extraction stuff 

filename=re.sub('[^A-Za-z0-9]+', '', post.title)

    filename=str(str(counter)+'+'+filename+'+'+str(post.upvote_ratio))

    counter+=1

    urllib.request.urlretrieve(url, filename+'.jpg')

### Pushshift.io stuff 

import math
import json
import requests
import itertools
import numpy as np
import time
from datetime import datetime, timedelta
def give_me_intervals(start_at, number_of_days_per_interval = 3):
    
    end_at = math.ceil(datetime.utcnow().timestamp())
        
    ## 1 day = 86400,
    period = (86400 * number_of_days_per_interval)

    end = start_at + period
    yield (int(start_at), int(end))

    padding = 1
    while end <= end_at:
        start_at = end + padding
        end = (start_at - padding) + period
        yield int(start_at), int(end)

def make_request(uri, max_retries = 5):
    def fire_away(uri):
        response = requests.get(uri)
        assert response.status_code == 200
        return json.loads(response.content)

    current_tries = 1
    while current_tries < max_retries:
        try:
            response = fire_away(uri)
            return response
        except:
            time.sleep(.150)
            current_tries += 1

    return fire_away(uri)
 
def map_posts(posts):
  return list(map(lambda post: {'id': post['id'],'created_utc': post['created_utc'],'prefix': 't4_'}, posts))
def pull_posts_for(subreddit, start_at, end_at):

    SIZE = 500
    URI_TEMPLATE = r'https://api.pushshift.io/reddit/search/submission?subreddit={}&after={}&before={}&size={}'
    
    post_collections = map_posts( \
        make_request( \
            URI_TEMPLATE.format(subreddit, start_at, end_at, SIZE))['data'])

    n = len(post_collections)
    while n == SIZE:
        last = post_collections[-1]
        new_start_at = last['created_utc'] - 10
        
        more_posts = map_posts( \
            make_request( \
                URI_TEMPLATE.format(subreddit, new_start_at, end_at, SIZE))['data'])
        
        n = len(more_posts)
        post_collections.extend(more_posts)

    return post_collections
subreddit = 'Malefashion'

end_at = math.ceil(datetime.utcnow().timestamp())
start_at = math.floor((datetime.utcnow() - timedelta(days=10)).timestamp())

posts = pull_posts_for(subreddit, start_at, end_at)

print('found:', len(posts))
print('unique:', len(np.unique([ post['id'] for post in posts ])))
## assert no dups,
sorted_by_occurence = sorted([ (k, len(list(g))) for k,g in itertools.groupby(posts, lambda x: x['id']) ], key=lambda x: x[1], reverse=True)
sorted_by_occurence[:10]
## quick look at the posts,
posts[:10]
posts_from_reddit = []
comments_from_reddit = []

TIMEOUT_AFTER_COMMENT_IN_SECS = .250

for submission_id in np.unique([ post['id'] for post in posts ]):
    submission = reddit.submission(id=submission_id)

    posts_from_reddit.append(submission)

    submission.comments.replace_more(limit=None)
    for comment in submission.comments.list():
        comments_from_reddit.append(comment)
        
        if TIMEOUT_AFTER_COMMENT_IN_SECS > 0:
            time.sleep(TIMEOUT_AFTER_COMMENT_IN_SECS)

len(posts_from_reddit)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12853529547022305, 'fraction_numerical': 0.0133363991722235, 'mean_word_length': 3.6424759871931696, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '285071', 'license': 'MIT', 'n_tokens_mistral': 1537, 'n_tokens_neox': 1497, 'n_words': 290, 'path': 'web scrapers/Reddit_scrapper.ipynb', 'repo_name': 'jimmy-io/Insighht_project-', 'size': '4356'}