<jupyter_start><jupyter_text>In these blocks we are crawling the last 100 posts from the hot page of r/worldnews * crawl from article:        title, 		id, 		url, 		permalink, 		date, 		number of comments,		karma        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>ids = []
titles = []
urls = []
permalinks = []
karmas = []
dates = []
num_comments = []

submissions = list(reddit.subreddit("worldnews").hot())
for submission in submissions:
    if(submission.num_comments < 50) :
        continue
    if(submission.is_self):
        continue
    ids.append(submission.id)
    titles.append(submission.title)
    urls.append(submission.url)
    permalinks.append(submission.permalink)
    karmas.append(submission.upvote_ratio *submission.score)
    dates.append(datetime.datetime.fromtimestamp(submission.created_utc))
    num_comments.append(submission.num_comments)
    
worldnews = pd.DataFrame(list(zip(ids, titles, urls, permalinks, karmas, dates, num_comments)),
                              columns = ['id','title','url','permalink','karma','date','comments'])
worldnews.head()
import os
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\articlesReddit\ '
name = 'worldnews' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
worldnews.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output><jupyter_text>In these blocks we are crawling the last 100 posts from the hot page of r/news * crawl from article:        title, 		id, 		url, 		permalink, 		date, 		number of comments,		karma        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>ids = []
titles = []
urls = []
permalinks = []
karmas = []
dates = []
num_comments = []

submissions = list(reddit.subreddit("news").hot())
for submission in submissions:
    if(submission.num_comments < 50) :
        continue
    if(submission.is_self):
        continue
    ids.append(submission.id)
    titles.append(submission.title)
    urls.append(submission.url)
    permalinks.append(submission.permalink)
    karmas.append(submission.upvote_ratio *submission.score)
    dates.append(datetime.datetime.fromtimestamp(submission.created_utc))
    num_comments.append(submission.num_comments)
    
news = pd.DataFrame(list(zip(ids, titles, urls, permalinks, karmas, dates, num_comments)),
                              columns = ['id','title','url','permalink','karma','date','comments'])
news.head()
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\articlesReddit\ '
name = 'news' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
news.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output><jupyter_text>In these blocks we are crawling the last 100 posts from the hot page of r/politics * crawl from article:        title, 		id, 		url, 		permalink, 		date, 		number of comments,		karma        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>ids = []
titles = []
urls = []
permalinks = []
karmas = []
dates = []
num_comments = []

submissions = list(reddit.subreddit("politics").hot())
for submission in submissions:
    if(submission.num_comments < 50) :
        continue
    if(submission.is_self):
        continue
    ids.append(submission.id)
    titles.append(submission.title)
    urls.append(submission.url)
    permalinks.append(submission.permalink)
    karmas.append(submission.upvote_ratio *submission.score)
    dates.append(datetime.datetime.fromtimestamp(submission.created_utc))
    num_comments.append(submission.num_comments)
    
politcs = pd.DataFrame(list(zip(ids, titles, urls, permalinks, karmas, dates, num_comments)),
                              columns = ['id','title','url','permalink','karma','date','comments'])
politcs.head()
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\articlesReddit\ '
name = 'politics' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
politcs.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output><jupyter_text>In these blocks we are crawling the last 100 posts from the hot page of r/conservative * crawl from article:        title, 		id, 		url, 		permalink, 		date, 		number of comments,		karma        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>ids = []
titles = []
urls = []
permalinks = []
karmas = []
dates = []
num_comments = []

submissions = list(reddit.subreddit("conservative").hot())
for submission in submissions:
    if(submission.num_comments < 50) :
        continue
    if(submission.is_self):
        continue
    ids.append(submission.id)
    titles.append(submission.title)
    urls.append(submission.url)
    permalinks.append(submission.permalink)
    karmas.append(submission.upvote_ratio *submission.score)
    dates.append(datetime.datetime.fromtimestamp(submission.created_utc))
    num_comments.append(submission.num_comments)
    
conservative = pd.DataFrame(list(zip(ids, titles, urls, permalinks, karmas, dates, num_comments)),
                              columns = ['id','title','url','permalink','karma','date','comments'])
conservative.head()
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\articlesReddit\ '
name = 'conservative' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
conservative.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output><jupyter_text>Now for each created dataframe, we are going to seek the immediet replies* crawl from comments:        article_id, 		id, 		body, 		karma, 		profile,        date,        parent_id (which is the article_id if first layer comment)        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>article_ids = []
ids = []
bodies = []
upvotes = []
#profiles = []
dates = []
parents = []
subreddits = []
links = list(worldnews.id)
for link in links:
    print(link)
    submission = reddit.submission(id=link)
    submission.comments.replace_more(limit=15)
    print(len(list(submission.comments)))
    for top_level_comment in submission.comments:
        if top_level_comment.id == None:
            continue
        article_ids.append(link)
        ids.append(top_level_comment.id)
        print(top_level_comment.id)
        bodies.append(str(top_level_comment.body.encode('ascii','ignore'))[2:-1])
        upvotes.append(top_level_comment.score)
        dates.append(datetime.datetime.fromtimestamp(top_level_comment.created_utc))
        parents.append(link)
        subreddits.append('worldnews')
        top_level_comment.replies.replace_more(limit=15)
        for com in top_level_comment.replies:
            article_ids.append(link)
            ids.append(com.id)
            bodies.append(str(com.body.encode('ascii','ignore'))[2:-1])
            upvotes.append(com.score)
            dates.append(datetime.datetime.fromtimestamp(com.created_utc))
            parents.append(str(com.parent_id)[3:])
            subreddits.append('worldnews')
       
    
            
worldnewsComs = pd.DataFrame(list(zip(article_ids, ids, bodies, upvotes, dates, parents, subreddits)),
                              columns = ['article_id','id','body','upvotes','date','parent_id','subreddit'])
worldnewsComs.head()
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\commentsReddit\worldnews '
name = 'worldnewsComs' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
worldnewsComs.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output><jupyter_text>Now for each created dataframe, we are going to seek the immediet replies and the replies to them (not further)* crawl from comments:        article_id, 		id, 		body, 		karma, 		profile,        date,        parent_id (which is the article_id if first layer comment)        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>article_ids = []
ids = []
bodies = []
upvotes = []
#profiles = []
dates = []
parents = []
subreddits = []
links = list(news.id)
cou =0
for link in links:
    print(cou)
    cou = cou +1
    print(link)
    submission = reddit.submission(id=link)
    submission.comments.replace_more(limit=15)
    print(len(list(submission.comments)))
    for top_level_comment in submission.comments:
        if top_level_comment.id == None:
            continue
        article_ids.append(link)
        ids.append(top_level_comment.id)
        bodies.append(str(top_level_comment.body.encode('ascii','ignore'))[2:-1])
        upvotes.append(top_level_comment.score)
        dates.append(datetime.datetime.fromtimestamp(top_level_comment.created_utc))
        parents.append(link)
        subreddits.append('news')
        top_level_comment.replies.replace_more(limit=15)
        for com in top_level_comment.replies:
            article_ids.append(link)
            ids.append(com.id)
            bodies.append(str(com.body.encode('ascii','ignore'))[2:-1])
            upvotes.append(com.score)
            dates.append(datetime.datetime.fromtimestamp(com.created_utc))
            parents.append(str(com.parent_id)[3:])
            subreddits.append('news')
       
newsComs = pd.DataFrame(list(zip(article_ids, ids, bodies, upvotes,  dates, parents, subreddits)),
                              columns = ['article_id','id','body','upvotes','date','parent_id','subreddit'])
newsComs.head()
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\commentsReddit\news '
name = 'newsComs' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
newsComs.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output><jupyter_text>Now for each created dataframe, we are going to seek the immediet replies and the replies to them (not further)* crawl from comments:        article_id, 		id, 		body, 		karma, 		profile,        date,        parent_id (which is the article_id if first layer comment)        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>article_ids = []
ids = []
bodies = []
upvotes = []
#profiles = []
dates = []
parents = []
subreddits = []
links = list(politcs.id)
cou =0
for link in links:
    print(cou)
    cou = cou +1
    print(link)
    submission = reddit.submission(id=link)
    submission.comments.replace_more(limit=15)
    print(len(list(submission.comments)))
    for top_level_comment in submission.comments:
        if top_level_comment.id == None:
            continue
        article_ids.append(link)
        ids.append(top_level_comment.id)
        bodies.append(str(top_level_comment.body.encode('ascii','ignore'))[2:-1])
        upvotes.append(top_level_comment.score)
        dates.append(datetime.datetime.fromtimestamp(top_level_comment.created_utc))
        parents.append(link)
        subreddits.append('politcs')
        top_level_comment.replies.replace_more(limit=15)
        for com in top_level_comment.replies:
            article_ids.append(link)
            ids.append(com.id)
            bodies.append(str(com.body.encode('ascii','ignore'))[2:-1])
            upvotes.append(com.score)
            dates.append(datetime.datetime.fromtimestamp(com.created_utc))
            parents.append(str(com.parent_id)[3:])
            subreddits.append('politcs')
       
politcsComs = pd.DataFrame(list(zip(article_ids, ids, bodies, upvotes,  dates, parents, subreddits)),
                              columns = ['article_id','id','body','upvotes','date','parent_id','subreddit'])
politcsComs.head()
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\commentsReddit\politics '
name = 'politicsComs' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
politcsComs.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output><jupyter_text>Now for each created dataframe, we are going to seek the immediet replies and the replies to them (not further)* crawl from comments:        article_id, 		id, 		body, 		karma, 		profile,        date,        parent_id (which is the article_id if first layer comment)        * save it on a dataframe with the title: (subreddit+date)<jupyter_code>article_ids = []
ids = []
bodies = []
upvotes = []
#profiles = []
dates = []
parents = []
subreddits = []
links = list(conservative.id)
cou =0
for link in links:
    print(cou)
    cou = cou +1
    print(link)
    submission = reddit.submission(id=link)
    submission.comments.replace_more(limit=15)
    print(len(list(submission.comments)))
    for top_level_comment in submission.comments:
        if top_level_comment.id == None:
            continue
        article_ids.append(link)
        ids.append(top_level_comment.id)
        bodies.append(str(top_level_comment.body.encode('ascii','ignore'))[2:-1])
        upvotes.append(top_level_comment.score)
        dates.append(datetime.datetime.fromtimestamp(top_level_comment.created_utc))
        parents.append(link)
        subreddits.append('conservative')
        top_level_comment.replies.replace_more(limit=15)
        for com in top_level_comment.replies:
            article_ids.append(link)
            ids.append(com.id)
            bodies.append(str(com.body.encode('ascii','ignore'))[2:-1])
            upvotes.append(com.score)
            dates.append(datetime.datetime.fromtimestamp(com.created_utc))
            parents.append(str(com.parent_id)[3:])
            subreddits.append('conservative')
       
conservativeComs = pd.DataFrame(list(zip(article_ids, ids, bodies, upvotes,  dates, parents, subreddits)),
                              columns = ['article_id','id','body','upvotes','date','parent_id','subreddit'])
conservativeComs.head()
path = r'C:\Users\nikob\Desktop\UnUtrecht\EmpathyDetection\code\SecondRQ\TwitterReddit\data\commentsReddit\conservative '
name = 'conservativeComs' + str(datetime.date.today().strftime("%d%m%y")) +".csv"
conservativeComs.to_csv(os.path.join(path[:-1],name))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13399396811719086, 'fraction_numerical': 0.005026569007611662, 'mean_word_length': 3.673489932885906, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '576954', 'license': 'MIT', 'n_tokens_mistral': 4685, 'n_tokens_neox': 4559, 'n_words': 767, 'path': 'crawling/RedditExploration.ipynb', 'repo_name': 'nikobent/EmpathyDetection', 'size': '13926'}