<jupyter_start><jupyter_text>we will scrap data like post, comments from reddit and make a dataset will be lated used for Natural language processing.<jupyter_code>from core.selenium_scraper import SeleniumScraper
from core.soup_scraper import SoupScraper
from core.progress_bar import ProgressBar
from core.sql_access import SqlAccess
import time
pip install selenium
reddit_home = 'https://www.reddit.com'
slash = '/r/'
subreddit = 'DataScience'
sort_by = '/hot/'
scroll_n_times = 1000
scrape_comments = True
erase_db_first = True
SQL = SqlAccess()
SelScraper = SeleniumScraper()
BSS = SoupScraper(reddit_home,
                  slash,
                  subreddit)

SelScraper.setup_chrome_browser()
# Collect links from subreddit
links = SelScraper.collect_links(page = reddit_home + 
                                        slash + subreddit + sort_by,
                                 scroll_n_times = scroll_n_times)
# Find the <script> with id='data' for each link
script_data = BSS.get_scripts(urls = links)

# Transforms each script with data into a Python dict, returned as [{}, {}...]
BSS.data = SelScraper.reddit_data_to_dict(script_data = script_data)

print('Scraping data...')
progress = ProgressBar(len(links))
for i, current_data in enumerate(BSS.data):
    progress.update()
    
    BSS.get_url_id_and_url_title(BSS.urls[i],
                                 current_data, i)
    BSS.get_title()
    BSS.get_upvote_ratio()
    BSS.get_score()
    BSS.get_posted_time()
    BSS.get_author()
    BSS.get_flairs()
    BSS.get_num_gold()
    BSS.get_category()
    BSS.get_total_num_comments()
    BSS.get_links_from_post()
    BSS.get_main_link()
    BSS.get_text()
    BSS.get_comment_ids()

print('Scraping data...')
start = time.time()
progress = ProgressBar(len(links))
for i, current_data in enumerate(BSS.data):
    progress.update()
    
    BSS.get_url_id_and_url_title(BSS.urls[i],
                                 current_data, i)
    BSS.get_title()
    BSS.get_upvote_ratio()
    BSS.get_score()
    BSS.get_posted_time()
    BSS.get_author()
    BSS.get_flairs()
    BSS.get_num_gold()
    BSS.get_category()
    BSS.get_total_num_comments()
    BSS.get_links_from_post()
    BSS.get_main_link()
    BSS.get_text()
    BSS.get_comment_ids()

time.sleep(1)
BSS.prepare_data_for_sql(scrape_comments=scrape_comments)

try:
    SQL.create_or_connect_db(erase_first=erase_db_first)
    # [0] = post, [1] = comment, [2] = link
    for i in range(len(BSS.post_data)):
        SQL.insert('post', data = BSS.post_data[i])
        SQL.insert('link', data = BSS.link_data[i])
        
        if scrape_comments:
            SQL.insert('comment', data = BSS.comment_data[i])
except Exception as ex:
    print(ex)
finally:
    SQL.save_changes()

time.sleep(1)
end = time.time()

print(('\nIt took {0} seconds to scrape and store {1} links').format(round(end - start, 1),
                                                                     len(links)))<jupyter_output>Scraping data...
Gathering all the scraped data, and scraping ALL comment data (very slow, dependent on number of comments)

It took 2.0 seconds to scrape and store 0 links<jupyter_text>dataset is saved into sql database<jupyter_code>#if want to enter new data to database
try:
    SQL.create_or_connect_db(erase_first=erase_db_first)
    # [0] = post, [1] = comment, [2] = link
    for i in range(len(BSS.post_data)):
        SQL.insert('post', data = BSS.post_data[i])
        SQL.insert('link', data = BSS.link_data[i])
        
        if scrape_comments:
            SQL.insert('comment', data = BSS.comment_data[i])
except Exception as ex:
    print(ex)
finally:
    SQL.save_changes()
#now lets make dataset
from core.sql_access import SqlAccess
import pandas as pd
#connect to database
SQL = SqlAccess()
SQL.create_or_connect_db()
c = SQL.conn
#retrieve data from database table
all_data = pd.read_sql_query("""
SELECT *
FROM post p 
LEFT JOIN comment c 
    ON p.id = c.post_id
LEFT JOIN link l
	ON p.id = l.post_id;
""", c)
#collect posts and comments data
post = pd.read_sql_query("""
SELECT *
FROM post;
""", c)

comment = pd.read_sql_query("""
SELECT *
FROM comment;
""", c)
#save as csv
all_data.to_csv('data/post_comment_link_data_demo.csv', columns=all_data.columns, index=False)
post.to_csv('data/post_data_demo.csv', columns=post.columns, index=False)
comment.to_csv('data/comment_data_demo.csv', columns=comment.columns, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11170808148645624, 'fraction_numerical': 0.004029550033579583, 'mean_word_length': 3.296153846153846, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '400156', 'license': 'Apache-2.0', 'n_tokens_mistral': 1567, 'n_tokens_neox': 1541, 'n_words': 348, 'path': 'reddit webscrapping and NLP Project/reddit web scrapper.ipynb', 'repo_name': 'shamiulshifat/leisure-time-projects', 'size': '4467'}