<jupyter_start><jupyter_text>You need GeckoDriver installed in order for the webdriver to work with Selenium<jupyter_code>from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.expected_conditions import presence_of_element_located
from bs4 import BeautifulSoup
import pandas as pd
import requests
import time
import sys
firefox_driver_path = 'C:/Users/Jsgri/Desktop/School/Fall 2020/CIS 4680/Project 1'
webdriver=webdriver.Firefox(firefox_driver_path)
d = {'url': [], 'review': [], 'review_dates':[], 'category':[]}
df = pd.DataFrame(data=d)
URLs = pd.read_csv('urls.csv')
urls = URLs['url']
for url in urls:
    category = URLs[URLs['url'] == url]['category'].to_string()
    webdriver.get(url)
    webdriver.execute_script("window.scrollTo(0, 7000);")
    results = webdriver.find_elements_by_id("cm-cr-dp-review-list")
    html = webdriver.execute_script("return document.documentElement.outerHTML")

    soup = BeautifulSoup(html,'html.parser')
    reviews_words = soup.find_all("div", class_="a-expander-content reviewText review-text-content a-expander-partial-collapse-content")
    review_dates = soup.find_all("span", class_ = "a-size-base a-color-secondary review-date")
    for i in range(len(reviews_words)):
        #print(reviews_words[i])
        d = {'url': [url], 'review': [reviews_words[i]], 'review_dates':[review_dates[i]], 'category':[category]}
        d = pd.DataFrame(data=d)
        df = df.append(d)
df
df.to_csv('keyboard_reviews.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13262910798122066, 'fraction_numerical': 0.008802816901408451, 'mean_word_length': 5.583011583011583, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '268361', 'license': 'Apache-2.0', 'n_tokens_mistral': 582, 'n_tokens_neox': 551, 'n_words': 114, 'path': 'Selenium_Webscraping.ipynb', 'repo_name': 'Calvin-Truong/Keyboard-Sentiment-Analysis', 'size': '1704'}