<jupyter_start><jupyter_text>Data Collection - Trip Reports This notebook contains the code used to scrape user-generated trail reviews from the Washington Trails Association website. A list of recent trip reports with their urls is located at https://www.wta.org/go-outside/trip-reports.  I opened this page using Selenium as opposed to something like Requests because Selenium allows us to interact with Javascript objects on the page.  In the following code, I scrape all reports listed on the first page, then repeatedly click through to the next most recent page of reviews to get more urls.  An implicit wait is added to allow the Javascript time to load.<jupyter_code>from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
url = 'https://www.wta.org/go-outside/trip-reports'
driver = webdriver.Chrome()
driver.implicitly_wait(30)
driver.get(url)
element = driver.find_elements_by_class_name('listitem-title'):
url = element.get_attribute('href')
hike = element.text
with open('trepurls3.csv', 'a') as csv_file:
    writer = csv.writer(csv_file)
    writer.writerow([str(url), str(hike)])
time.sleep(7)

for m in range(1,100):   
    link = driver.find_element_by_link_text(str(m))
    link.click()
    driver.implicitly_wait(30)
    time.sleep(10)
    for el in driver.find_elements_by_class_name('listitem-title'):
        url = el.get_attribute('href')
        hike = el.text
        with open('trepurls3.csv', 'a') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow([str(url), str(hike)])
    time.sleep(7)
print('finished')
driver.quit()<jupyter_output>finished<jupyter_text>Check the scraped data<jupyter_code>import pandas as pd
tripreports = pd.read_csv("trepurls3.csv", header=None)

tripreportnames = ['url', 'hike']

tripreports.columns = tripreportnames
tripreports.head()
tripreports.describe()
tripreports = tripreports.drop_duplicates()
tripreports.describe()
#close the page where we were just scraping
driver.quit()<jupyter_output><empty_output><jupyter_text>Get the report text and attributes for reports found above<jupyter_code>tripreports['url'][:3]
from bs4 import BeautifulSoup
import json
import csv
import time

chunk = tripreports['url'][0:10000]

for hike in chunk:
    print('getting data for ' + str(hike))
    driver = webdriver.Chrome()
    driver.get(hike)
    driver.implicitly_wait(30)
    
    page = driver.page_source
    driver.quit()
    
    try:
        soup = BeautifulSoup(page, 'html.parser')
    
        title = soup.find('title').get_text()
    
        body = soup.find(attrs={'id':'tripreport-body-text'}).get_text()
        bodyclean = body.replace(',','')
    
        traillinkline = soup.find(attrs={'class':'documentFirstHeading'})
        traillinkhtml = traillinkline.find('a')
        traillink = traillinkhtml.get('href')
    
        dateclass = soup.find(attrs={'class':'elapsed-time'})
        date = dateclass.attrs['datetime']
        dateclean = date.replace(',','')

        cond = soup.find(attrs={'id':'trip-conditions'}).get_text()
        cond = cond.replace('\n', '')
    
        feat = soup.find(attrs={'id':'trip-features'})
    
        thumbs = soup.find(attrs={'class':'tally-total'}).get_text()
    
        with open('peoplereviews3.csv', 'a') as csv_file:
            writer = csv.writer(csv_file)
            writer.writerow([str(hike), str(title), str(traillink), str(bodyclean), str(dateclean), str(cond), str(feat), str(thumbs)])
        
    except:
        continue
        
print('Finished')        
peoplereports = pd.read_csv("peoplereviews3.csv", header=None)

#tripreportnames = ['url', 'hike']

#tripreports.columns = tripreportnames
peoplereports
peoplereports.describe()
peoplereports = peoplereports.drop_duplicates()
peoplereports.describe()<jupyter_output><empty_output><jupyter_text>Optional: Store in postgreSQL database<jupyter_code>from sqlalchemy import create_engine
from sqlalchemy_utils import database_exists, create_database
from sqlalchemy import *
import psycopg2
# Define a database name (we're using a dataset on births, so we'll call it birth_db)
# Set your postgres username/password, and connection specifics
username = 'postgres'
password = '<PASSWORD>'     # change this
host     = 'localhost'
port     = '5432'            # default port that postgres listens on
db_name  = 'trail_project2'
## 'engine' is a connection to a database
## Here, we're using postgres, but sqlalchemy can connect to other things too.
engine = create_engine( 'postgresql://{}:{}@{}:{}/{}'.format(username, password, host, port, db_name) )
print(engine.url)
## create a database (if it doesn't exist)
if not database_exists(engine.url):
    create_database(engine.url)
print(database_exists(engine.url))
## insert data into database from Python (proof of concept - this won't be useful for big data, of course)
peoplereports.to_sql('peoplereports', engine, if_exists='replace')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11946902654867257, 'fraction_numerical': 0.00663716814159292, 'mean_word_length': 4.268008474576271, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '420593', 'license': 'MIT', 'n_tokens_mistral': 1617, 'n_tokens_neox': 1538, 'n_words': 401, 'path': 'DataCollection_TripReports.ipynb', 'repo_name': 'amthomps/HikeIt', 'size': '4972'}