<jupyter_start><jupyter_text>Webscrape college applicationsThis notebook was developed following the `ColEarlyDecScrape.ipynb` prototype. Step One: Prepare Directory Information<jupyter_code>import os
import sys
import datetime
import numpy as np
import pandas as pd
from pandas import Series, DataFrame
# Load directory data.
IPEDSfile = pd.read_stata('../../statadata/IPEDSDirInfo02to16smlr.dta', preserve_dtypes=False)

# Keep 2 and 4yr institutions.
IPEDSfile['filter'] = np.where((IPEDSfile['sector']=='Public, 4-year or above') | 
                               (IPEDSfile['sector']=='Public, 2-year') |
                               (IPEDSfile['sector']=='Private not-for-profit, 4-year or above') |
                               (IPEDSfile['sector']=='Private not-for-profit, 2-year'), 1, 0)
IPEDSfile = IPEDSfile[IPEDSfile['filter']==1]

# Remove www. prefix from webaddress.
IPEDSfile['rootdom'] = IPEDSfile.loc[:, 'webaddr'].replace(regex=True, to_replace='www.', value='')

# Remove miscellaneous slashes & other from webaddress.
IPEDSfile['rootdom'] = IPEDSfile.loc[:, 'rootdom'].replace(regex=True, to_replace=r'/', value='')
IPEDSfile['rootdom'] = IPEDSfile.loc[:, 'rootdom'].replace(regex=True, to_replace=r'HTTPS:', value='')
IPEDSfile['rootdom'] = IPEDSfile.loc[:, 'rootdom'].replace(regex=True, to_replace=r'https:', value='')
IPEDSfile['rootdom'] = IPEDSfile.loc[:, 'rootdom'].replace(regex=True, to_replace=r'about', value='')

# Reset the data frame's index.
IPEDSfile = IPEDSfile.reset_index()
# This cell copes with unanticipated errors and/or connection time-out problems.
# Keep record of last unsuccessful iteration here:
#    Encountered CAPTCHA at stillman.edu               April 17, 2018
#    Encountered NaN after alaskapacific.edu           April 18, 2018
#    Encountered CAPTCHA at buc.edu                    April 18, 2018
#    Program STALLED at arapahoe.edu                   April 18, 2018
#    Program STALLED at howard.edu                     April 19, 2018
#    Took a break from project at babson.edu           April 19, 2018
#    Took a break from project at marrimack.edu        April 25, 2018

# Get user input regarding previous scrape attempts / error log above.
print('Enter the school starting school root domain name. No entry will start at beginning of the list.')
start_school = input()

# If user provided starting school, remove preceeding schools.
if start_school != '':
    new_school_loc = IPEDSfile[IPEDSfile['rootdom'] == start_school].index.tolist()[0]
    IPEDSfile = IPEDSfile[new_school_loc:]
# Create list of domains to scrape & check results.
weblist = IPEDSfile['rootdom']
weblist.head()
# This cell writes weblist to a csv that can be used in App_Rec_Train/aaa_appcodes.do
with open('aaa_appcodestarter' + '.csv', mode='w') as csvfile:
            print('webindx', file = csvfile)
            for csv_lines in weblist:
                print(csv_lines, file = csvfile)
csvfile.close<jupyter_output><empty_output><jupyter_text>Step Two: Prepare Browser Object<jupyter_code>from time import sleep
from selenium import webdriver
import os
import requests

# Short list of url domains set aside for testing purposes.
# weblist = ['babson.edu', 'boston.edu', 'uwec.edu', 'wisc.edu']
# weblist
# browser = webdriver.Chrome()
browser = webdriver.Firefox()
print('Loaded Browser Here')<jupyter_output><empty_output><jupyter_text>Step Three: Scrape For Each School<jupyter_code># Define a list to store record of errors.
log_list = []

# Iteratively loop through each institution's website.
for school in weblist:
    print('STARTING SCHOOL', str(school).upper())
    log_list.append('STARTING SCHOOL ' + str(school).upper())
    searchstr = ''.join((r'https://www.google.com/search?q=',
                         r'application+admission+AND+(printable+OR+paper+OR+mail)+site:', 
                         school, r'+filetype:pdf'))
    log_list.append('Search string is ' + searchstr)
    browser.get(searchstr)
    results = browser.find_elements_by_css_selector('h3 > a')

    # Test if there were results.
    if len(results) == 0:
        try:
            # Occassionally Google responds with CAPTCHA verification challenge.
            # If Google responds with CAPTCHA routine will wait for user to complete CAPTCHA challenge.
            check_captcha = browser.find_element_by_partial_link_text('Why did this happen')
            print('At school, ', school.lower(), 'Google responded with CAPTCHA - Will wait for user input.', end='\n\n')
            log_list.append(school + '-' + '.' + '-EncounteredCAPTCHA-' + '.')
            discarded_wait = input()
            results = browser.find_elements_by_css_selector('h3 > a')
        except:
            # Occasionally Google responds with no results.
            print('No results for school ', school.lower())
            log_list.append(school + '-' + '.' + '-NoResults-' + '.')
            sleep(.5)

    # Test the number of results. Download up to the first three results.
    if len(results) < 3:
        doc_count = len(results)
        log_list.append('There were {} results from Google. Will download all.'.format(len(results)))
        sleep(.1)
    else:
        doc_count = 3
        log_list.append('There were 10 or more results from Google. Will download first three.')
        sleep(.2)

    # From above if len(results) == 0 then doc_count will also be zero which will skip this loop.
    for i in range(doc_count):
        filelink = results[i].get_attribute('href')
        try:
            pdf = requests.get(filelink)
            fname = ''.join((school, str(i), '.pdf'))
            open(os.path.join('pprapps', fname), 'wb').write(pdf.content)
            log_list.append('Filelink = ' + filelink)
            log_list.append('Saved as : ' + fname)
            log_list.append('Header info :' + str(pdf.headers))
        except ConnectionError:
            print('There was a ConnectionError on the {}th iteration at : {}'.format(str(i), school.lower()))
            log_list.append(school + '-' + str(i) + '-ConnectionError-' + filelink)
            sleep(.1)
        except:
            print('There was an UnspecifiedError on the {}th iteration at : {}'.format(str(i), school))
            log_list.append(school + '-' + str(i) + '-UnspecifiedError-' + filelink)
            sleep(.1)
# Save the error log to a file for later reference.
with open('log_' + str(datetime.datetime.now())[2:16].replace(" ", "-").replace(":","") + '.log',
          mode='w') as logfile:
            print('This is the error log file from {}'.format(str(datetime.datetime.now())), file = logfile)
            for log_lines in log_list:
                print(log_lines, file = logfile)
logfile.close<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1253147681824915, 'fraction_numerical': 0.010665086653829063, 'mean_word_length': 3.1120584652862364, 'pattern_counts': {'":': 1, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '137212', 'license': 'MIT', 'n_tokens_mistral': 2150, 'n_tokens_neox': 2017, 'n_words': 567, 'path': 'ColAppScrape.ipynb', 'repo_name': 'adamrossnelson/CollegeApps', 'size': '6751'}