<jupyter_start><jupyter_text>Cleaning up the codeSteps:1. Compile list of possible police-dept options from dropdown menu search box2. From each one retrieve:  - Likelihood of force (black vs white) based on population  - Likelihood of force (black vs white) based on arrests  - No. of police officers that would be flagged under LAPD standards  - No. of police officers that would be flagged under NYPD standards  - No. of police officers that would be flagged under Chicago PD standards  Notes: No. of police officers flagged only appears when the page is visited by a browser. If you use `requests` the `` where these numbers would appears shows up empty.<jupyter_code>import pandas as pd
from selenium import webdriver
import requests
from pathlib import Path
from tools import tree
from datetime import datetime as dt

today = dt.today().strftime("%Y-%m-%d")
RAW_DATA = Path("../data/raw/")
INTERIM_DATA = Path("../data/interim/")
PROCESSED_DATA = Path("../data/processed/")
FINAL_DATA = Path("../data/final/")
BASE_URL = 'http://force.nj.com'
browser = webdriver.Chrome("./chromedriver")
browser.get(BASE_URL)
police_depts_list = browser.find_element_by_id("mylist")
police_depts = police_depts_list.find_elements_by_tag_name("option")
test_option = police_depts[0]
test_option.get_attribute('value')
test_option.get_attribute('innerHTML')
depts_list = []
for dept in police_depts:
    dept_info = {}
    dept_info['name'] = dept.get_attribute("innerHTML").strip()
    dept_info['relative_url'] = dept.get_attribute("value")
    depts_list.append(dept_info)
test_dept = depts_list[0]
browser.get(BASE_URL + test_dept['relative_url'])
rbk = browser.find_element_by_class_name("racial_breakdown")
left_div = rbk.find_element_by_class_name("left")
right_div = rbk.find_element_by_class_name("right")
left_div.find_element_by_class_name("important_num2").get_property("innerHTML")
left_div.find_element_by_class_name("important_num1").get_property("innerHTML")
def get_likelihood_numbers(div_name = 'left'):
    race_breakdown = browser.find_element_by_class_name("racial_breakdown")
    div = rbk.find_element_by_class_name(div_name)
    based_on = div.find_element_by_class_name("important_num2").get_property("innerHTML").strip()
    likelihood = div.find_element_by_class_name("important_num1").get_property("innerHTML").strip()
    return (based_on, likelihood)
get_likelihood_numbers('right')
early_warning = browser.find_element_by_class_name("earlywarning")
early_warning.find_elements_by_class_name("important_num_red2")
def get_flagging_officer_numbers():
    early_warning = browser.find_element_by_class_name("earlywarning")
    elements = early_warning.find_elements_by_class_name("important_num_red2")
    if len(elements) == 6:
        first_city = elements[0].get_property("innerHTML")
        first_city_n = elements[1].get_property("innerHTML")
        second_city = elements[2].get_property("innerHTML")
        second_city_n = elements[3].get_property("innerHTML")
        third_city = elements[4].get_property("innerHTML")
        third_city_n = elements[5].get_property("innerHTML")
    return (first_city, first_city_n, second_city, second_city_n, third_city, third_city_n)
get_flagging_officer_numbers()
for dept in depts_list:
    # go to page
    browser.get(BASE_URL + dept['relative_url'])
    
    # get likelihood numbers
    # by population
    try:
        (prop, number) = get_likelihood_numbers('left')
        dept[prop] = number
    except:
        dept['population'] = 'Not found'
    # by arrests
    try:
        (prop, number) = get_likelihood_numbers('right')
        dept[prop] = number
    except:
        dept['arrests'] = 'Not found'
    
    # get no of officers that would be flagged in other cities
    try:
        (first_city, first_city_n, second_city, second_city_n, third_city, third_city_n) = get_flagging_officer_numbers()
        dept[first_city] = first_city_n
        dept[second_city] = second_city_n
        dept[third_city] = third_city_n
    except:
        dept['Los Angeles'] = 'Not found'
        dept['New York City'] = 'Not found'
        dept['Chicago'] = 'Not found'
        
pd.DataFrame(depts_list)<jupyter_output><empty_output><jupyter_text>***Restarting from `src/tools/scraper.py`<jupyter_code>import pandas as pd
from selenium import webdriver
from pathlib import Path
from datetime import datetime as dt 
today = dt.today().strftime("%Y-%m-%d")

# THIS_FILE = Path(__file__)
# THIS_DIR = THIS_FILE.parent
# CHROMEDRIVER_PATH = THIS_DIR.joinpath("./../../notebooks/chromedriver")
# INTERIM_DATA = THIS_DIR.joinpath("./../../data/interim/")
# PROCESSED_DATA = THIS_DIR.joinpath("./../../data/processed/")

BASE_URL = "http://force.nj.com"
browser = webdriver.Chrome("./chromedriver")
# Compile list police dept of options from dropdown menu
browser.get(BASE_URL)

police_depts_list = browser.find_element_by_id("mylist")
police_depts = police_depts_list.find_elements_by_tag_name("option")
depts_list = []
for dept in police_depts:
    dept_info = {}
    dept_info['name'] = dept.get_attribute("innerHTML").strip()
    dept_info['relative_url'] = dept.get_attribute("value")
    depts_list.append(dept_info)
def get_likelihood_numbers(div_name = 'left'):
    """Retrieves 'likelihood of use of force' numbers from force.nj.com website. These numbers are in 2 divs inside the 'race_breakdown' div.

    Parameters
    ----------
    div_name : str, optional
        Either the left or the right div, by default 'left'

    Returns
    -------
    touple : touple
        A touple containing the property name and likelihood number (text) for either the 'left' or 'right' div.
    """
    race_breakdown = browser.find_element_by_class_name("racial_breakdown")
    div = race_breakdown.find_element_by_class_name(div_name)
    based_on = div.find_element_by_class_name("important_num2").get_property("innerHTML").strip().replace(",", "")
    likelihood = div.find_element_by_class_name("important_num1").get_property("innerHTML").strip().replace(",", "")
    more_or_less = div.find_elements_by_class_name("important_num2")[-1].get_property("innerHTML").strip()

    return (based_on, likelihood, more_or_less)
def get_flagging_officer_numbers():
    """Retrieve the 'number of officers that would be flagged in other agencies' from force.nj.com website. These numbers are inside the 'earlywarning' div.

    Returns
    -------
    touple : touple
        A six-part touple containing the city name and number for Los Angeles, New York City, and Chicago.
    """
    early_warning = browser.find_element_by_class_name("earlywarning")
    elements = early_warning.find_elements_by_class_name("important_num_red2")
    if len(elements) == 6:
        first_city = elements[0].get_property("innerHTML").strip().replace("'s", "").replace("'", "")
        first_city_n = elements[1].get_property("innerHTML").strip()
        second_city = elements[2].get_property("innerHTML").strip().replace("'s", "").replace("'", "")
        second_city_n = elements[3].get_property("innerHTML").strip()
        third_city = elements[4].get_property("innerHTML").strip().replace("'s", "").replace("'", "")
        third_city_n = elements[5].get_property("innerHTML").strip()
    return (first_city, first_city_n, second_city, second_city_n, third_city, third_city_n)
browser.get(BASE_URL + depts_list[0]['relative_url'])
get_likelihood_numbers()
def get_rate_of_force():
    """Retrieve the rate of force ('this dept uses force at a rate higher than X other depts').
        
    Returns
    -------
    sentnce : str
       `innerHTML` of h3 inside the rank_five_years section.
    """
    rank_five_years = browser.find_element_by_id("rank_five_years")
    rate_of_force_sentence = rank_five_years.find_element_by_class_name('third_label').get_property('innerHTML')
    clean_sentence = " ".join(rate_of_force_sentence.split()).replace("<div class=\"important_num2\">", "").replace("</div>", "")
    return clean_sentence
get_rate_of_force()
def get_pd_info():
    """Retrieve the rate of force ('this dept uses force at a rate higher than X other depts').
        
    Returns
    -------
    sentnce : str
       `innerHTML` of h3 inside the rank_five_years section.
    """
    pd_info = browser.find_element_by_class_name("pd_info")
    h1 = pd_info.find_element_by_class_name("biggest_hed").get_property("innerHTML")
    second_label = pd_info.find_element_by_class_name("second_label").get_property("innerHTML")
    patrol_area = pd_info.find_element_by_class_name("town_description").find_element_by_class_name("left").find_element_by_class_name("town_label").get_property('innerHTML')

    return (h1, second_label, patrol_area)
get_pd_info()
def get_incidents_table():
    """Retrieve the incidents table presented by default in each police depts page.
        
    Returns
    -------
    table : pd.DataFrame
       table containing `n_rows` officers per dept
    """
    incidents_table = browser.find_element_by_id("incidents_table").get_property('innerHTML')
    return incidents_table
pd_info = browser.find_element_by_class_name("pd_info")
town_info = pd_info.find_element_by_class_name("town_description")
town_info.find_element_by_class_name("right").find_element_by_class_name("town_label").get_property("innerHTML")
for dept in depts_list[:15]:
    # go to page
    browser.get(BASE_URL + dept['relative_url'])

    print(f"Getting info for {dept['name']}")
    
    # get likelihood numbers
    # by population
    try:
        (prop, number) = get_likelihood_numbers('left')
        dept[prop] = number
    except:
        dept['population'] = 'Not found'
    # by arrests
    try:
        (prop, number) = get_likelihood_numbers('right')
        dept[prop] = number
    except:
        dept['arrests'] = 'Not found'
    
    # get no of officers that would be flagged in other cities
    try:
        (first_city, first_city_n, second_city, second_city_n, third_city, third_city_n) = get_flagging_officer_numbers()
        dept[first_city] = first_city_n
        dept[second_city] = second_city_n
        dept[third_city] = third_city_n
    except:
        dept['Los Angeles'] = 'Not found'
        dept['New York City'] = 'Not found'
        dept['Chicago'] = 'Not found'

browser.close()
data = pd.DataFrame(depts_list)
data['full_url'] = BASE_URL + data['relative_url']
data
#data.to_csv(INTERIM_DATA / f'rough-scrape-data-{today}.csv', encoding = 'utf-8', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12218618847768027, 'fraction_numerical': 0.0037199542159481115, 'mean_word_length': 4.469483568075117, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '296846', 'license': 'RSA-MD', 'n_tokens_mistral': 3408, 'n_tokens_neox': 3333, 'n_words': 703, 'path': 'notebooks/cleaning-up-for-scraper-py.ipynb', 'repo_name': 'chekos/scraping-force-report', 'size': '10484'}