<jupyter_start><jupyter_text>Validated department boundaries vs government units with highest incident share comparisonThe backing theory for this notebook is proving that we will be able to use the government unit with the greatest number of fire incidents are the department's boundary.  We will compare the department boundaries vs the chosen government unit for departments have have a validated boundary.  Validated boundaries are those that have been deemed accurate by department administrators.<jupyter_code>import psycopg2
from psycopg2.extras import RealDictCursor
import pandas as pd
# import geopandas as gpd
# from shapely import wkb
# from shapely.geometry import mapping as to_geojson
# import folium

pd.options.display.max_columns = None
pd.options.display.max_rows = None
#pd.set_option('display.float_format', lambda x: '%.3f' % x)

%matplotlib inline
conn = psycopg2.connect('service=firecares')
nfirs = psycopg2.connect('service=nfirs')<jupyter_output><empty_output><jupyter_text>Processing<jupyter_code>q = """
select id, fdid, state, name
from firestation_firedepartment
where boundary_verified = true;
"""

with nfirs.cursor(cursor_factory=RealDictCursor) as c:
    c.execute(q)
    fds = c.fetchall()
q = """
with fires as (select * from joint_buildingfires
    inner join joint_incidentaddress
    using (fdid, inc_no, inc_date, state, exp_no)
    where fdid = %(fdid)s and state = %(state)s
),
govt_units as (
    select gu.name, gu.source, gu.id, gu.geom, fd.id as fc_id, fd.geom as fd_geom, ST_Distance(addr.geom, ST_Centroid(gu.geom)) as distance_to_headquarters
    from firestation_firedepartment fd
    inner join firecares_core_address addr
        on addr.id = fd.headquarters_address_id
    join usgs_governmentunits gu
    on ST_Intersects(ST_Buffer(addr.geom, 0.05), gu.geom)
    where 
        fd.fdid = %(fdid)s and fd.state = %(state)s and source != 'stateorterritoryhigh'
    )
select gu.fc_id, count(fires), ST_Area(ST_SymDifference(gu.fd_geom, gu.geom)) / ST_Area(gu.fd_geom) as percent_difference_to_verified_boundary, ST_Area(gu.geom), gu.distance_to_headquarters, gu.name, gu.id, gu.source from fires
inner join govt_units gu
on ST_Intersects(fires.geom, gu.geom)
group by gu.name, gu.id, gu.geom, gu.source, gu.distance_to_headquarters, gu.fd_geom, gu.fc_id
order by (count(fires) / (select count(1) from fires)::float) desc;
"""

for fd in fds[62:]:
    with nfirs.cursor(cursor_factory=RealDictCursor) as c:
        print 'Analyzing: {} (id: {} fdid: {} {})'.format(fd['name'], fd['id'], fd['fdid'], fd['state'])
        c.execute(q, dict(fdid=fd['fdid'], state=fd['state']))
        items = c.fetchall()
        df = pd.DataFrame(items)
        df.to_csv('./boundary-incident-share-analysis-{}.csv'.format(fd['id']))<jupyter_output>Analyzing: Fall River Fire Department (id: 81258 fdid: 05095 MA)
Analyzing: Florence Fire Rescue (id: 81599 fdid: 04101 AL)
Analyzing: Berkeley Heights Fire Department (id: 74867 fdid: 20001 NJ)
Analyzing: Orono Fire Department (id: 91180 fdid: J3820 ME)
Analyzing: Des Moines Fire Department (id: 79671 fdid: 77008 IA)
Analyzing: Marshfield Fire and Rescue Department (id: 87956 fdid: 71130 WI)
Analyzing: Flagler County Fire Rescue (id: 81531 fdid: 61091 FL)
Analyzing: Isleton Fire Department (id: 85026 fdid: 34055 CA)
Analyzing: Larchmont Fire Department (id: 86361 fdid: 60025 NY)
Analyzing: Henrietta Fire District (id: 84012 fdid: 28019 NY)
Analyzing: Delta Fire Protection District (id: 79597 fdid: 34130 CA)
Analyzing: St Louis Park Fire Department (id: 95940 fdid: 27229 MN)
Analyzing: Delta Township Fire Department (id: 79600 fdid: 02304 MI)
Analyzing: Berryhill Fire Protection District (id: 74913 fdid: 72010 OK)<jupyter_text>Results<jupyter_code>from glob import glob
df = None
for f in glob("boundary-incident-share-analysis*.csv"):
    if df is not None:
        df = df.append(pd.read_csv(f))
    else:
        df = pd.read_csv(f)
df.rename(columns={'Unnamed: 0': 'rank'}, inplace=True)
selected_government_units = df[df['rank'] == 0].set_index('fc_id')
total_validated_department_count = len(selected_government_units)
perfect_fits = len(selected_government_units[selected_government_units['percent_difference_to_verified_boundary'] == 0])
print 'Perfect fits: {}/{} ({:.2%})'.format(perfect_fits, total_validated_department_count, float(perfect_fits) / total_validated_department_count)
print 'Machine-selected government unit area difference mean: {:.2%}'.format(df[df['rank'] == 0].percent_difference_to_verified_boundary.mean())
selected_government_units['percent_difference_to_verified_boundary'].hist(bins=50)
selected_government_units
df.set_index('fc_id')
df.to_csv('./validated-boundary-vs-government-unit-incident-share.csv')
pd.read_csv('./validated-boundary-vs-government-unit-incident-share.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1117306897976043, 'fraction_numerical': 0.032631144155307726, 'mean_word_length': 5.598092643051771, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '558608', 'license': 'MIT', 'n_tokens_mistral': 1765, 'n_tokens_neox': 1597, 'n_words': 442, 'path': 'analysis/validated-boundaries-vs-government-unit-incident-share.ipynb', 'repo_name': 'tbuffington7/data', 'size': '4842'}