<jupyter_start><jupyter_text>Import required libraries<jupyter_code>import httpx
import json
import math
import os
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>Setting mapbox access tokenYou can easily get your own mapbox access token via the following link: https://account.mapbox.com/access-tokens/<jupyter_code>mapbox_access_token = '<KEY>'
px.set_mapbox_access_token(mapbox_access_token)<jupyter_output><empty_output><jupyter_text>Configuration **load_from_file**  set to True after having run the notebook once to prevent re-scraping the data when already available<jupyter_code>load_from_file = True<jupyter_output><empty_output><jupyter_text>Helper functions<jupyter_code>def combine_csvs_in_dir_to_df(dir_name):
    df = pd.DataFrame()
    for root, directories, files in os.walk(dir_name):
        for file in tqdm(files):
            df = df.append(pd.read_csv(f'{dir_name}/{file}'))
    return df<jupyter_output><empty_output><jupyter_text>Load in datasets<jupyter_code>df_postcodes = pd.read_csv('data/2019_postcodes.csv', low_memory=False)
df_postcodes = df_postcodes[df_postcodes['rgn'] == 'E12000007']

LSOAS_all_of_london = list(df_postcodes['lsoa11'].unique())

df_income = pd.read_csv('data/2019_income.csv', low_memory=False)
df_income = df_income[df_income["LSOA Code (2011)"].isin(LSOAS_all_of_london)]

df_employment = pd.read_csv('data/2019_employment.csv', low_memory=False)
df_employment = df_employment[df_employment["LSOA Code (2011)"].isin(LSOAS_all_of_london)]

df_stop_and_search = pd.read_csv('data/2019_city_of_london/2019_city_of_london_stop_and_search.csv').append(pd.read_csv('data/2019_metropolitan/2019_metropolitan_stop_and_search.csv'))
df_arrests = df_stop_and_search[df_stop_and_search['Outcome linked to object of search'] == True]

df_searches = df_stop_and_search[df_stop_and_search['Latitude'].notnull()]
df_searches = df_searches[df_searches['Longitude'].notnull()]

df_arrests = df_arrests[df_arrests['Latitude'].notnull()]
df_arrests = df_arrests[df_arrests['Longitude'].notnull()]
df_metropolitan_street = combine_csvs_in_dir_to_df('multidata/metropolitan/street')
df_metropolitan_street = df_metropolitan_street[df_metropolitan_street["LSOA code"].isin(LSOAS_all_of_london)]

df_cityoflondon_street = combine_csvs_in_dir_to_df('multidata/city-of-london/street')
df_cityoflondon_street = df_cityoflondon_street[df_cityoflondon_street["LSOA code"].isin(LSOAS_all_of_london)]

df_alloflondon_street = df_metropolitan_street.append(df_cityoflondon_street)
df_alloflondon_street = df_alloflondon_street.dropna(axis=0, subset=['Crime type'])<jupyter_output><empty_output><jupyter_text>Group data<jupyter_code>df_alloflondon_street_counts = df_alloflondon_street.groupby('LSOA code').size().to_frame('size')
df_alloflondon_street_counts.reset_index(level=0, inplace=True)
df_alloflondon_street_counts['log(size)'] = df_alloflondon_street_counts['size'].apply(lambda x: math.log(x))
df_alloflondon_street_counts = df_alloflondon_street_counts.rename(columns={"size": "Crime rate", "log(size)": "Crime rate log"})
df_alloflondon_street_counts.head(5)<jupyter_output><empty_output><jupyter_text>Scrape boundary GeoJSON from findthatpostcode.uk<jupyter_code>if not load_from_file:
    GEOJSON = {}
    LSOA_length = len(LSOAS_all_of_london)
    for idx, LSOA in enumerate(LSOAS_all_of_london):
        res = httpx.get(f'https://findthatpostcode.uk/areas/{LSOA}.geojson')
        body = json.loads(res.content.decode())
        if "message" in body:
            if "boundary not found" in body["message"]:
                print('Boundary not found for LSOA: '+str(LSOA))
                continue
        else:
            print(f"Successfully scraped {LSOA} ({idx}/{LSOA_length})")
        GEOJSON[LSOA] = body
else:
    GEOJSON_FULL = json.load(open('LSOAS_all_of_london.geojson'))<jupyter_output><empty_output><jupyter_text>Merge the different GeoJSON boundaries that were scraped to one object<jupyter_code>if not load_from_file:
    GEOJSON_FULL = {
        "type": "FeatureCollection",
        "features": []
    }

    for GEO in GEOJSON:
        TEMP = GEOJSON[GEO]
        TEMP["features"][0]["id"] = GEO
        GEOJSON_FULL["features"].append(TEMP["features"][0])

    with open('LSOAS_all_of_london.geojson', 'w+') as outfile:
        json.dump(GEOJSON_FULL, outfile)<jupyter_output><empty_output><jupyter_text>Create the Mapbox choropleth map using Plotly<jupyter_code>fig = px.choropleth_mapbox(
    df_income,
    geojson = GEOJSON_FULL,
    locations = 'LSOA Code (2011)',
    color = 'Income Domain Rank (where 1 is most deprived)',
    color_continuous_scale = "Viridis",
    mapbox_style = "carto-positron",
    zoom = 8.6,
    center = {"lat": 51.49, "lon": -0.12},
    opacity = 0.9
)

fig.update_traces(marker_line_width=0)

fig.show()
fig = px.choropleth_mapbox(
    df_employment,
    geojson = GEOJSON_FULL,
    locations = 'LSOA Code (2011)',
    color = 'Employment Domain Rank (where 1 is most deprived)',
    color_continuous_scale = "Viridis",
    mapbox_style = "carto-positron",
    zoom = 8.6,
    center = {"lat": 51.49, "lon": -0.12},
    opacity = 0.9
)

fig.update_traces(marker_line_width=0)

fig.show()
fig = go.Figure(go.Scattermapbox(
        lat=list(df_searches['Latitude']),
        lon=list(df_searches['Longitude']),
        mode='markers',
        marker=go.scattermapbox.Marker(
            size=2,
            opacity=1,
            color="orange"
        ),
        text=list(df_searches['Type']),
        name="Searches"
    ))

fig.add_trace(
    go.Scattermapbox(
        lat=list(df_arrests['Latitude']),
        lon=list(df_arrests['Longitude']),
        mode='markers',
        marker=go.scattermapbox.Marker(
            size=5,
            opacity=1,
            color="blue"
        ),
        text=list(df_arrests['Type']),
        name="Arrests"
    )
)

fig.update_layout(
    hovermode='closest',
    mapbox=dict(
        accesstoken=mapbox_access_token,
        bearing=0,
        center=go.layout.mapbox.Center(
            lat=51.49,
            lon=-0.12
        ),
        pitch=0,
        zoom=8.6
    )
)

fig.show()
fig = px.choropleth_mapbox(
    df_alloflondon_street_counts,
    geojson = GEOJSON_FULL,
    locations = 'LSOA code',
    color = 'Crime rate',
    color_continuous_scale = "Viridis",
    mapbox_style = "carto-positron",
    zoom = 8.6,
    center = {"lat": 51.49, "lon": -0.12},
    opacity = 0.9
)

fig.update_traces(marker_line_width=0)

fig.show()
fig = px.choropleth_mapbox(
    df_alloflondon_street_counts,
    geojson = GEOJSON_FULL,
    locations = 'LSOA code',
    color = 'Crime rate log',
    color_continuous_scale = "Viridis",
    mapbox_style = "carto-positron",
    zoom = 8.6,
    center = {"lat": 51.49, "lon": -0.12},
    opacity = 0.9
)

fig.update_traces(marker_line_width=0)

fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12419239052404882, 'fraction_numerical': 0.01780330222541278, 'mean_word_length': 4.515439429928741, 'pattern_counts': {'":': 12, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '43839', 'license': 'MIT', 'n_tokens_mistral': 2722, 'n_tokens_neox': 2543, 'n_words': 390, 'path': 'Support notebooks/2019_choropleth_income_employment_searches.ipynb', 'repo_name': 'MoshaLangerak/Data_Challenge_2_Group_18', 'size': '7049'}