<jupyter_start><jupyter_text>Making sense of humongous location datasets  InstallationsThe geospatial libraries are not pre installed in google colab as standard python library, therefore we need to install some libraries to use. Luckily this is an easy process. You can use either apt install or pip install. You can also create anaconda environment, but that is a bit complex so pip and apt are enough in our case to get the libraries we need. These are the libraries we need to install in this tutorial:Gdal Geopandas FoliumThe installation might take 1 minute.<jupyter_code>%%time 
!apt update --quiet
!apt upgrade --quiet
# GDAL Important library for many geopython libraries
!apt install gdal-bin python-gdal python3-gdal --quiet
# Install rtree - Geopandas requirment
!apt install python3-rtree --quiet
# Install Geopandas
!pip install git+git://github.com/geopandas/geopandas.git --quiet
# Install descartes - Geopandas requirment
!pip install descartes --quiet
# Install Folium for Geographic data visualization
!pip install folium --quiet
# Install Pysal
!pip install pysal --quiet
# Install splot --> pysal
!pip install splot --quiet
# Install mapclassify
!pip install mapclassify --quiet
import pandas as pd
import numpy as np
import geopandas as gpd
from shapely.geometry import Point
from pysal.explore import esda
from pysal.lib import weights
#import libysal as lps
from pysal.viz.splot.esda import plot_moran, plot_local_autocorrelation, lisa_cluster
import matplotlib
import matplotlib.pyplot as plt 
import folium
import os
import seaborn as sns
from sklearn.cluster import KMeans
from sklearn.cluster import dbscan

from libpysal.weights.contiguity import Queen
from esda.moran import Moran
from splot.esda import moran_scatterplot
from esda.moran import Moran_Local
from splot.esda import lisa_cluster

import pysal as ps
ps.__version__<jupyter_output><empty_output><jupyter_text>Data The dataset for this chapter is stored in the dropbox link. It is a valuable skill to learn how to access data on the web, so we will use WGET. WGET is great utility in accessing files from the web and supports different protocols.<jupyter_code># Get the data from dropbox link
!wget https://www.dropbox.com/s/xvs0ybc402mkrn8/2019-02-avon-and-somerset-street.zip --quiet
# see the folders available
import os
os.listdir(os.getcwd())
# We have zipped data so let us unzip it
!unzip 2019-02-avon-and-somerset-street.zip 
crime_somerset = pd.read_csv("2019-02-avon-and-somerset-street.csv")
crime_somerset.head()
crime_somerset.shape
crime_somerset.isnull().sum()
# Drop columns with high missing values
crime_somerset.drop(['Last outcome category','Context', 'Crime ID' ], axis=1, inplace=True)
crime_somerset.head()
crime_somerset.isnull().sum()
# Drop rows with missin values
crime_somerset.dropna(axis=0,inplace=True)
crime_somerset.isnull().sum()
crime_somerset.shape

crime_somerset.head()<jupyter_output><empty_output><jupyter_text>Convert to GeoDataFrame<jupyter_code># Function to create a Geodataframe
def create_gdf(df, lat, lon):
    """ Convert pandas dataframe into a Geopandas GeoDataFrame"""
    crs = {'init': 'epsg:4326'}
    geometry = [Point(xy) for xy in zip(df[lon], df[lat])]
    gdf = gpd.GeoDataFrame(df, crs=crs, geometry=geometry)
    return gdf
crime_somerset_gdf = create_gdf(crime_somerset, 'Latitude', 'Longitude')
crime_somerset_gdf.head()
fig, ax = plt.subplots(figsize=(12,10))
crime_somerset_gdf.plot(markersize=20, ax=ax);
plt.savefig('crime_somerset_map.png', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>KMeans Clustering Location Data<jupyter_code>crime_somerset_gdf.head()<jupyter_output><empty_output><jupyter_text>* Split training and test dataset<jupyter_code>train = crime_somerset_gdf.sample(frac=0.7, random_state=14)
test = crime_somerset_gdf.drop(train.index)
train.shape, test.shape
# Get coordinates for the train and test dataset
train_coords = train[['Latitude', 'Longitude']].values
test_coords = test[['Latitude', 'Longitude']].values
# Fit Kmeans clustering on training dataset
kmeans = KMeans(n_clusters=5)
kmeans.fit(train_coords)
# Predict on the test dataset by clustering 
preds = kmeans.predict(test_coords)

# Get centers of the clusters
centers = kmeans.cluster_centers_
fig, ax = plt.subplots(figsize=(12,10))
plt.scatter(test_coords[:, 0], test_coords[:, 1], c=preds, s=30, cmap='viridis')
plt.scatter(centers[:,0], centers[:,1], c='Red', marker="s", s=50);<jupyter_output><empty_output><jupyter_text>DBSCAN   Detecting Outliers/Noise<jupyter_code>coords = crime_somerset_gdf[['Latitude', 'Longitude']]
coords[:5]
# Get labels of each cluster 
_, labels  = dbscan(crime_somerset_gdf[['Latitude', 'Longitude']], eps=0.1, min_samples=10)

# Create a labels dataframe with the index of the dataset
labels_df = pd.DataFrame(labels, index=crime_somerset_gdf.index, columns=['cluster'])
labels_df.head()
# Groupby Labels
labels_df.groupby('cluster').size()
# Plot the groupedby labels
sns.countplot(labels_df.cluster);
plt.show()
# Get Noise (Outliers) with label -1 
noise = crime_somerset_gdf.loc[labels_df['cluster']==-1, ['Latitude', 'Longitude']]

# Get core with labels 0
core = crime_somerset_gdf.loc[labels_df['cluster']== 0, ['Latitude', 'Longitude']]
# Display scatter plot with noises as stars and core as circle points

fig, ax = plt.subplots(figsize=(12,10))
ax.scatter(noise['Latitude'], noise['Longitude'],marker= '*', s=40, c='blue' )
ax.scatter(core['Latitude'], core['Longitude'], marker= 'o', s=20, c='red')
plt.savefig('outliers.png');
plt.show();

noise<jupyter_output><empty_output><jupyter_text>Detecting Clusters<jupyter_code>_, labels  = dbscan(crime_somerset_gdf[['Latitude', 'Longitude']], eps=0.01, min_samples=300)
labels_df = pd.DataFrame(labels, index=crime_somerset_gdf.index, columns=['cluster'])
labels_df.groupby('cluster').size()
noise = crime_somerset_gdf.loc[labels_df['cluster']==-1, ['Latitude', 'Longitude']]
core = crime_somerset_gdf.loc[labels_df['cluster']== 0, ['Latitude', 'Longitude']]
bp1 = crime_somerset_gdf.loc[labels_df['cluster']== 1, ['Latitude', 'Longitude']]
bp2 = crime_somerset_gdf.loc[labels_df['cluster']== 2, ['Latitude', 'Longitude']]
bp3 = crime_somerset_gdf.loc[labels_df['cluster']== 3, ['Latitude', 'Longitude']]

fig, ax = plt.subplots(figsize=(12,10))
ax.scatter(noise['Latitude'], noise['Longitude'], markers=10, c='gray' )
ax.scatter(core['Latitude'], core['Longitude'], s=100, c='red')
ax.scatter(bp1['Latitude'], bp1['Longitude'], s=50, c='yellow')
ax.scatter(bp2['Latitude'], bp2['Longitude'], s=50, c='green')
ax.scatter(bp3['Latitude'], bp3['Longitude'], s=50, c='blue')
plt.savefig('cluster_ex1.png');
plt.show()

fig, ax = plt.subplots(figsize=(15,12))
ax.scatter(noise['Latitude'], noise['Longitude'],s=1, c='gray' )
ax.scatter(core['Latitude'], core['Longitude'],marker= "*", s=10, c='red')
ax.scatter(bp1['Latitude'], bp1['Longitude'], marker = "v", s=10, c='yellow')
ax.scatter(bp2['Latitude'], bp2['Longitude'], marker= "P", s=10, c='green')
ax.scatter(bp3['Latitude'], bp3['Longitude'], marker= "d", s=10, c='blue')
ax.set_xlim(left=50.8, right=51.7)
ax.set_ylim(bottom=-3.5, top=-2.0)
plt.savefig('cluster_zoomed.png');
plt.show()

#Creates four polar axes, and accesses them through the returned array
fig, axes = plt.subplots(2, 2, figsize=(12,10))
axes[0, 0].scatter(noise['Latitude'], noise['Longitude'],s=0.01, c='gray' )
axes[0, 0].title.set_text('Noise')
axes[0, 1].scatter(core['Latitude'], core['Longitude'],marker= "*", s=10, c='red')
axes[0, 1].title.set_text('Core')
axes[1, 0].scatter(bp1['Latitude'], bp1['Longitude'], marker = "v", s=50, c='yellow')
axes[1, 0].title.set_text('Border Points 1')
axes[1,1].scatter(bp2['Latitude'], bp2['Longitude'],  marker= "P", s=50, c='green')
axes[1, 1].title.set_text('Border Points 2')
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Spatial AutocorellationWe will Polygon data for this section. Let us first get the data from the dropbox URL<jupyter_code>!wget https://www.dropbox.com/s/k2ynddy79k2r46i/ASC_Beats_2016.zip 
!unzip ASC_Beats_2016.zip
boundaries = gpd.read_file('ASC_Beats_2016.shp')
boundaries.head()
boundaries.crs, crime_somerset_gdf.crs
boundaries_4326 = boundaries.to_crs({'init': 'epsg:4326'}) 
fig, ax = plt.subplots(figsize=(12,10))

boundaries_4326.plot(ax=ax)
crime_somerset_gdf.plot(ax=ax, markersize=10, color='red')
plt.savefig('overlayed_map.png')
# Points in Polygon
crimes_with_boundaries = gpd.sjoin(boundaries_4326,crime_somerset_gdf, op='contains' )
crimes_with_boundaries.head()
grouped_crimes = crimes_with_boundaries.groupby('BEAT_CODE').size()
grouped_crimes.head()
df = grouped_crimes.to_frame().reset_index()
df.columns = ['BEAT_CODE', 'CrimeCount']
df.head()
final_result = boundaries.merge(df, on='BEAT_CODE')
final_result.head()<jupyter_output><empty_output><jupyter_text>* Choropleth Map of the Crime Count<jupyter_code>fig, ax = plt.subplots(figsize=(12,10))
final_result.plot(column='CrimeCount', scheme='Quantiles', k=5, cmap='YlGnBu', legend=True, ax=ax);
plt.tight_layout()
ax.set_axis_off()
plt.savefig('choroplethmap.png')
plt.title('Crimes Choropleth Map ')
plt.show()<jupyter_output><empty_output><jupyter_text>GLobal Spatial Autocorrelation<jupyter_code># Create y variable values
y = final_result['CrimeCount'].values

# Sptial lag
ylag = weights.lag_spatial(wq, y)
final_result['ylag'] = ylag

# Get Weights (Queen)
wq = Queen.from_dataframe(final_result)
wq.transform = 'r'

moran = Moran(y, wq)
moran.I
from splot.esda import plot_moran

plot_moran(moran, zstandard=True, figsize=(10,4))
plt.tight_layout()
plt.savefig('moronPlot.png')
plt.show()

moran.p_sim<jupyter_output><empty_output><jupyter_text>Visualizing Local Autocorrelation with splot - Hot Spots, Cold Spots and Spatial Outliers<jupyter_code># calculate Moran_Local and plot
moran_loc = Moran_Local(y, w)
fig, ax = moran_scatterplot(moran_loc)
plt.savefig('moron_local.png')
plt.show()
fig, ax = moran_scatterplot(moran_loc, p=0.05)
plt.show()
lisa_cluster(moran_loc, final_result, p=0.05, figsize = (10,8))
plt.tight_layout()
plt.savefig('lisa_clusters.png')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14382685686178062, 'fraction_numerical': 0.022725036891293654, 'mean_word_length': 7.211631663974152, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 46, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '6813', 'license': 'MIT', 'n_tokens_mistral': 3939, 'n_tokens_neox': 3731, 'n_words': 837, 'path': 'Chapter04/Chapter4.ipynb', 'repo_name': 'AdityaAnuragMishra/Geospatial-Data-Science-Quick-Start-Guide', 'size': '10165'}