<jupyter_start><jupyter_text>Clustering and Regions The previous notebook provided several illustrations of the power ofvisualization in the analysis of spatial data. This power stems fromvisualizations ability to tap into our human pattern recognition machinery.In this notebook we introduce methods for regionalization and clustering. Imports<jupyter_code>import pandas as pd
import geopandas as gpd
import libpysal as lp
import matplotlib.pyplot as plt
import rasterio as rio
import numpy as np
import contextily as ctx
import shapely.geometry as geom
%matplotlib inline
neighborhoods = gpd.read_file('../data/neighborhoods.gpkg')
# was created in previous notebook with neighborhoods.to_file('data/neighborhoods.gpkg')
listings = gpd.read_file('../data/listings.gpkg')
# was created in previous notebook with listings.to_file('data/neighborhoods.gpkg')
listings['price'] = listings.price.str.replace('$','').str.replace(',','_').astype(float)
neighborhoods.head()
summaries = gpd.sjoin(listings[['price', 'accommodates', 'geometry']], neighborhoods, op='within')\
                .eval('price_per_head = price / accommodates')\
                .groupby('index_right')\
                .agg(dict(price_per_head='median',
                          price='median'))
neighborhoods['median_pph'] = summaries.price_per_head
neighborhoods['median_pri'] = summaries.price
f,ax = plt.subplots(1,2,figsize=(8,3))
neighborhoods.plot(column='median_pri', ax=ax[0])
neighborhoods.plot(column='median_pph', ax=ax[1])
ax[0].set_title('Median Price')
ax[1].set_title('Median Price per Head')
import libpysal
wq = libpysal.weights.Queen.from_dataframe(neighborhoods)
from sklearn.cluster import KMeans, AgglomerativeClustering
kmeans = KMeans(n_clusters=5)
import numpy
numpy.random.seed(0)
cluster_variables = ['median_pri']
k5cls = kmeans.fit(neighborhoods[cluster_variables])
k5cls.labels_
# Assign labels into a column
neighborhoods['k5cls'] = k5cls.labels_
# Setup figure and ax
f, ax = plt.subplots(1, figsize=(9, 9))
# Plot unique values choropleth including a legend and with no boundary lines
neighborhoods.plot(column='k5cls', categorical=True, legend=True, linewidth=0, ax=ax)
# Remove axis
ax.set_axis_off()
# Keep axes proportionate
plt.axis('equal')
# Add title
plt.title(r'Price Clusters (k-means, $k=5$)')
# Display the map
plt.show()
k5sizes = neighborhoods.groupby('k5cls').size()
k5sizes
k5means = neighborhoods.groupby('k5cls')[cluster_variables].mean()
k5means.T
import numpy
numpy.random.seed(0)
ward5 = AgglomerativeClustering(linkage='ward', n_clusters=5)
ward5.fit(neighborhoods[cluster_variables])
neighborhoods['ward5'] = ward5.labels_
ward5sizes = neighborhoods.groupby('ward5').size()
ward5sizes
ward5means = neighborhoods.groupby('ward5')[cluster_variables].mean()
ward5means.T
# Setup figure and ax
f, ax = plt.subplots(1, figsize=(9, 9))
# Plot unique values choropleth including a legend and with no boundary lines
neighborhoods.plot(column='ward5', categorical=True, legend=True, linewidth=0, ax=ax)
# Remove axis
ax.set_axis_off()
# Keep axes proportionate
plt.axis('equal')
# Add title
plt.title('Price Clusters (AHC, $k=5$)')
# Display the map
plt.show()
# Setup figure and ax
f, axs = plt.subplots(1, 2, figsize=(12, 6))

ax = axs[0]
# Plot unique values choropleth including a legend and with no boundary lines
neighborhoods.plot(column='ward5', categorical=True, cmap='Set2', 
                   legend=True, linewidth=0, ax=ax)
# Remove axis
ax.set_axis_off()
# Keep axes proportionate
ax.axis('equal')
# Add title
ax.set_title('K-Means solution ($k=5$)')

ax = axs[1]
# Plot unique values choropleth including a legend and with no boundary lines
neighborhoods.plot(column='k5cls', categorical=True, cmap='Set3',
                   legend=True, linewidth=0, ax=ax)
# Remove axis
ax.set_axis_off()
# Keep axes proportionate
ax.axis('equal')
# Add title
ax.set_title('AHC solution ($k=5$)')

# Display the map
plt.show()


numpy.random.seed(123456)
model = AgglomerativeClustering(linkage='ward',
                                connectivity=wq.sparse,
                                n_clusters=5)
model.fit(neighborhoods[cluster_variables])
neighborhoods['ward5wq'] = model.labels_
# Setup figure and ax
f, ax = plt.subplots(1, figsize=(9, 9))
# Plot unique values choropleth including a legend and with no boundary lines
neighborhoods.plot(column='ward5wq', categorical=True, legend=True, linewidth=0, ax=ax)
# Remove axis
ax.set_axis_off()
# Keep axes proportionate
plt.axis('equal')
# Add title
plt.title(r'Price Regions (Ward, $k=5$, Queen Contiguity)')
# Display the map
plt.show()<jupyter_output><empty_output><jupyter_text>Spatial Clustering Based on Points<jupyter_code>listings.shape
clipped_thiessens = gpd.read_file('../data/thiessens.gpkg')
clipped_thiessens.shape
wtq = libpysal.weights.Queen.from_dataframe(clipped_thiessens)
wtq.n
numpy.random.seed(123456)
logprice = numpy.log(listings[['price']]+1)
model = AgglomerativeClustering(linkage='ward',
                                            connectivity=wtq.sparse,
                                            n_clusters=5)
model.fit(logprice)
listings['region'] = model.labels_
numpy.unique(model.labels_, return_counts=True)
listings.sort_values('region').plot(column='region', marker='.')
region_sizes = listings.groupby('region').size()
region_sizes
model = AgglomerativeClustering(linkage='ward',n_clusters=5)
model.fit(logprice)
listings['region5'] = model.labels_
listings.groupby('region5').size()
region1 = listings[listings.region5==1]
region1.plot()
for region in range(5):
    region_ = listings[listings.region5==region]
    region_.plot()
    print(region_.shape)
region_.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12631210636808957, 'fraction_numerical': 0.015745276417074877, 'mean_word_length': 5.200650759219089, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '605494', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1990, 'n_tokens_neox': 1867, 'n_words': 433, 'path': 'notebooks/gds6-spatial_clusters.ipynb', 'repo_name': 'sjsrey/scipy2019-intermediate-gds-1', 'size': '5716'}