<jupyter_start><jupyter_text>Part 2: Clustering (DSFS Chapter 19)<jupyter_code>import csv # get the csv reader
import matplotlib.pyplot as plt
import pandas as pd
%matplotlib inline
from IPython.display import Image
from sklearn.cluster import KMeans
import numpy as np
import json
data=pd.read_csv("../week4/SFPD_Incidents_-_from_1_January_2003.csv")
#Create a dataframe with the required info of coordinates of prostitution.
DataProstitution=data[data["Category"]=="PROSTITUTION"]
DataProstitution=DataProstitution.filter(items=["X","Y"])
DataProstitution = DataProstitution[DataProstitution["Y"]!=90]
DataProstitutionKmeans=DataProstitution.filter(items=['X', 'Y'])
DataProstitutionKmeans.head()
# Converting to numpy array.
DataProstitutionKmeansArray=np.array(DataProstitutionKmeans)
# Concat all the dataframes together to create the csv file.
DataProstitutionKmeans=DataProstitution.filter(items=['X', 'Y'])

kmeans2 = KMeans(n_clusters=2, random_state=0).fit_predict(DataProstitutionKmeansArray)
df2=pd.DataFrame(kmeans2)
df2=df2.rename(columns = {0:'k2'})

kmeans3 = KMeans(n_clusters=3, random_state=0).fit_predict(DataProstitutionKmeansArray)
df3=pd.DataFrame(kmeans3)
df3=df3.rename(columns = {0:'k3'})

kmeans4 = KMeans(n_clusters=4, random_state=0).fit_predict(DataProstitutionKmeansArray)
df4=pd.DataFrame(kmeans4)
df4=df4.rename(columns = {0:'k4'})

kmeans5 = KMeans(n_clusters=5, random_state=0).fit_predict(DataProstitutionKmeansArray)
df5=pd.DataFrame(kmeans5)
df5=df5.rename(columns = {0:'k5'})

kmeans6 = KMeans(n_clusters=6, random_state=0).fit_predict(DataProstitutionKmeansArray)
df6=pd.DataFrame(kmeans6)
df6=df6.rename(columns = {0:'k6'})

result = pd.concat([DataProstitution.reset_index(drop=True),df2, df3,df4,df5,df6], axis=1)
result.reset_index(drop=True)
# write the results to a scv file
result.to_csv(path_or_buf="./PointsAndClusters.csv",index=False)
for i in range(2,10):
    kmeans = KMeans(n_clusters=i, random_state=0).fit_predict(DataProstitutionKmeansArray)
    plt.scatter(DataProstitutionKmeansArray[:, 0], DataProstitutionKmeansArray[:, 1], c=kmeans)
    plt.show()
# Writing the coordinates of centroids to a json file.
centroids=[]
for i in range(2,7):
    kmeans = KMeans(n_clusters=i, random_state=0).fit(DataProstitutionKmeansArray)
    centroids.append(kmeans.cluster_centers_)
    
centroids = [[list(el) for el in list(i)] for i in centroids]
centroids_dict = {"k%d" %(i+2): el for i,el in enumerate(centroids)}
with open('jsondataCentroids.json','w') as f:
    json.dump(centroids_dict,f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13029949436017113, 'fraction_numerical': 0.028004667444574097, 'mean_word_length': 8.421245421245422, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '197844', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1015, 'n_tokens_neox': 952, 'n_words': 168, 'path': 'Kmeans_data  (from week5).ipynb', 'repo_name': 'nickzafi/socialdata', 'size': '2571'}