<jupyter_start><jupyter_text>Week-3 IBM DS Professional Assignment-Part3 Assignment Details & Submission Guidelines Explore and cluster the neighborhoods in Toronto. You can decide to work with only boroughs that contain the word Toronto and then replicate the same analysis we did to the New York City data. It is up to you.Just make sure:to add enough Markdown cells to explain what you decided to do and to report any observations you make.to generate maps to visualize your neighborhoods and how they cluster together.Once you are happy with your analysis, submit a link to the new Notebook on your Github repository. (3 marks)<jupyter_code>#importing dependencies
import pandas as pd
import numpy as np
from geopy.geocoders import Nominatim
import folium
import matplotlib.cm as cm
import matplotlib.colors as colors
from sklearn.cluster import KMeans
print("************Dependencies Imported***************")
from IPython.core.display import HTML
import folium
import datetime
from datetime import datetime
import requests
from bs4 import BeautifulSoup
import lxml.html as lh
import pandas as pd
import re
import time
import psutil
import json

import numpy as np
from PIL import Image
import os
from os import path
import matplotlib.pyplot as plt

import plotly.graph_objects as go
from pandas.plotting import register_matplotlib_converters
import plotly.express as px
from plotly.subplots import make_subplots
from IPython.display import display, Markdown, Latex
import matplotlib as plot
from matplotlib.pyplot import figure
import seaborn as sns

register_matplotlib_converters()
from IPython.display import Markdown

import plotly.offline as py
import plotly.express as px

dataset = pd.DataFrame()
mumbai_pincodes=pd.read_csv('./Mumbai_Data/Mumbai_Pincodes_latitude_longitude.csv')
mumbai_pincodes.head()
mumbai_pincodes.shape
#List of covid zones in mumbai
covid_zones_mumbai=pd.read_csv('./Mumbai_Data/Covid_Zones_Mumbai.csv')
covid_zones_mumbai.head()
#Merging Covid zones & mumbai pincode data to get covid zones latitude & Longitude
covid19_mumbai=covid_zones_mumbai.merge(mumbai_pincodes,how='left',on='Pincode',validate='m:1')
#Comprehensive dataset of covid-19 cases in mumbai uptill 18th April
#Source: http://stopcoronavirus.mcgm.gov.in/
covid19_mumbai.head()
#Looking at Testing labs from india's list & extract mumbai's data
testing_labs_india=pd.read_csv('./covid19/Covid_data_srk/ICMRTestingLabs.csv')
testing_labs_india.rename(columns={"pincode": "Pincode"},inplace=True)
mumbai_testing_labs=testing_labs_india.loc[testing_labs_india['city']== 'Mumbai']
mumbai_testing_labs.head()
#Map Mumbai's Testing labs with latitude & Longitude:Key{PinCode}
mumbai_labs_ll=mumbai_testing_labs.merge(mumbai_pincodes,how='left',on='Pincode',validate='m:1')
mumbai_labs_ll.head()
lat=[]
long=[]
pincd=[]

#address = 'Mumbai, india'
for pincode in mumbai_labs_ll['Pincode']:
    add= str(pincode) + ',india'
    address = add
    geolocator = Nominatim(user_agent="can_explorer")
    location = geolocator.geocode(address,timeout=15)
    latitude = lat.append(location.latitude)
    longitude =long.append(location.longitude)
    pincd.append(pincode)
    print('The geograpical coordinate found for {}.'.format(add))
Mumbai_covid_labs = pd.DataFrame({"Pincode": pincd, 
                           "Latitude": lat,
                           "Longitude": long})

Mumbai_covid_labs.head()
Mumbai_covid_labs.drop_duplicates('Pincode',inplace=True)
mumbai_labs_final=mumbai_testing_labs.merge(Mumbai_covid_labs,how='left',on='Pincode',validate='m:1')
mumbai_labs_final.dropna(inplace=True)
mumbai_labs_final
address = ' Mumbai,India'
geolocator = Nominatim(user_agent="can_explorer")
location = geolocator.geocode(address,timeout=15)
latitude = location.latitude
longitude =location.longitude
print('The geograpical coordinate found for {} Lat: {} Long:{}'.format(address,latitude,longitude))<jupyter_output>The geograpical coordinate found for  Mumbai,India Lat: 18.9387711 Long:72.8353355<jupyter_text>Find Data for Hospitals using FourSquare Location APICategory ID is used to find a particular type of veune in neighbourhoodRefer https://developer.foursquare.com/docs/build-with-foursquare/categories/ for more details on venue category IdsCategory Id for Hospitals = 4bf58dd8d48988d196941735<jupyter_code>LIMIT = 100

CLIENT_ID = 'ZVLUP1PU41P2XVZN1R15EWMRBMOUU5MA3HTHOYKFKZTHWMYJ' # your Foursquare ID
CLIENT_SECRET = '<KEY>' # your Foursquare Secret
VERSION = '20180605' # Foursquare API version
CATEGORYID='4bf58dd8d48988d196941735'

print('Your credentails:')
print('CLIENT_ID: ' + CLIENT_ID)
print('CLIENT_SECRET:' + CLIENT_SECRET)
lat='19.25023195'
lng='73.16017493'
radius=500
 # create the API request URL
radius=500
latitude=19.0025 #Mumbai
longitude=72.8421 #Mumbai
#for latitude,longitude in zip(mumbai_covid_clusters['Latitude'],mumbai_covid_clusters['Longitude']):
url = 'https://api.foursquare.com/v2/venues/search?categoryId={}&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(CATEGORYID,CLIENT_ID,  CLIENT_SECRET, VERSION, latitude, longitude, radius, LIMIT)
results = requests.get(url).json()["response"]#['groups'][0]['items']
    #results = requests.get(url).json()
results
#Plot Toronto city on Map using folium
mumbai_labs= folium.Map(location=[latitude, longitude], zoom_start=9)
for lat, lng, Pincode, lab,typeh in zip(mumbai_labs_final['Latitude'], mumbai_labs_final['Longitude'], mumbai_labs_final['Pincode'], mumbai_labs_final['lab'],mumbai_labs_final['type']):
    label = '{}, {},{}'.format(Pincode,lab,typeh)
    label = folium.Popup(label, parse_html=True)
    if typeh =='Private Laboratory':
        val='blue'
    else:
        val='green'
    

    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color=val,
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.7,
        parse_html=False).add_to(mumbai_labs)  

mumbai_labs
covid19_zones=covid19_mumbai
covid19_zones.drop_duplicates('Pincode',inplace=True)
covid19_zones.shape
covid19_zones.head()
zn=covid19_zones['Ward'].unique()
zn
covid19_zones
covid19_zones.dropna(subset=['Latitude', 'Longitude'],inplace=True)
covid19_zones.shape
#Plot Mumbai City Covid-19 Zones on Map using folium
mumbai_19= folium.Map(location=[latitude, longitude], zoom_start=10)

x = np.arange(21)
ys = [i + x + (i*x)**2 for i in range(21)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]
colors_fl = [
    'red',
    'blue',
    'gray',
    'darkred',
    'lightred',
    'orange',
    'beige',
    'green',
    'darkgreen',
    'lightgreen',
    'darkblue',
    'lightblue',
    'purple',
    'darkpurple',
    'pink',
    'cadetblue',
    'lightgray',
    'black',
    'darkyellow',
    'lightyello',
    'yellow'
]
for lat, lng, Pincode,ward,address in zip(covid19_zones['Latitude'], covid19_zones['Longitude'], covid19_zones['Pincode'], covid19_zones['Ward'],covid19_zones['Address']):
    label = '{}, {},{}'.format(Pincode,ward,address)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='red',
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.7,
        parse_html=False).add_to(mumbai_19)  

mumbai_19
covid_zones_mumbai.shape
covid_mumbai_wards=covid_zones_mumbai
covid_mumbai_wards.shape
ward_group=covid_mumbai_wards.groupby('Ward')['Address'].count().reset_index()
ward_group.rename(columns={"Address": "Cases_Count"},inplace=True)
ward_group.columns
f, ax = plt.subplots(figsize=(15, 8))
data = ward_group[['Ward','Cases_Count']]
data.sort_values('Ward',ascending=False,inplace=True)
sns.set_color_codes("pastel")
sns.barplot(x="Ward", y="Cases_Count", data=data,
            label="Total", color="r")

            

# Add a legend and informative axis label
ax.legend(ncol=2, loc="lower right", frameon=True)
ax.set(xlim=(0, 10), ylabel="",
       xlabel="Cases")
sns.despine(left=True, bottom=True)
plt.show()
postal_cd=mumbai_pincodes
#postal_cd.shape
mum_postal_cd=postal_cd.loc[postal_cd['Region']== 'Mumbai']
mum_postal_cd.columns
mum_final=mum_postal_cd[['Pincode',' Area']]
mum_final.shape
zones=covid_zones_mumbai.groupby('Pincode')['Address'].count().reset_index()
zones.head()
zones.rename(columns={"Address": "Cases_Count"},inplace=True)
zones.columns
model_mum=mum_final.merge(zones,how='left',on='Pincode')
#mumbai_labs_final=mumbai_testing_labs.merge(Mumbai_covid_labs,how='left',on='Pincode',validate='m:1')
model_mum.head()
model_mum["Cases_Count"].fillna(0.0, inplace = True) 
model_mum1=model_mum[['Pincode','Cases_Count']]
kclusters = 3

clustering_covid_mumbai = model_mum1

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(clustering_covid_mumbai)

# check cluster labels generated for each row in the dataframe
kmeans.labels_[0:80]
pincd_lst=mum_postal_cd['Pincode'].unique().tolist()
covid_lat_lst=mum_postal_cd['Latitude'].tolist()
covid_long_lst=mum_postal_cd['Longitude'].tolist()
cluster_labels=kmeans.labels_[0:]
type(cluster_labels)
cluster_lst=cluster_labels.tolist()
#=cluster_labels
len(covid_lat_lst)
mumbai_clust=mumbai_covid_clusters.merge(covid_zones_mumbai,how='left',on='Pincode')
#Finally Plotting Clusters for Mumbai Covid-19 Zone
map_clusters = folium.Map(location=[latitude, longitude],tiles="Stamen Toner", zoom_start=10)

# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i + x + (i*x)**2 for i in range(kclusters)]
#colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
#rainbow = [colors.rgb2hex(i) for i in colors_array]
color_code = ['green','blue','red']

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(mumbai_covid_clusters['Latitude'], mumbai_covid_clusters['Longitude'], mumbai_covid_clusters['Pincode'], mumbai_covid_clusters['Cluster_label']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    if cluster != 'Nan':
        val=int(cluster)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=color_code[val],
        fill=True,
        fill_color=rainbow[val-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12634330525704326, 'fraction_numerical': 0.021880143285894084, 'mean_word_length': 5.918955123911587, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '272220', 'license': 'CC-BY-3.0', 'n_tokens_mistral': 3944, 'n_tokens_neox': 3727, 'n_words': 738, 'path': '.ipynb_checkpoints/Week5 IBM DS Coursera Capstone_Mumbai_Final-checkpoint.ipynb', 'repo_name': 'shasaankdave/Coursera_Capstone', 'size': '10372'}