<jupyter_start><jupyter_text>Importing another requiered libraries<jupyter_code>import json # library to handle JSON files

import geopy # install it in Anaconda Prompt
from geopy.geocoders import Nominatim # convert an address into latitude and longitude values

import requests # library to handle requests
from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe

# Matplotlib and associated plotting modules
import matplotlib.cm as cm
import matplotlib.colors as colors

# import k-means from clustering stage
from sklearn.cluster import KMeans

import folium # map rendering library

from IPython.display import Image<jupyter_output><empty_output><jupyter_text>Get toronto neighbourhoods data<jupyter_code>neighborhoods=pd.read_csv('TRT_geo.csv')

# Restrict to boroughs that contain the word Toronto 
toronto_data=neighborhoods[neighborhoods.Borough.str.contains('Toronto')]

# The first five rows of the data frame
toronto_data.head()
toronto_data.shape<jupyter_output><empty_output><jupyter_text>Step 6: Use geopy library to get the latitude and longitude values of Toronto<jupyter_code>address = 'Toronto'

geolocator = Nominatim(user_agent="trt_explorer")
location = geolocator.geocode(address)
latitude = location.latitude
longitude = location.longitude
print('The geograpical coordinate of Toronto are {}, {}.'.format(latitude, longitude))<jupyter_output>The geograpical coordinate of Toronto are 43.6534817, -79.3839347.<jupyter_text>create map of toronto<jupyter_code># create map of Manhattan using latitude and longitude values
map_toronto = folium.Map(location=[latitude, longitude], zoom_start=11)

# add markers to map
for lat, lng, label in zip(toronto_data['Latitude'], toronto_data['Longitude'], toronto_data['Neighborhood']):
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='blue',
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.7,
        parse_html=False).add_to(map_toronto)  
    
map_toronto
CLIENT_ID = 'YANZIUUV1UJLFUJRQXD1RP0Z2GFAOLQWAE14R50KKC5WWMVC' 
CLIENT_SECRET = '<KEY>' 
VERSION = '20180605' # Foursquare API version
LIMIT=100 # limit of number of venues returned by Foursquare API
radius=500 # define radius
def getNearbyVenues(names, latitudes, longitudes, radius=500):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
        print(name)
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Neighborhood', 
                  'Neighborhood Latitude', 
                  'Neighborhood Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)<jupyter_output><empty_output><jupyter_text>Apply generated function above<jupyter_code># This will generate venues for each neighborhood 
toronto_venues = getNearbyVenues(names=toronto_data['Neighborhood'],
                                   latitudes=toronto_data['Latitude'],
                                   longitudes=toronto_data['Longitude'])<jupyter_output>The Beaches
The Danforth West / Riverdale
India Bazaar / The Beaches West
Studio District
Lawrence Park
Davisville North
North Toronto West
Davisville
Moore Park / Summerhill East
Summerhill West / Rathnelly / South Hill / Forest Hill SE / Deer Park
Rosedale
St. James Town / Cabbagetown
Church and Wellesley
Regent Park / Harbourfront
Garden District, Ryerson
St. James Town
Berczy Park
Central Bay Street
Richmond / Adelaide / King
Harbourfront East / Union Station / Toronto Islands
Toronto Dominion Centre / Design Exchange
Commerce Court / Victoria Hotel
Roselawn
Forest Hill North & West
The Annex / North Midtown / Yorkville
University of Toronto / Harbord
Kensington Market / Chinatown / Grange Park
CN Tower / King and Spadina / Railway Lands / Harbourfront West / Bathurst Quay / South Niagara / Island airport
Stn A PO Boxes
First Canadian Place / Underground city
Christie
Dufferin / Dovercourt Village
Little Portugal / Trinity
Brockton / Parkdale Village / Exhibition Place
High Park / [...]<jupyter_text>Step 6: Check toronto_venues dataframe<jupyter_code>print("The shape of the dataframe is {}. The dataset has {} rows.".format
      (toronto_venues.shape,toronto_venues.shape[0]))
toronto_venues.head()<jupyter_output>The shape of the dataframe is (1620, 7). The dataset has 1620 rows.<jupyter_text>check how many venues returned for each neighbourhood<jupyter_code>toronto_venues.groupby('Neighborhood').count()<jupyter_output><empty_output><jupyter_text>List all potential values ofr each neighboirhood<jupyter_code>toronto_onehot = pd.get_dummies(toronto_venues[['Venue Category']], prefix="", prefix_sep="")

# add neighborhood column back to dataframe
toronto_onehot['Neighbourhood']=toronto_venues['Neighborhood']
# Note: In the lab, the column name in the dummy variale df is the same as the one in no-dv df. Here it won't work.
#       Column names must be different.

# move neighborhood column to the first column
fixed_columns = [toronto_onehot.columns[-1]] + list(toronto_onehot.columns[:-1])
toronto_onehot = toronto_onehot[fixed_columns]

# check the number of rows. It must be equal to that of toronto_venues
print("The shape of the dataframe is {}. The dataset has {} rows.".format
      (toronto_onehot.shape,toronto_onehot.shape[0]))

# print top 5 rows
toronto_onehot.head()
toronto_grouped = toronto_onehot.groupby('Neighbourhood').mean().reset_index()

# Check the shape of toronto_grouped. The number of rows is equal to that of toronto_data (both are grouped by neighborhood)
print("The shape of the dataframe is {}. The dataset has {} rows.".format
      (toronto_grouped.shape,toronto_grouped.shape[0]))

# Print the top 5 rows
toronto_grouped.head()<jupyter_output>The shape of the dataframe is (39, 231). The dataset has 39 rows.<jupyter_text>Step 7: Clustering<jupyter_code>kclusters = 5

toronto_grouped_clustering = toronto_grouped.drop('Neighbourhood', 1)

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(toronto_grouped_clustering)

# check cluster labels generated for each row in the dataframe
kmeans.labels_
toronto_grouped.insert(0, 'Cluster Labels', kmeans.labels_)

# Left join
toronto_merged=pd.merge(toronto_data,toronto_grouped,how='left',left_on='Neighborhood',right_on='Neighbourhood')

# Delete duplicate neighborhood column
toronto_merged.drop(['Neighbourhood'],axis=1,inplace=True)

# Rename column
toronto_merged.rename(columns={'Neighborhood_x':'Neighborhood'},inplace=True)

# Check top 5 rows
toronto_merged.head()
toronto_merged.shape<jupyter_output><empty_output><jupyter_text>Step 8: Visualizing the resulting cluster<jupyter_code>map_clusters = folium.Map(location=[latitude, longitude], zoom_start=11)

# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i + x + (i*x)**2 for i in range(kclusters)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(toronto_merged['Latitude'], toronto_merged['Longitude'], toronto_merged['Neighborhood'], toronto_merged['Cluster Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09725480994838104, 'fraction_numerical': 0.011731581417175035, 'mean_word_length': 4.172936893203883, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '388289', 'license': 'Apache-2.0', 'n_tokens_mistral': 2801, 'n_tokens_neox': 2638, 'n_words': 708, 'path': 'Assignment-part3.ipynb', 'repo_name': 'rjtrip/Coursera-data-science-capstone-project', 'size': '8567'}