<jupyter_start><jupyter_text>Segmenting and Clustering Neighborhoods in Toronto<jupyter_code>import pandas as pd
import numpy as np
import re<jupyter_output><empty_output><jupyter_text>obtain the data that is in the table of postal codes and transform the data into a pandas dataframe<jupyter_code>wikidata = pd.read_html("https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M", 
             match='Postcode', header=0)[0]<jupyter_output><empty_output><jupyter_text>The dataframe will consist of three columns: PostalCode, Borough, and Neighborhood<jupyter_code>wikidata = wikidata.rename(
    mapper={'Postcode':'PostalCode', 'Neighbourhood':'Neighborhood'},
    axis=1)
wikidata.head()<jupyter_output><empty_output><jupyter_text>Only process the cells that have an assigned borough. Ignore cells with a borough that is Not assigned.<jupyter_code>befrows = wikidata.shape[0]
wikidata = wikidata[wikidata['Borough']!='Not assigned']
assert wikidata.shape[0] < befrows, "No rows were removed"<jupyter_output><empty_output><jupyter_text>More than one neighborhood can exist in one postal code area. For example, in the table on the Wikipedia page, you will notice that M5A is listed twice and has two neighborhoods: Harbourfront and Regent Park. These two rows will be combined into one row with the neighborhoods separated with a comma<jupyter_code>grouped = wikidata.groupby(['PostalCode','Borough'], 
                           as_index=False).aggregate(lambda x: ', '.join(x))<jupyter_output><empty_output><jupyter_text>If a cell has a borough but a Not assigned neighborhood, then the neighborhood will be the same as the borough.<jupyter_code>for index, row in grouped.iterrows():
    row['Neighborhood'] = row['Borough'] if re.search('Not assigned', row['Neighborhood']) else row['Neighborhood']<jupyter_output><empty_output><jupyter_text>In the last cell of your notebook, use the .shape method to print the number of rows of your dataframe.<jupyter_code>grouped.shape<jupyter_output><empty_output><jupyter_text>Section 2: Geocoder<jupyter_code>import geocoder
geodata = pd.read_csv("https://cocl.us/Geospatial_data").rename(
    mapper={"Postal Code": "PostalCode"}, axis=1)
geodata['PostalCode'] = geodata['PostalCode'].apply(str)
grouped['PostalCode'] = grouped['PostalCode'].apply(str)
merged = grouped.merge(geodata, on=['PostalCode'])
merged.head()<jupyter_output><empty_output><jupyter_text>Section 3: Explore and cluster neighborhoods  You can decide to work with only boroughs that contain the word Toronto<jupyter_code>tordata = merged[merged['Borough'].str.contains('Toronto')]<jupyter_output><empty_output><jupyter_text>Then replicate the same analysis we did to the New York City data.<jupyter_code>import folium
from geopy.geocoders import Nominatim
from pandas.io.json import json_normalize
import requests
from sklearn.cluster import KMeans
import matplotlib.cm as cm
import matplotlib.colors as colors
address = 'Toronto, ONT'
geolocator = Nominatim(user_agent="tor_explorer")
location = geolocator.geocode(address)
latitude = location.latitude
longitude = location.longitude
#adjust
longitude +=.25
print(f'The geograpical coordinate of Toronto are {latitude}, {longitude}.')
# create map of Toronto using latitude and longitude values
map_toronto = folium.Map(location=[latitude, longitude], zoom_start=11)
# add markers to map
for lat, lng, borough, neighborhood in zip(tordata['Latitude'], 
                                           tordata['Longitude'], 
                                           tordata['Borough'], 
                                           tordata['Neighborhood']):
    label = '{}, {}'.format(neighborhood, borough)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='blue',
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.7,
        parse_html=False).add_to(map_toronto)
map_toronto<jupyter_output>The geograpical coordinate of Toronto are 43.6781255, -79.3821235335026.<jupyter_text>Next, we are going to start utilizing the Foursquare API to explore the neighborhoods and segment them.  Credentials<jupyter_code>with open('/Users/kutch/Desktop/IBM/foursquarecreds.txt') as f:
    fsq = f.read()
client_id = re.search("(?<=CLIENT_ID)[ =]+'([A-Z0-9]+)", fsq).group(1)
client_secret = re.search("(?<=CLIENT_SECRET)[ =]+'([A-Z0-9]+)", fsq).group(1)
#not sure if coursera wants me to update this or not
VERSION = '20180605' # Foursquare API version
#from foursquare
#function that extracts the category of the venue
def get_category_type(row):
    try:
        categories_list = row['categories']
    except:
        categories_list = row['venue.categories']
        
    if len(categories_list) == 0:
        return None
    else:
        return categories_list[0]['name']<jupyter_output><empty_output><jupyter_text>Get all venues<jupyter_code>#for a given list of neighborhoods
def get_nearby_venues(neighborhoods, latitudes, longitudes):
    radius, limit = 500, 100
    global client_id, client_secret 
    venues_list=[]
    for name, lat, lng in zip(neighborhoods, latitudes, longitudes):
        # create the API request URL
        url = f'https://api.foursquare.com/v2/venues/explore?&client_id={client_id}&client_secret={client_secret}&v={VERSION}&ll={lat},{lng}&radius={radius}&limit={limit}'
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])
    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Neighborhood', 
                  'Neighborhood Latitude', 
                  'Neighborhood Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    return(nearby_venues)

toronto_venues = get_nearby_venues(neighborhoods=tordata['Neighborhood'], latitudes=tordata['Latitude'], longitudes=tordata['Longitude'])<jupyter_output><empty_output><jupyter_text>Categorize venues<jupyter_code>toronto_onehot = pd.get_dummies(toronto_venues[['Venue Category']], prefix="", prefix_sep="")
toronto_onehot = toronto_onehot.drop(columns=['Neighborhood'], axis=1)
toronto_onehot.insert(loc=0, 
                      column='Neighborhood', 
                      value=toronto_venues['Neighborhood'])
#find mean frequency of categories by neighborhood
toronto_grouped = toronto_onehot.groupby('Neighborhood').mean().reset_index()<jupyter_output><empty_output><jupyter_text>Most common categories per neighborhood<jupyter_code>def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    return row_categories_sorted.index.values[0:num_top_venues]
def get_common_venues(location_df):
    num_top_venues = 10
    indicators = {1:'st', 2:'nd', 3:'rd'}
    indicators.update(dict((el,'th') for el in range(4, num_top_venues+1)))
    cols = ['Neighborhood']
    for num, ind in indicators.items():
        cols.append(f'{num}{ind} Most Common Venue')
    neighborhoods_venues_sorted = pd.DataFrame(columns=cols)
    neighborhoods_venues_sorted['Neighborhood'] = location_df['Neighborhood']
    for ind in np.arange(location_df.shape[0]):
        neighborhoods_venues_sorted.iloc[ind, 1:] = return_most_common_venues(location_df.iloc[ind, :], num_top_venues)
    return neighborhoods_venues_sorted

neighborhoods_venues_sorted = get_common_venues(toronto_grouped)<jupyter_output><empty_output><jupyter_text>Cluster model<jupyter_code>kclusters = 5
toronto_grouped_clustering = toronto_grouped.drop('Neighborhood', 1)
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(toronto_grouped_clustering)
# add clustering labels
neighborhoods_venues_sorted.insert(0, 'Cluster Labels', kmeans.labels_)
toronto_merged = tordata
toronto_merged = toronto_merged.join(neighborhoods_venues_sorted.set_index('Neighborhood'),on='Neighborhood')<jupyter_output><empty_output><jupyter_text>Visualize the clusters<jupyter_code>map_clusters = folium.Map(location=[latitude, longitude], zoom_start=11)
# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i + x + (i*x)**2 for i in range(kclusters)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]
# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(toronto_merged['Latitude'], toronto_merged['Longitude'], toronto_merged['Neighborhood'], toronto_merged['Cluster Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11672942442173212, 'fraction_numerical': 0.010435718128025821, 'mean_word_length': 4.678680513133782, 'pattern_counts': {'":': 1, '<': 67, '<?xml version=': 0, '>': 64, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '67568', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3112, 'n_tokens_neox': 2955, 'n_words': 644, 'path': 'Segmenting and Clustering Neighborhoods in Toronto, Canada3.ipynb', 'repo_name': 'davidschneider04/Coursera_Capstone', 'size': '9295'}