<jupyter_start><jupyter_text>Capstone Project  Capstone project guided by Coursera. All subprojects are given in one notebook.<jupyter_code>import pandas as pd
import numpy as np

print('Hello Capstone Project Course')<jupyter_output>Hello Capstone Project Course<jupyter_text>Part 1. Clustering neighborhoods in Toronto<jupyter_code># Private information, which will be used in this notebook

key = ' ???? '  # Google Geocoder API key
CLIENT_ID = ' ???? ' # my Foursquare ID
CLIENT_SECRET = ' ???? ' # my Foursquare Secret
VERSION = '20180605' # Foursquare API version
LIMIT = 100 # A default Foursquare API limit value<jupyter_output><empty_output><jupyter_text>Task 1. Create Dataframe.<jupyter_code>from bs4 import BeautifulSoup as bs
import pandas as pd
import numpy as np
import requests
url = 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'
raw_data = requests.get(url).text
soup = bs(raw_data, 'html5lib')

table_contents=[]
table=soup.find('table')

for row in table.findAll('td'):
    cell = {}
    if (row.span.text=='Not assigned'):
        pass
    else:
        cell['PostalCode'] = row.p.text[:3]
        cell['Borough'] = (row.span.text).split('(')[0]
        cell['Neighborhood'] = (((((row.span.text).split('(')[1]).strip(')')).replace(' /',',')).replace(')',' ')).strip(' ')
        table_contents.append(cell)
        
df = pd.DataFrame(table_contents)
df['Borough']=df['Borough'].replace({'Downtown TorontoStn A PO Boxes25 The Esplanade':'Downtown Toronto Stn A',
                                             'East TorontoBusiness reply mail Processing Centre969 Eastern':'East Toronto Business',
                                             'EtobicokeNorthwest':'Etobicoke Northwest','East YorkEast Toronto':'East York/East Toronto',
                                             'MississaugaCanada Post Gateway Processing Centre':'Mississauga'})

df.dropna(how = 'any', inplace = True)
df.head()

df.shape<jupyter_output><empty_output><jupyter_text>Task 2. Getting locations data.Note!  Please, be kindly informed that because I am using Google API and not Python Geocoderpackage, I can collect more precise locations for every neighbor even with thesame Postal Code. So probably my dataframe final shape can be longer than for those whoare using Python Geocoder.<jupyter_code>#    I am using Google Geocoding Api, but I need to hide my API key for privacy reasons


import googlemaps


gmaps = googlemaps.Client(key=key)



df_with_coord = {'index':[], 'pc':[], 'borough':[], 'neighborhood':[], 'lat':[], 'long':[]}  

index = 0  # for being sure that neighbors with the same Postal Code will not collapse, 
           # indexing will be used

for hood, borough, pc in zip(df.Neighborhood, df.Borough, df.PostalCode):
    
    # Firstly, care about hoods with two or more names per row
    
    if ',' in hood:
        hood_splitted = hood.split(',')
        for name in hood_splitted:
            
            index += 1
            
            # for every name find location, using Geocodong API
            
            location_str = f'{name}, {borough}'
            geocode_result = gmaps.geocode(location_str)
            lat = geocode_result[0]['geometry']['location']['lat']
            long = geocode_result[0]['geometry']['location']['lng']
            
            # update dict
            
            df_with_coord['index'].append(index)
            df_with_coord['pc'].append(pc)
            df_with_coord['borough'].append(borough)
            df_with_coord['neighborhood'].append(name)
            df_with_coord['lat'].append(lat)
            df_with_coord['long'].append(long)
            
    else:
        
        index += 1
            
        # for every hood find location, using Geocodong API

        location_str = f'{hood}, {borough}'
        geocode_result = gmaps.geocode(location_str)
        lat = geocode_result[0]['geometry']['location']['lat']
        long = geocode_result[0]['geometry']['location']['lng']

        # update dict

        df_with_coord['index'].append(index)
        df_with_coord['pc'].append(pc)
        df_with_coord['borough'].append(borough)
        df_with_coord['neighborhood'].append(hood)
        df_with_coord['lat'].append(lat)
        df_with_coord['long'].append(long)





# Creating final dataframe from dictionary with latitudes and longitudes

final_data = pd.DataFrame(df_with_coord)
final_data.drop('index', axis = 1, inplace = True)
final_data.columns = ['Postal_Code', 'Borough', 'Neighborhood', 'Latitude', 'Longitude']
final_data.head()<jupyter_output><empty_output><jupyter_text>Task 3. Clustering the neighborhoods of one borough.<jupyter_code>#   I want to find borough with highest number of neighborhoods and use it
# for analysis

boroughs = list(final_data.Borough.unique())
length = []

for i in range(len(boroughs)):
    length.append(len(final_data[(final_data.Borough == boroughs[i])]))
    
index_of_search = length.index(max(length))


if length.count(max(length)) > 1:
    print(f'There are {length.count(max(length))} Boroughs with max number of neighborhoods')
else:
    print (f'We will use this borough with highest number of neighborhoods: \n {boroughs[index_of_search]}')

    
data_etobicoke = final_data[(final_data.Borough == 'Etobicoke')].reset_index(drop=True)
data_etobicoke.head()

# function that finds nearby venues

    
def getNearbyVenues(names, latitudes, longitudes, radius=500):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Neighborhood', 
                  'Neighborhood Latitude', 
                  'Neighborhood Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)

etobicoke_venues = getNearbyVenues(data_etobicoke.Neighborhood, data_etobicoke.Latitude, 
                                   data_etobicoke.Longitude, radius=500)


# Venues dataframe

print(etobicoke_venues.shape)
etobicoke_venues.head()<jupyter_output>(388, 7)<jupyter_text>Let's check how many venues were returned for each neighborhood<jupyter_code>etobicoke_venues.groupby('Neighborhood').count()<jupyter_output><empty_output><jupyter_text>Let's find out how many unique categories can be curated from all the returned venues<jupyter_code>print('There are {} uniques categories.'.format(len(etobicoke_venues['Venue Category'].unique())))<jupyter_output>There are 97 uniques categories.<jupyter_text>Neighborhood analysis<jupyter_code># one hot encoding
etobicoke_onehot = pd.get_dummies(etobicoke_venues[['Venue Category']], prefix="", prefix_sep="")

# add neighborhood column back to dataframe
etobicoke_onehot['Neighborhood'] = etobicoke_venues['Neighborhood'] 

# move neighborhood column to the first column
fixed_columns = [etobicoke_onehot.columns[-1]] + list(etobicoke_onehot.columns[:-1])
etobicoke_onehot = etobicoke_onehot[fixed_columns]

etobicoke_onehot.head()
etobicoke_onehot.shape<jupyter_output><empty_output><jupyter_text>Next, let's group rows by neighborhood and by taking the mean of the frequency of occurrence of each category<jupyter_code>etobicoke_grouped = etobicoke_onehot.groupby('Neighborhood').mean().reset_index()
etobicoke_grouped<jupyter_output><empty_output><jupyter_text>New size:<jupyter_code>etobicoke_grouped.shape<jupyter_output><empty_output><jupyter_text>Let's print each neighborhood along with the top 5 most common venues<jupyter_code>num_top_venues = 5

for hood in etobicoke_grouped['Neighborhood']:
    print("----"+hood+"----")
    temp = etobicoke_grouped[etobicoke_grouped['Neighborhood'] == hood].T.reset_index()
    temp.columns = ['venue','freq']
    temp = temp.iloc[1:]
    temp['freq'] = temp['freq'].astype(float)
    temp = temp.round({'freq': 2})
    print(temp.sort_values('freq', ascending=False).reset_index(drop=True).head(num_top_venues))
    print('\n')<jupyter_output>---- Albion Gardens----
                  venue  freq
0  Caribbean Restaurant   0.4
1      Asian Restaurant   0.2
2              Pharmacy   0.2
3           Supermarket   0.2
4   American Restaurant   0.0


---- Beaumond Heights----
                  venue  freq
0     Indian Restaurant   0.3
1  Caribbean Restaurant   0.2
2                  Bank   0.1
3           Pizza Place   0.1
4        Ice Cream Shop   0.1


---- Bloordale Gardens----
               venue  freq
0  Convenience Store  0.25
1      Deli / Bodega  0.12
2               Park  0.12
3        Coffee Shop  0.12
4     Sandwich Place  0.12


---- Cloverdale----
                  venue  freq
0  Fast Food Restaurant  0.10
1           Coffee Shop  0.10
2           Supermarket  0.10
3      Department Store  0.05
4           Gas Station  0.05


---- Humber Bay----
                venue  freq
0         Coffee Shop  0.16
1  Italian Restaurant  0.05
2       Smoothie Shop  0.05
3         Pizza Place  0.05
4      Farmers Market  0.05


---[...]<jupyter_text>Now we will put that into _pandas_ dataframe<jupyter_code># function for sorting the venues in descending order.

def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]<jupyter_output><empty_output><jupyter_text>New dataframe with the top 10 venues for each neighborhood.<jupyter_code>num_top_venues = 10

indicators = ['st', 'nd', 'rd']

# create columns according to number of top venues
columns = ['Neighborhood']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))

# create a new dataframe
neighborhoods_venues_sorted = pd.DataFrame(columns=columns)
neighborhoods_venues_sorted['Neighborhood'] = etobicoke_grouped['Neighborhood']

for ind in np.arange(etobicoke_grouped.shape[0]):
    neighborhoods_venues_sorted.iloc[ind, 1:] = return_most_common_venues(etobicoke_grouped.iloc[ind, :], num_top_venues)

neighborhoods_venues_sorted.head()<jupyter_output><empty_output><jupyter_text>And finally, clustering<jupyter_code>etobicoke_grouped_clustering.shape
from sklearn.cluster import KMeans
import folium
import matplotlib.cm as cm
import matplotlib.colors as colors


# set number of clusters
kclusters = 5

etobicoke_grouped_clustering = etobicoke_grouped.drop('Neighborhood', 1)

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(etobicoke_grouped_clustering)

# check cluster labels generated for each row in the dataframe
kmeans.labels_[0:10]<jupyter_output><empty_output><jupyter_text>Let's create a new dataframe that includes the cluster as well as the top 10 venues for each neighborhood.<jupyter_code># add clustering labels
neighborhoods_venues_sorted.insert(0, 'Cluster Labels', kmeans.labels_)

etobicoke_merged = data_etobicoke

# merge manhattan_grouped with manhattan_data to add latitude/longitude for each neighborhood
etobicoke_merged = etobicoke_merged.join(neighborhoods_venues_sorted.set_index('Neighborhood'), on='Neighborhood')
etobicoke_merged.drop('Postal_Code', axis = 1, inplace=True)
etobicoke_merged.head() # check the last columns!<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>geocode_result = gmaps.geocode('Etobicoke, Toronto')
latitude = geocode_result[0]['geometry']['location']['lat']
longitude = geocode_result[0]['geometry']['location']['lng']

# create map
map_clusters = folium.Map(location=[latitude, longitude], zoom_start=11)

# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i + x + (i*x)**2 for i in range(kclusters)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(etobicoke_merged['Latitude'], etobicoke_merged['Longitude'], etobicoke_merged['Neighborhood'], etobicoke_merged['Cluster Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters<jupyter_output><empty_output><jupyter_text>In case if map is not showing in the Github, I took the screenshot![map.jpg](attachment:b3b4c801-049d-4c7f-8a20-b121d049f4f9.jpg)<jupyter_code>etobicoke_merged[(etobicoke_merged['Cluster Labels'] == 0)]
etobicoke_merged.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11570734579714412, 'fraction_numerical': 0.014721036360959812, 'mean_word_length': 3.5471887550200805, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 66, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '196470', 'license': 'MIT', 'n_tokens_mistral': 4561, 'n_tokens_neox': 4245, 'n_words': 999, 'path': 'Capstone.ipynb', 'repo_name': 'To-to-je/Coursera_Capstone', 'size': '13586'}