<jupyter_start><jupyter_text>1) Extract data of Toronto neighborhoods from Wikipedia.  - Clean and display the top 10 rows along with shape head - Import Libraries Use pandas, or the BeautifulSoup package, or any other way you are comfortable with to transform the data in the table on the Wikipedia page into the above pandas dataframe.<jupyter_code># importing necessary libraries 
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import folium
import requests
import json
from bs4 import BeautifulSoup
import matplotlib.cm as cm
import matplotlib.colors as colors

%matplotlib inline
print('Packages installed')
#Extracting data from the URL. Used old version of Wiki: 18:01, 8 March 2021 because a new one has another format..
url='https://en.wikipedia.org/w/index.php?title=List_of_postal_codes_of_Canada:_M&oldid=1011037969'
result = requests.get(url)
data_html = BeautifulSoup(result.content)
soup = BeautifulSoup(str(data_html))
neigh = soup.find('table')
table_str = str(neigh.extract())
df = pd.read_html(table_str)[0]
df.head()
df_dropna = df[df.Borough != 'Not assigned'].reset_index(drop=True)
#renaming the colomn for better reading
df_dropna.rename(columns={'Postal Code' : 'PostalCode'}, inplace=True)
#Dropping "Not assigned"
df = df_dropna
#Displaying first 5 rows 
df.head()
#Grouping data based on "Borough"
df_grouped = df.groupby(['Borough', 'PostalCode'], as_index=False).agg(lambda x:','.join(x))
df_grouped.head()
# Checking if there are neighborhoods that are Not Assigned
df_grouped.loc[df_grouped['Borough'].isin(["Not assigned"])]
#adding the Latitude and Longitudes (LL) of each specific location
df = df_grouped
print('The DataFrame shape is', df.shape)<jupyter_output>The DataFrame shape is (103, 3)<jupyter_text>*the dataframe should be group by the Postal code, ending with a dataframe with 103 rows.*  2) Latitudes and Longitudes corresponding to the different PostalCodes<jupyter_code>geo_url = "https://cocl.us/Geospatial_data"

geo_df = pd.read_csv(geo_url)
geo_df.rename(columns={'Postal Code': 'PostalCode'}, inplace=True)
geo_df.head()
# Merging data from two tables
df = pd.merge(df, geo_df, on='PostalCode')
df.head()
# finding how many neighborhoods in each borough
df.groupby('Borough').count()['Neighbourhood']
#finding all the neighborhoods of Toronto
df_toronto = df
df_toronto.head()
#Create list with the boroughs
boroughs = df_toronto['Borough'].unique().tolist()
#Obtaining LL coordinates of Toronto itself
lat_toronto = df_toronto['Latitude'].mean()
lon_toronto = df_toronto['Longitude'].mean()
print('The geographical coordinates of Toronto are {}, {}'.format(lat_toronto, lon_toronto))
# color categorization of each borough
borough_color = {}
for borough in boroughs:
    borough_color[borough]= '#%02X%02X%02X' % tuple(np.random.choice(range(256), size=3)) #Random color
map_toronto = folium.Map(location=[lat_toronto, lon_toronto], zoom_start=10.5)

# adding markers to map
for lat, lng, borough, neighborhood in zip(df_toronto['Latitude'], 
                                           df_toronto['Longitude'],
                                           df_toronto['Borough'], 
                                           df_toronto['Neighbourhood']):
    label_text = borough + ' - ' + neighborhood
    label = folium.Popup(label_text)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color=borough_color[borough],
        fill_color=borough_color[borough],
        fill_opacity=0.8).add_to(map_toronto)  
    
map_toronto
CLIENT_ID = '4510O2EFHUUWKW4WLHQJT2BUYYKD10YZ53DSL1XLQH2IIZES' # your Foursquare ID
CLIENT_SECRET = '<KEY>' # your Foursquare Secret
VERSION = 20200514 # Foursquare API version

print('Credentials Stored')
df.loc[3, 'Neighbourhood']<jupyter_output><empty_output><jupyter_text>*We will analyze the fourth Neighborhood, Davisville*<jupyter_code>law_lat = df.loc[3, 'Latitude']
law_long = df.loc[3, 'Longitude']
law_name = df.loc[3, 'Neighbourhood']


print('Latitude and longitude values of {} are {}, {}.'.format(law_name, 
                                                               law_lat, 
                                                               law_long))<jupyter_output>Latitude and longitude values of Davisville are 43.7043244, -79.3887901.<jupyter_text>*Now, let's get the top 100 venues that are in Davisville within a radius of 500 meters.*<jupyter_code>LIMIT = 100 # limit of number of venues returned by Foursquare API
radius = 500 # define radius

# create URL
url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
    CLIENT_ID, 
    CLIENT_SECRET, 
    VERSION, 
    law_lat, 
    law_long, 
    radius, 
    LIMIT)
url
results = requests.get(url).json()
results
# extracting the category of the venue
def get_category_type(row):
    try:
        categories_list = row['categories']
    except:
        categories_list = row['venue.categories']
        
    if len(categories_list) == 0:
        return None
    else:
        return categories_list[0]['name']
#structuring json into pandas dataframe
venues = results['response']['groups'][0]['items']
    
nearby_venues = pd.json_normalize(venues) # flatten JSON

# filter columns
filtered_columns = ['venue.name', 'venue.categories', 'venue.location.lat', 'venue.location.lng']
nearby_venues =nearby_venues.loc[:, filtered_columns]

# filter the category for each row
nearby_venues['venue.categories'] = nearby_venues.apply(get_category_type, axis=1)

# clean columns
nearby_venues.columns = [col.split(".")[-1] for col in nearby_venues.columns]

nearby_venues.head()
#finding how many venues around Davisville were found
print('{} venues were returned by Foursquare.'.format(nearby_venues.shape[0]))<jupyter_output>34 venues were returned by Foursquare.<jupyter_text>*Exploring other neighborhoods of Toronto*<jupyter_code>def getNearbyVenues(names, latitudes, longitudes, radius=500):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
        print(name)
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Neighbourhood', 
                  'Neighborhood Latitude', 
                  'Neighborhood Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)
#creating a new dataframe with toronto_venues (from the previous request)
toronto_venues = getNearbyVenues(names=df['Neighbourhood'],
                                   latitudes=df['Latitude'],
                                   longitudes=df['Longitude']
                                  )
#getting the size and shape of the dataframe
print(toronto_venues.shape)
toronto_venues.head()
toronto_venues.groupby('Neighbourhood').count()
#Checking how many unique Venues there are that can be curated
print('There are {} uniques categories.'.format(len(toronto_venues['Venue Category'].unique())))<jupyter_output>There are 276 uniques categories.<jupyter_text>Analyzing each neighborhood<jupyter_code>toronto_onehot = pd.get_dummies(toronto_venues[['Venue Category']], prefix="", prefix_sep="")

# adding neighbourhood to DF
toronto_onehot['Neighbourhood'] = toronto_venues['Neighbourhood'] 

# move neighbourhood column to the first column
fixed_columns = [toronto_onehot.columns[-1]] + list(toronto_onehot.columns[:-1])
toronto_onehot = toronto_onehot[fixed_columns]

toronto_onehot.head()
toronto_onehot.shape
toronto_grouped = toronto_onehot.groupby('Neighbourhood').mean().reset_index()
toronto_grouped<jupyter_output><empty_output><jupyter_text>*neighborhood along with the top 3 most common venues:*<jupyter_code>num_top_venues = 3

for hood in toronto_grouped['Neighbourhood']:
    print("----"+hood+"----")
    temp = toronto_grouped[toronto_grouped['Neighbourhood'] == hood].T.reset_index()
    temp.columns = ['venue','freq']
    temp = temp.iloc[1:]
    temp['freq'] = temp['freq'].astype(float)
    temp = temp.round({'freq': 2})
    print(temp.sort_values('freq', ascending=False).reset_index(drop=True).head(num_top_venues))
    print('\n')
#converting into pandas

def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]
#top 10 venues for each neighbourhood
num_top_venues = 10

indicators = ['st', 'nd', 'rd']

# create columns according to number of top venues
columns = ['Neighbourhood']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))

# create a new dataframe
neighborhoods_venues_sorted = pd.DataFrame(columns=columns)
neighborhoods_venues_sorted['Neighbourhood'] = toronto_grouped['Neighbourhood']

for ind in np.arange(toronto_grouped.shape[0]):
    neighborhoods_venues_sorted.iloc[ind, 1:] = return_most_common_venues(toronto_grouped.iloc[ind, :], num_top_venues)

neighborhoods_venues_sorted.head(11)<jupyter_output><empty_output><jupyter_text>Clustering neighbourhoods<jupyter_code>num_top_venues = 10

indicators = ['st', 'nd', 'rd']

# create columns according to number of top venues
columns = ['Neighbourhood']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))

# create a new dataframe
neighborhoods_venues_sorted = pd.DataFrame(columns=columns)
neighborhoods_venues_sorted['Neighbourhood'] = toronto_grouped['Neighbourhood']

for ind in np.arange(toronto_grouped.shape[0]):
    neighborhoods_venues_sorted.iloc[ind, 1:] = return_most_common_venues(toronto_grouped.iloc[ind, :], num_top_venues)

neighborhoods_venues_sorted.head(11)
kmeans = KMeans(n_clusters=3, init='k-means++', max_iter=15, random_state=8)
X = toronto_grouped.drop(['Neighbourhood'], axis=1)

kmeans.fit(X)
kmeans.labels_[0:10]
def get_inertia(n_clusters):
    km = KMeans(n_clusters=n_clusters, init='k-means++', max_iter=15, random_state=8)
    km.fit(X)
    return km.inertia_
scores = [get_inertia(x) for x in range(2, 21)]
plt.figure(figsize=[10, 8])
sns.lineplot(x=range(2, 21), y=scores)
plt.title("K vs Error")
plt.xticks(range(2, 21))
plt.xlabel("K")
plt.ylabel("Error")<jupyter_output><empty_output><jupyter_text>*from the plot we see that K=7 is the best choise*<jupyter_code>kclusters = 7

toronto_grouped_clustering = toronto_grouped.drop('Neighbourhood', 1)

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(toronto_grouped_clustering)

# check cluster labels generated for each row in the dataframe
kmeans.labels_[0:10]
# add clustering labels
neighborhoods_venues_sorted.insert(0, 'Cluster_Labels', kmeans.labels_)

toronto_merged = df

# merge toronto_grouped with toronto_data to add latitude/longitude for each neighborhood
toronto_merged = toronto_merged.join(neighborhoods_venues_sorted.set_index('Neighbourhood'), on='Neighbourhood')

toronto_merged.head()
toronto_merged.tail()
#Invalid types of Clusters or Venues? 
toronto_drop = toronto_merged[toronto_merged.Cluster_Labels != 'NaN'].reset_index(drop=True)
toronto_merged.dropna(axis=0, how='any', thresh=None, subset=None, inplace=True)<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code># create map
map_clusters = folium.Map(location=[lat_toronto, lon_toronto], zoom_start=10.5)

# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i + x + (i*x)**2 for i in range(kclusters)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(toronto_merged['Latitude'], toronto_merged['Longitude'], toronto_merged['Neighbourhood'], toronto_merged['Cluster_Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[int(cluster)-1],
        fill=True,
        fill_color=rainbow[int(cluster)-1],
        fill_opacity=0.75).add_to(map_clusters)
       
map_clusters<jupyter_output><empty_output><jupyter_text>Cluster 1<jupyter_code>toronto_merged.loc[toronto_merged['Cluster_Labels'] == 0, toronto_merged.columns[[1] + list(range(5, toronto_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster 2<jupyter_code>toronto_merged.loc[toronto_merged['Cluster_Labels'] == 1, toronto_merged.columns[[1] + list(range(5, toronto_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster 3<jupyter_code>toronto_merged.loc[toronto_merged['Cluster_Labels'] == 2, toronto_merged.columns[[1] + list(range(5, toronto_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster 4<jupyter_code>toronto_merged.loc[toronto_merged['Cluster_Labels'] == 3, toronto_merged.columns[[1] + list(range(5, toronto_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster 5<jupyter_code>toronto_merged.loc[toronto_merged['Cluster_Labels'] == 5, toronto_merged.columns[[1] + list(range(5, toronto_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster 6<jupyter_code>toronto_merged.loc[toronto_merged['Cluster_Labels'] == 6, toronto_merged.columns[[1] + list(range(5, toronto_merged.shape[1]))]]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12415271821828745, 'fraction_numerical': 0.015423986720154931, 'mean_word_length': 4.626070038910506, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 62, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '631203', 'license': 'MIT', 'n_tokens_mistral': 5062, 'n_tokens_neox': 4765, 'n_words': 1001, 'path': 'Segmenting and Clustering Neighborhoods in Toronto.ipynb', 'repo_name': 'aholovchenko/Coursera_Capstone', 'size': '14501'}