<jupyter_start><jupyter_text>Clustering Neighbourhoods in Toronto Alifa Izzan  Gathering Data From Wikipedia<jupyter_code>import pandas as pd
from pandas import DataFrame
import numpy as np
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as bsoup
import lxml
import requests
url='https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M' 
r=requests.get(url)
page=bsoup(r.text,"html.parser")
page<jupyter_output><empty_output><jupyter_text>finding the table<jupyter_code>rtable=page.table
rtable
results=rtable.find_all('tr')
nrows=len(results)
nrows
results[0:5]
header=results[0].text.split()
header
records =[]
n=1
while n < nrows :
    Postcode=results[n].text.split('\n')[1]
    Borough=results[n].text.split('\n')[3]
    Neighborhood=results[n].text.split('\n')[5]
    records.append((Postcode, Borough,Neighborhood))
    n=n+1

df=pd.DataFrame(records, columns=['PostalCode', 'Borough', 'Neighbourhood'])
df.head()


df.shape
df.tail()    #  Perfect! all data is included.<jupyter_output><empty_output><jupyter_text>Preprocess<jupyter_code># How many rows have Borough equal to 'Not assigned'?
df[df['Borough']=='Not assigned'].count()
df1=df[~df.Borough.str.contains("Not assigned")]
df1=df1.reset_index(drop=True)
df1.head()
df1.shape
df1.loc[df1['Neighbourhood'] == 'Not assigned', 'Neighbourhood'] = df1['Borough']
postalcodes = df1['PostalCode'].nunique()
boroughs = df1['Borough'].nunique()
neighbourhoods= df1['Neighbourhood'].nunique()
print('Unique Postalcodes : ' + str(postalcodes))
print('Unique Boroughs  : '+ str(boroughs))
print('Unique Neighbourhoods  :' + str(neighbourhoods))

     
nrows1=len(df1)
nrows1
#df2 
df2=df1
df2.head()
nrows2=len(df2)-1
nrows2<jupyter_output><empty_output><jupyter_text>Extract Unique Post Code<jupyter_code>n=0
while n < nrows2 :
    post1=df2.iloc[n,0]
    m=n+1
    post2=df2.iloc[m,0]
    neigh1=df2.iloc[n,2]
    neigh2=df2.iloc[m,2]
    if post1==post2:
        df2.Neighbourhood[n,2] = neigh1=neigh1+','+neigh2
        df2=df2.drop(df2.index[m])
        nrows2=nrows2-1
        df2 = df2.reset_index(drop=True)
    else:
        n=n+1
df2.index
df2.head()
df2.index
df2.head()
df2.shape<jupyter_output><empty_output><jupyter_text>Part 2<jupyter_code>import json 
url='http://cocl.us/Geospatial_data'
df_pcodes=pd.read_csv(url)
df_pcodes.head()
df_pcodes.columns = ['PostalCode', 'Latitude', 'Longitude']
df_pcodes.head()
df2s=df2.sort_values('PostalCode')
df2s.head()
neighborhoods=pd.merge(df2s,df_pcodes, how='right', on = 'PostalCode')
neighborhoods.head()
neighborhoods.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15411222136817831, 'fraction_numerical': 0.027671022290545733, 'mean_word_length': 6.270949720670391, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '246828', 'license': 'MIT', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1051, 'n_words': 160, 'path': 'Week 3/Toronto Cluster - 2 .ipynb', 'repo_name': 'Symefa/Coursera_Capstone-Madiun-Cafe', 'size': '2602'}