<jupyter_start><jupyter_text>Beginning of Analysis- Visualization<jupyter_code>import numpy as np 
import pandas as pd 
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import json 
from geopy.geocoders import Nominatim 
import requests 
from pandas.io.json import json_normalize 
import matplotlib.cm as cm
import matplotlib.colors as colors
from sklearn.cluster import KMeans
import folium 
from bs4 import BeautifulSoup
import urllib3
tst=pd.read_csv('postcodes')
print('Libraries imported.')<jupyter_output>Libraries imported.<jupyter_text>Quick visualization using folium of the various postal codes  Each bubble contains the neighbourhoods contained in the postcode plus the name of the borough<jupyter_code>latitude=43.6532
longitude=-79.3832
map_Toronto = folium.Map(location=[latitude, longitude], zoom_start=10)

# plotting coordinates of postal code, with Neighbourhood, Borough as labels in popup
for lat, lng, borough, neighborhood in zip(tst['Latitude'], tst['Longitude'], tst['Borough'], tst['Neighbourhoods']):
    label = 'Neighbourhoods contained:{} Borough: {}'.format(neighborhood, borough)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='blue',
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.7).add_to(map_Toronto)  
    
map_Toronto<jupyter_output><empty_output><jupyter_text>Using Foursquare API to get venues around point of interest<jupyter_code>CLIENT_ID = 'QIJAH1HDYSODDWAH4UOHSNJGGNXD1D5WOLU3SFOUWCRRI53T' 
CLIENT_SECRET = '<KEY>' 
VERSION = '20180605'
LIMIT=100

def getNearbyVenues(names, latitudes, longitudes, radius=1000):
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
        print(name)            
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)           
        results = requests.get(url).json()["response"]['groups'][0]['items']        
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])
    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Postcode', 
                  'Postcode Latitude', 
                  'Postcode Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']    
    return(nearby_venues)     
toronto_venues = getNearbyVenues(names=tst['Postcode'],latitudes=tst['Latitude'],longitudes=tst['Longitude'])

toronto_venues.groupby('Postcode').count()
print('There are {} uniques categories.'.format(len(toronto_venues['Venue Category'].unique())))<jupyter_output>There are 329 uniques categories.<jupyter_text>One Hot Encoding for venue categories to be used later<jupyter_code>toronto_onehot = pd.get_dummies(toronto_venues[['Venue Category']], prefix="", prefix_sep="")
toronto_onehot['Postcode'] = toronto_venues['Postcode'] 
fixed_columns = [toronto_onehot.columns[-1]] + list(toronto_onehot.columns[:-1])
toronto_onehot = toronto_onehot[fixed_columns]

toronto_onehot.head()<jupyter_output><empty_output><jupyter_text>Top five venues per neighbourhood and corresponding frequency<jupyter_code>toronto_grouped = toronto_onehot.groupby('Postcode').mean().reset_index()
toronto_grouped.to_csv('torgroup',index=None)
num_top_venues = 5

for hood in toronto_grouped['Postcode']:
    print("----"+hood+"----")
    temp = toronto_grouped[toronto_grouped['Postcode'] == hood].T.reset_index()
    temp.columns = ['venue','freq']
    temp = temp.iloc[1:]
    temp['freq'] = temp['freq'].astype(float)
    temp = temp.round({'freq': 2})
    print(temp.sort_values('freq', ascending=False).reset_index(drop=True).head(num_top_venues))
    print('\n')
def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]<jupyter_output><empty_output><jupyter_text>Splicing each venues top 10 venues into a sorted neighbourhood venues dataframe<jupyter_code>num_top_venues = 10

indicators = ['st', 'nd', 'rd']

columns = ['Postcode']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))


neighborhoods_venues_sorted = pd.DataFrame(columns=columns)
neighborhoods_venues_sorted['Postcode'] = toronto_grouped['Postcode']

for ind in np.arange(toronto_grouped.shape[0]):
    neighborhoods_venues_sorted.iloc[ind, 1:] = return_most_common_venues(toronto_grouped.iloc[ind, :], num_top_venues)

neighborhoods_venues_sorted.to_csv('venuessorted',index=None)    
neighborhoods_venues_sorted<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12788497919031402, 'fraction_numerical': 0.012485811577752554, 'mean_word_length': 4.2764471057884235, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '592322', 'license': 'MIT', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1744, 'n_words': 317, 'path': '02Analysis-Visual.ipynb', 'repo_name': 'shamovsky/NeighborhoodClustering', 'size': '5329'}