<jupyter_start><jupyter_text>Tokyo vs Osaka  Scrape wikipedia for wards<jupyter_code>!conda install -c anaconda beautifulsoup4 -y
import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
from geopy.geocoders import Nominatim

# get wards for osaka
response = requests.get('https://en.wikipedia.org/wiki/Category:Wards_of_Osaka')
soup = BeautifulSoup(response.content)
soup.title
columns = ['City', 'Ward', 'Latitude', 'Longitude']
df = pd.DataFrame(columns=columns)

for item in soup.find('div', {'id': 'mw-pages'}).find_all('li'):
    ward = item.a.get_text()[0:-7]
    geolocator = Nominatim(user_agent="final-proj")
    location = geolocator.geocode('{}, Osaka'.format(ward))
    df = df.append({'City': 'Osaka',
                    'Ward': ward,
                    'Latitude': location.latitude,
                    'Longitude': location.longitude}, ignore_index=True)
df
# get wards for tokyo
response = requests.get('https://en.wikipedia.org/wiki/Special_wards_of_Tokyo')
soup = BeautifulSoup(response.content)
soup.title
for item in soup.find_all('table')[3].find_all('tr'):
    if item.th is not None:
        continue
    try:
        ward = item.find_all('td')[2].a.get_text()
    except:
        break
    
    geolocator = Nominatim(user_agent="final-proj")
    location = geolocator.geocode('{}, Tokyo'.format(ward))
    df = df.append({'City': 'Tokyo',
                    'Ward': ward,
                    'Latitude': location.latitude,
                    'Longitude': location.longitude}, ignore_index=True)

df
osaka_wards_count = df[df['City'] == 'Osaka']['Ward'].unique().size
tokyo_wards_count = df[df['City'] == 'Tokyo']['Ward'].unique().size

print('There are {} number of wards in Tokyo and {} number of wards in Osaka.'
      .format(tokyo_wards_count, osaka_wards_count))

df.to_csv('dataset.csv')
geolocator = Nominatim(user_agent="final-proj")
tokyo = geolocator.geocode('Tokyo')
osaka = geolocator.geocode('Osaka')
df = pd.read_csv('dataset.csv', index_col=0)
df.head()<jupyter_output><empty_output><jupyter_text>Visualise data points<jupyter_code>import folium

map = folium.Map(location=[ tokyo.latitude, tokyo.longitude], zoom_start=11)

# add markers
for lat, lng, city, ward in zip(df['Latitude'], df['Longitude'], df['City'], df['Ward']):
    label = '{}, {}'.format(city, ward)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='blue',
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.7,
        parse_html=False).add_to(map) 
map<jupyter_output><empty_output><jupyter_text>Map of Tokyo![map of tokyo](./screenshots/tokyo_map.png) Map of Osaka![map of osaka](./screenshots/osaka_map.png)  Determine size of radius to look for venues<jupyter_code># scrape wikipedia for ward area sizes
response = requests.get('https://en.wikipedia.org/wiki/Wards_of_Japan')
soup = BeautifulSoup(response.content)
soup.title
df_area_tokyo = pd.DataFrame(columns=['Ward', 'Area'])

# get area sizes for tokyo
for item in soup.find_all('table')[3].find_all('tr'):
    if item.th is not None:
        continue
    area = item.find_all('td')[3].get_text().strip('\n')
    ward = item.find_all('td')[0].get_text()
    df_area_tokyo = df_area_tokyo.append({'Ward': ward, 'Area': area}, ignore_index=True)

df_area_tokyo
median_area_tokyo = df_area_tokyo['Area'].median()
print('The median area of each ward in Tokyo is {} km.'.format(median_area_tokyo))
# calculate radius in metres assuming area is a circle
radius_tokyo = math.sqrt(median_area_tokyo / np.pi) * 1000
print('The median radius of wards in Tokyo is {} metres.'.format(radius_tokyo))
# Repeat for osaka
df_area_osaka = pd.DataFrame(columns=['Ward', 'Area'])

# get area sizes for tokyo
for item in soup.find_all('table')[2].find_all('tr'):
    if item.th is not None:
        continue
        
    city = item.find_all('td')[2].get_text()
    if city != 'Osaka':
        continue
    
    area = item.find_all('td')[3].get_text().strip('\n')
    ward = item.find_all('td')[0].get_text()
    df_area_osaka = df_area_osaka.append({'Ward': ward, 'Area': area}, ignore_index=True)

df_area_osaka
median_area_osaka = df_area_osaka['Area'].median()
print('The median area of each ward in Osaka is {} km.'.format(median_area_osaka))

# calculate radius in metres assuming area is a circle
radius_osaka = math.sqrt(median_area_osaka / np.pi) * 1000
print('The median radius of wards in Osaka is {} metres.'.format(radius_osaka))<jupyter_output>The median area of each ward in Osaka is 8.375 km.
The median radius of wards in Osaka is 1632.7416503504917 metres.<jupyter_text>Get nearby venues<jupyter_code>import json

# import foursquare credentials
with open('credentials.json') as f:
    data = json.load(f)
    CLIENT_ID = data['CLIENT_ID']
    CLIENT_SECRET = data['CLIENT_SECRET']
    VERSION = '20190522'
# get venues within wards

import requests

LIMIT = 100

def getNearbyVenues(cities, names, latitudes, longitudes):
    venues_list=[]
    for city, name, lat, lng in zip(cities, names, latitudes, longitudes):        
        if city == 'Tokyo':
            radius = radius_tokyo
        elif city == 'Osaka':
            radius = radius_osaka
        
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Ward', 
                  'Ward Latitude', 
                  'Ward Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)

venues = getNearbyVenues(df['City'], df['Ward'], df['Latitude'], df['Longitude'])
venues.head()
venues.shape
print('There are {} uniques categories.'.format(len(venues['Venue Category'].unique())))<jupyter_output>There are 275 uniques categories.<jupyter_text>Find top venues for each ward<jupyter_code># one hot encoding
onehot = pd.get_dummies(venues[['Venue Category']], prefix="", prefix_sep="")

# add neighborhood column back to dataframe
onehot['Ward'] = venues['Ward'] 

# move neighborhood column to the first column
fixed_columns = [onehot.columns[-1]] + list(onehot.columns[:-1])
onehot = onehot[fixed_columns]

onehot.head()
grouped = onehot.groupby('Ward').mean().reset_index()
grouped.head()
grouped.shape
def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]

num_top_venues = 10

indicators = ['st', 'nd', 'rd']

# create columns according to number of top venues
columns = ['Ward']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))

# create a new dataframe
venues_sorted = pd.DataFrame(columns=columns)
venues_sorted['Ward'] = grouped['Ward']

for ind in np.arange(grouped.shape[0]):
    venues_sorted.iloc[ind, 1:] = return_most_common_venues(grouped.iloc[ind, :], num_top_venues)

venues_sorted.head()<jupyter_output><empty_output><jupyter_text>Apply clustering algorithm<jupyter_code>from sklearn.cluster import KMeans

# set number of clusters
kclusters = 6

grouped_clustering = grouped.drop('Ward', 1)

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(grouped_clustering)

# check cluster labels generated for each row in the dataframe
kmeans.labels_
# add clustering labels
venues_sorted.insert(0, 'Cluster Labels', kmeans.labels_)

merged = pd.read_csv('dataset.csv', index_col=0)

merged = merged.join(venues_sorted.set_index('Ward'), on='Ward')

merged.head()
import matplotlib.cm as cm
import matplotlib.colors as colors

# create map
map_clusters = folium.Map(location=[tokyo.latitude, tokyo.longitude], zoom_start=11)

# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i + x + (i*x)**2 for i in range(kclusters)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(merged['Latitude'], merged['Longitude'], merged['Ward'], merged['Cluster Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters<jupyter_output><empty_output><jupyter_text>Tokyo and Osaka![map of tokyo and osaka clustered](./screenshots/tokyo_osaka_clustered.png) Tokyo![map of tokyo clustered](./screenshots/tokyo_clustered.png) Tokyo and Osaka![map of osaka clustered](./screenshots/osaka_clustered.png) Putting the two cities side by side, we can see that although these are two large cities in Japan, based on the venue categories in each city, both cities are unique in its own ways.  Analyse each cluster<jupyter_code># Find unique cluster labels
merged['Cluster Labels'].unique()
merged[merged['Cluster Labels'] == 0]
merged[merged['Cluster Labels'] == 1]
merged[merged['Cluster Labels'] == 2]
merged[merged['Cluster Labels'] == 3]
merged[merged['Cluster Labels'] == 4]
merged[merged['Cluster Labels'] == 5]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13776999416228838, 'fraction_numerical': 0.009826814555360965, 'mean_word_length': 3.8854562737642584, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '652805', 'license': 'MIT', 'n_tokens_mistral': 3610, 'n_tokens_neox': 3439, 'n_words': 785, 'path': 'Tokyo_vs_Osaka.ipynb', 'repo_name': 'dsychin/Coursera_Capstone', 'size': '10278'}