<jupyter_start><jupyter_text>Urban Diversity Within Seattle’s Neighborhoods  Contents1. Data Collection2. Data Cleaning3. Exploration4. Modeling5. Interpretation Data Collection  Web scraping neighborhood list from  https://seattle.findwell.com/seattle-neighborhoods/<jupyter_code>from bs4 import BeautifulSoup
import requests                   #Get the webpage
import pandas as pd
def getSoup(url):
    webpage = requests.get(url)
    return webpage, BeautifulSoup(webpage.content, 'html.parser')
#obtain the website and display html content

page, soup = getSoup('https://seattle.findwell.com/seattle-neighborhoods/')
soup
seaList = soup.findAll(class_="col-sm-2")
len(seaList)
a = seaList[0]
temp = a.findAll('a')
central = [name.get_text() for name in temp]

central
central_df = pd.DataFrame(central)
central_df['Zone'] = 'Central'
central_df
b = seaList[1]
temp = b.findAll('a')
north_east = [name.get_text() for name in temp]
north_east
ne_df = pd.DataFrame(north_east)
ne_df['Zone'] = 'North East'
ne_df
c = seaList[2]
temp = c.findAll('a')
north_west = [name.get_text() for name in temp]
north_west
nw_df = pd.DataFrame(north_west)
nw_df['Zone'] = 'North West'
nw_df
d = seaList[3]
temp = d.findAll('a')
south = [name.get_text() for name in temp]
south
s_df = pd.DataFrame(south)
s_df['Zone'] = 'South'
s_df
e = seaList[4]
temp = e.findAll('a')
west = [name.get_text() for name in temp]
west
# Delete 'West Seattle'
del west[0]
west
west_df = pd.DataFrame(west)
west_df['Zone'] = 'West'
west_df
frames = [central_df, ne_df, nw_df, s_df, west_df]
neigh_df = pd.concat(frames, ignore_index=True)
neigh_df
# rename first column
neigh_df.rename(columns={0:"Neighborhood"}, inplace=True)
neigh_df.head()
# save to csv
sea_neigh_list.to_csv('Data/sea_neigh_list.csv', index=False)<jupyter_output><empty_output><jupyter_text>Latitud and Longitude for each neighborhood<jupyter_code>#read neighborhoods df

neighborhoods = pd.read_csv('Data/sea_neigh_list.csv')
neighborhoods.head()
#read coordinates df
coordinates = pd.read_csv('Data/coordinates_shaped.csv')
coordinates.head()
# @hidden_cell
# check how to Load data from local files IBM https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/load-and-access-data.html

body = client_11766bbc34154972b7ed0fd38ff1d66a.get_object(Bucket='ourserafinal-donotdelete-pr-vjm5icmxxrql9r',Key='data_asset/coordinates.csv_shaped_f635055.csv')['Body']
# add missing __iter__ method, so pandas accepts body as file-like object
if not hasattr(body, "__iter__"): body.__iter__ = types.MethodType( __iter__, body )

coordinates = pd.read_csv(body)
import types
from botocore.client import Config
import ibm_boto3

def __iter__(self): return 0

# @hidden_cell
# The following code accesses a file in your IBM Cloud Object Storage. It includes your credentials.
# You might want to remove those credentials before you share the notebook.
client_11766bbc34154972b7ed0fd38ff1d66a = ibm_boto3.client(service_name='s3',
    ibm_api_key_id='<KEY>',
    ibm_auth_endpoint="https://iam.ng.bluemix.net/oidc/token",
    config=Config(signature_version='oauth'),
    endpoint_url='https://s3-api.us-geo.objectstorage.service.networklayer.com')

body = client_11766bbc34154972b7ed0fd38ff1d66a.get_object(Bucket='ourserafinal-donotdelete-pr-vjm5icmxxrql9r',Key='sea_neigh_list.csv')['Body']
# add missing __iter__ method, so pandas accepts body as file-like object
if not hasattr(body, "__iter__"): body.__iter__ = types.MethodType( __iter__, body )

neighborhoods = pd.read_csv(body)<jupyter_output><empty_output><jupyter_text>Merge both data frames<jupyter_code>data = pd.merge(neighborhoods, coordinates, on='Neighborhood', how='inner')
data.head()
data.shape
# save to csv
data.to_csv('Data/data.csv', index=False)<jupyter_output><empty_output><jupyter_text>Exploration and Clustering<jupyter_code>data = pd.read_csv('Data/data.csv')
data.head()
#!pip install folium
import folium
latitude = 47.60621
longitude = -122.33207
seattle_map = folium.Map(location=[latitude, longitude], zoom_start=11)

# add markers for B
for lat, lng, label in zip(data.Latitude, data.Longitude, data.Neighborhood):
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker([lat, lng], radius=5, popup=label, color='purple', fill=True, parse_html=False).add_to(seattle_map)

seattle_map<jupyter_output><empty_output><jupyter_text>Explore Neighborhoods with Foursquare API<jupyter_code>import numpy as np
import requests
# @hidden_cell

CLIENT_ID = 'SRJ3OGDDEJTQEZBPDGROOVSOUF1ROULP0G4NRGTRJBLE5TS2' # your Foursquare ID
CLIENT_SECRET = '<KEY>' # your Foursquare Secret
VERSION = '20200104' # Foursquare API version
LIMIT = 100
def getVenues(zones, names, latitudes, longitudes, radius=500):    #radius -> meters
    
    venues_list=[]
    
    for zone, name, lat, lng in zip (zones, names, latitudes, longitudes):
        print(name)
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            zone,
            name, 
            v['venue']['name'], 
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Zone', 'Neighborhood', 'Venue', 'Venue Category']
    
    return(nearby_venues)
seattle_venues = getVenues(zones = data['Zone'],
                           names = data['Neighborhood'],
                           latitudes = data['Latitude'],
                           longitudes = data['Longitude']
                          )
print(seattle_venues.shape)
seattle_venues.head(10)
#Unique categories

print(len(seattle_venues['Venue Category'].unique()), 'unique venues')
print(seattle_venues['Venue Category'].unique())
seattle_venues.describe()<jupyter_output><empty_output><jupyter_text>10 most popular venues in each neighborhood<jupyter_code># one hot encoding
seattle_onehot = pd.get_dummies(seattle_venues[['Venue Category']], prefix="", prefix_sep="")

# add neighborhood column back to dataframe
seattle_onehot['Neighborhood'] = seattle_venues['Neighborhood'] 

# move neighborhood column to the first column
fixed_columns = [seattle_onehot.columns[-1]] + list(seattle_onehot.columns[:-1])
seattle_onehot = seattle_onehot[fixed_columns]

#seattle_onehot.head()
seattle_onehot.shape
# group rows by neighborhood and by taking the mean of the frequency of occurrence of each category
sea_grouped = seattle_onehot.groupby('Neighborhood').mean().reset_index()
sea_grouped.shape
top_venues = 10

for hood in sea_grouped['Neighborhood']:
    print("----" + hood + "----")
    temp = sea_grouped[sea_grouped['Neighborhood'] == hood].T.reset_index()
    temp.columns = ['Venue','Freq.']
    temp = temp.iloc[1:]
    temp['Freq.'] = temp['Freq.'].astype(float)
    temp = temp.round({'Freq.': 2})
    print(temp.sort_values('Freq.', ascending=False).reset_index(drop=True).head(top_venues))
    print('\n')<jupyter_output>----Admiral----
                  Venue  Freq.
0           Coffee Shop   0.12
1                   Pub   0.05
2         Grocery Store   0.05
3       Thai Restaurant   0.02
4    Salon / Barbershop   0.02
5     Convenience Store   0.02
6                Market   0.02
7  Gym / Fitness Center   0.02
8    Falafel Restaurant   0.02
9          Cocktail Bar   0.02


----Alki----
                Venue  Freq.
0         Coffee Shop   0.08
1               Beach   0.06
2                Park   0.06
3          Restaurant   0.06
4  Italian Restaurant   0.06
5         Art Gallery   0.06
6          Sports Bar   0.03
7   Fish & Chips Shop   0.03
8           Beach Bar   0.03
9              Market   0.03


----Ballard----
               Venue  Freq.
0         Hobby Shop   0.17
1  French Restaurant   0.17
2      Jewelry Store   0.17
3               Park   0.17
4          Gift Shop   0.17
5        Coffee Shop   0.17
6      Movie Theater   0.00
7             Museum   0.00
8              Motel   0.00
9        Mu[...]<jupyter_text>Making Clusters<jupyter_code>#Create a DF with the common venues

def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]
top_venues = 10

indicators = ['st', 'nd', 'rd']

# create columns according to number of top venues
columns = ['Neighborhood']
for ind in np.arange(top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))
venues_sorted = pd.DataFrame(columns=columns)
venues_sorted['Neighborhood'] = sea_grouped['Neighborhood']

for ind in np.arange(sea_grouped.shape[0]):
    venues_sorted.iloc[ind, 1:] = return_most_common_venues(sea_grouped.iloc[ind, :], top_venues)
venues_sorted.head()
from sklearn.cluster import KMeans

import matplotlib.cm as cm
import matplotlib.colors as colors
# set number of clusters
kclusters = 5

sea_clusters = sea_grouped.drop('Neighborhood', 1)

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(sea_clusters)

# check cluster labels generated for each row in the dataframe
kmeans.labels_[0:10]
# add clustering labels
venues_sorted.insert(0, 'Cluster Labels', kmeans.labels_)

sea_merged = data

# merge toronto_grouped with toronto_data to add latitude/longitude for each neighborhood
sea_merged = sea_merged.join(venues_sorted.set_index('Neighborhood'), on='Neighborhood')
sea_merged.head()
# create map
map_clusters = folium.Map(location=[latitude, longitude], zoom_start=11)

# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i + x + (i*x)**2 for i in range(kclusters)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(sea_merged['Latitude'], sea_merged['Longitude'], sea_merged['Neighborhood'], sea_merged['Cluster Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters
sea_merged
import matplotlib.pyplot as plt
sea_merged.groupby(['Zone', '1st Most Common Venue']).size().unstack().plot(kind='barh', figsize=(20,20), stacked=True)<jupyter_output><empty_output><jupyter_text>Cluster Examination  Cluster  1<jupyter_code>#Cluster 0
sea_merged.loc[sea_merged['Cluster Labels'] == 0, sea_merged.columns[[0] + [1] + list(range(5, sea_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster  2<jupyter_code>#Cluster 1
sea_merged.loc[sea_merged['Cluster Labels'] == 1, sea_merged.columns[[0] + [1] + list(range(5, sea_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster  3<jupyter_code>sea_merged.loc[sea_merged['Cluster Labels'] == 2, sea_merged.columns[[0] + [1] + list(range(5, sea_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster  4<jupyter_code>sea_merged.loc[sea_merged['Cluster Labels'] == 3, sea_merged.columns[[0] + [1] + list(range(5, sea_merged.shape[1]))]]<jupyter_output><empty_output><jupyter_text>Cluster  5<jupyter_code>sea_merged.loc[sea_merged['Cluster Labels'] == 4, sea_merged.columns[[0] + [1] + list(range(5, sea_merged.shape[1]))]]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12518790713211958, 'fraction_numerical': 0.02672456990145315, 'mean_word_length': 4.139484978540772, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 51, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '470815', 'license': 'MIT', 'n_tokens_mistral': 4388, 'n_tokens_neox': 4140, 'n_words': 896, 'path': '.ipynb_checkpoints/Main-checkpoint.ipynb', 'repo_name': 'naquech/CourseraCapstone', 'size': '12056'}