<jupyter_start><jupyter_text>Seattle Capstone Project Load libraries<jupyter_code>!pip install -U numpy

!pip install -U pandas

!pip install -U scipy

!pip install -U scikit-learn

!pip install -U imbalanced-learn
import numpy as np # library to handle data in a vectorized manner

import pandas as pd # library for data analsysis
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

import json # library to handle JSON files

!conda install -c conda-forge geopy --yes # uncomment this line if you haven't completed the Foursquare API lab
from geopy.geocoders import Nominatim # convert an address into latitude and longitude values

import requests # library to handle requests
from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe

# Matplotlib and associated plotting modules
import matplotlib.cm as cm
import matplotlib.colors as colors

print('Libraries imported.')<jupyter_output>Collecting package metadata (current_repodata.json): done
Solving environment: done

# All requested packages already installed.

Libraries imported.<jupyter_text>Download Dataset<jupyter_code>#neighborhoods = pd.read_csv("seattle_district_trial.csv")
#neighborhoods.head()<jupyter_output><empty_output><jupyter_text>import geo coordinatesimport os, typesimport pandas as pdfrom botocore.client import Configimport ibm_boto3def __iter__(self): return 0 @hidden_cell The following code accesses a file in your IBM Cloud Object Storage. It includes your credentials. You might want to remove those credentials before you share the notebook.if os.environ.get('RUNTIME_ENV_LOCATION_TYPE') == 'external':    endpoint_7593f4a6aa6442c19b792a4c5828a495 = 'https://s3-api.us-geo.objectstorage.softlayer.net'else:    endpoint_7593f4a6aa6442c19b792a4c5828a495 = 'https://s3-api.us-geo.objectstorage.service.networklayer.com'client_7593f4a6aa6442c19b792a4c5828a495 = ibm_boto3.client(service_name='s3',    ibm_api_key_id='<KEY>',    ibm_auth_endpoint="https://iam.cloud.ibm.com/oidc/token",    config=Config(signature_version='oauth'),    endpoint_url=endpoint_7593f4a6aa6442c19b792a4c5828a495)body = client_7593f4a6aa6442c19b792a4c5828a495.get_object(Bucket='week3torontodataclustering-donotdelete-pr-zh1gdodnliwldw',Key='seattle_district_trial.csv')['Body'] add missing __iter__ method, so pandas accepts body as file-like objectif not hasattr(body, "__iter__"): body.__iter__ = types.MethodType( __iter__, body )df_data_1 = pd.read_csv(body)df_data_1.head()<jupyter_code>import numpy as np # library to handle data in a vectorized manner

import pandas as pd # library for data analsysis
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

import json # library to handle JSON files

!conda install -c conda-forge geopy --yes # uncomment this line if you haven't completed the Foursquare API lab
from geopy.geocoders import Nominatim # convert an address into latitude and longitude values

import requests # library to handle requests
from pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe

# Matplotlib and associated plotting modules
import matplotlib.cm as cm
import matplotlib.colors as colors

print('Libraries imported.')
#Download datasets (from wikipedia) and also the geo coordinates csv file. 
url= 'https://docs.google.com/spreadsheets/d/1vO_mPBbxWIt0cnhicr_SkKgSLA_I1qdCKvQyNV87A-c/edit#gid=0'
neighborhoods = pd.read_html(url, header=1)[0]

#geo = pd.read_csv("Geospatial_Coordinates.csv")

neighborhoods.head()
del neighborhoods['1']
neighborhoods.head()
#get the geo coordinates of Seattle, WA
address = 'Seattle, WA'

geolocator = Nominatim(user_agent="wa_explorer")
location = geolocator.geocode(address)
latitude = location.latitude
longitude = location.longitude
print('The geograpical coordinates of Seattle are {}, {}.'.format(latitude, longitude))
#pause here 2:!5
#!conda install -c conda-forge folium=0.5.0 --yes # uncomment this line if you haven't completed the Foursquare API lab
!pip install folium # map rendering library
#!pip install beautifulsoup4 


!pip install bs4
!pip install lxml
!pip install html5lib
import folium 
# create map of Seattle using latitude and longitude values
map_seattle = folium.Map(location=[latitude, longitude], zoom_start=10)

# add markers to map
for lat, lng, neighborhood in zip(neighborhoods['latitude'], neighborhoods['longitude'], neighborhoods['seattle_district']):
    label = '{}'.format(neighborhood)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='blue',
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.7,
        parse_html=False).add_to(map_seattle)  
    
map_seattle
# @hidden_cell
CLIENT_ID = 'R2XCHP4ZWLG1E4EGN4R24V0PME3HGXGUPU3JEF2JW1HXDCAM' # your Foursquare ID
CLIENT_SECRET = '<KEY>' # your Foursquare Secret
VERSION = '20180605' # Foursquare API version
LIMIT = 100 # A default Foursquare API limit value

#print('Your credentails:')
#print('CLIENT_ID: ' + CLIENT_ID)
#print('CLIENT_SECRET:' + CLIENT_SECRET)
atlantic_data = neighborhoods[neighborhoods['seattle_district'] == 'Atlantic'].reset_index(drop=True)
atlantic_data.head()

neighborhood_latitude = atlantic_data.loc[0, 'latitude'] # neighborhood latitude value
neighborhood_longitude = atlantic_data.loc[0, 'longitude'] # neighborhood longitude value

neighborhood_name = atlantic_data.loc[0, 'seattle_district'] # neighborhood name

print('Latitude and longitude values of {} are {}, {}.'.format(neighborhood_name, 
                                                               neighborhood_latitude, 
                                                               neighborhood_longitude))
LIMIT = 100 # limit of number of venues returned by Foursquare API

radius = 500 #define radius

# create URL
url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
    CLIENT_ID, 
    CLIENT_SECRET, 
    VERSION, 
    neighborhood_latitude, 
    neighborhood_longitude, 
    radius, 
    LIMIT)

url # display URL
results = requests.get(url).json()
#results
# function that extracts the category of the venue
def get_category_type(row):
    try:
        categories_list = row['categories']
    except:
        categories_list = row['venue.categories']
        
    if len(categories_list) == 0:
        return None
    else:
        return categories_list[0]['name']
venues = results['response']['groups'][0]['items']
    
nearby_venues = json_normalize(venues) # flatten JSON

# filter columns
filtered_columns = ['venue.name', 'venue.categories', 'venue.location.lat', 'venue.location.lng']
nearby_venues =nearby_venues.loc[:, filtered_columns] #throws an error

# filter the category for each row
nearby_venues['venue.categories'] = nearby_venues.apply(get_category_type, axis=1)

# clean columns
nearby_venues.columns = [col.split(".")[-1] for col in nearby_venues.columns]

nearby_venues.head()
print('{} venues were returned by Foursquare.'.format(nearby_venues.shape[0]))<jupyter_output>37 venues were returned by Foursquare.<jupyter_text>Explore Districts in Seattle<jupyter_code>def getNearbyVenues(names, latitudes, longitudes, radius=500):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
        print(name)
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Neighborhood', 
                  'Neighborhood Latitude', 
                  'Neighborhood Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)
neighborhoods_venues = getNearbyVenues(names=neighborhoods['seattle_district'],
                                   latitudes=neighborhoods['latitude'],
                                   longitudes=neighborhoods['longitude']
                                  )

print(neighborhoods_venues.shape)
neighborhoods_venues.head()
neighborhoods_venues.groupby('Neighborhood').count()
print('There are {} uniques categories.'.format(len(neighborhoods_venues['Venue Category'].unique())))<jupyter_output>There are 215 uniques categories.<jupyter_text>Analyze each neighborhood<jupyter_code># one hot encoding
seattle_onehot = pd.get_dummies(neighborhoods_venues[['Venue Category']], prefix="", prefix_sep="")

# add neighborhood column back to dataframe
seattle_onehot['Neighborhood'] = neighborhoods_venues['Neighborhood'] 

# move neighborhood column to the first column
fixed_columns = [seattle_onehot.columns[-1]] + list(seattle_onehot.columns[:-1])
seattle_onehot = seattle_onehot[fixed_columns]

seattle_onehot.head()
#Examine the new dataframe size.

seattle_onehot.shape
#Next, let's group rows by neighborhood and by taking the mean of the frequency of occurrence of each category
seattle_grouped = seattle_onehot.groupby('Neighborhood').mean().reset_index()
seattle_grouped
#size of dataset
seattle_grouped.shape<jupyter_output><empty_output><jupyter_text>What are the top 5 commonn venues in each neighborhood?<jupyter_code>num_top_venues = 5

for hood in seattle_grouped['Neighborhood']:
    print("----"+hood+"----")
    temp = seattle_grouped[seattle_grouped['Neighborhood'] == hood].T.reset_index()
    temp.columns = ['venue','freq']
    temp = temp.iloc[1:]
    temp['freq'] = temp['freq'].astype(float)
    temp = temp.round({'freq': 2})
    print(temp.sort_values('freq', ascending=False).reset_index(drop=True).head(num_top_venues))
    print('\n')
# Stick that into a pandas dataframe
## First, let's write a function to sort the venues in descending order.

def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]
#Make new datafram and display the top 10 venues for each neighborhood.
num_top_venues = 10

indicators = ['st', 'nd', 'rd']

# create columns according to number of top venues
columns = ['Neighborhood']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))

# create a new dataframe
neighborhoods_venues_sorted = pd.DataFrame(columns=columns)
neighborhoods_venues_sorted['Neighborhood'] = seattle_grouped['Neighborhood']

for ind in np.arange(seattle_grouped.shape[0]):
    neighborhoods_venues_sorted.iloc[ind, 1:] = return_most_common_venues(seattle_grouped.iloc[ind, :], num_top_venues)

neighborhoods_venues_sorted.head()
neighborhoods_venues_sorted
num_venues = neighborhoods_venues.groupby('Neighborhood').count()

#Drop neighborhoods with less than 10 venues

df_filtered = num_venues[num_venues['Venue'] >= 10]
df_filtered
!pip install sklearn
from sklearn.cluster import KMeans<jupyter_output><empty_output><jupyter_text>Cluster NeighborhoodsRun K-means to cluster neighborhood into 5 clusters<jupyter_code># set number of clusters
kclusters = 5

seattle_grouped_clustering = seattle_grouped.drop('Neighborhood', 1)

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(seattle_grouped_clustering)

# check cluster labels generated for each row in the dataframe
kmeans.labels_[0:10]<jupyter_output>/opt/conda/envs/Python-3.7-main/lib/python3.7/site-packages/ipykernel/__main__.py:4: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only<jupyter_text>Let's create a new dataframe that includes the cluster as well as the top 10 venues for each neighborhood.<jupyter_code>#Clustering labels can only be run once or else will throw an error "already made" basically

# add clustering labels
neighborhoods_venues_sorted.insert(0, 'Cluster Labels', kmeans.labels_)



seattle_merged = neighborhood

# merge manhattan_grouped with manhattan_data to add latitude/longitude for each neighborhood
seattle_merged = seattle_merged.join(neighborhoods_venues_sorted.set_index('Neighborhood'), on='Neighbourhood')

seattle_merged.head() # check the last columns!<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10344827586206896, 'fraction_numerical': 0.017964527669939865, 'mean_word_length': 4.538785834738618, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 32, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '311050', 'license': 'MIT', 'n_tokens_mistral': 4379, 'n_tokens_neox': 4073, 'n_words': 1003, 'path': 'Seattle_Capstone_Project.ipynb', 'repo_name': 'aperinchery/CAPSTONE_IBM_certificate', 'size': '13219'}