<jupyter_start><jupyter_text>Capstone Notebook IBM Data Science Capstone Project on Coursera  Import/Install Required Libraries<jupyter_code>import pandas as pd
pd.set_option('max_rows', 200)
import numpy as np
! pip install geocoder
import geocoder as gc
import requests
from sklearn.cluster import KMeans
from geopy.geocoders import Nominatim
!conda install -c conda-forge folium=0.5.0
import folium
import random
import matplotlib.cm as cm
import matplotlib.colors as colors
print('--Imports Completed--')<jupyter_output>Collecting geocoder
[?25l  Downloading https://files.pythonhosted.org/packages/4f/6b/13166c909ad2f2d76b929a4227c952630ebaf0d729f6317eb09cbceccbab/geocoder-1.38.1-py2.py3-none-any.whl (98kB)
[K    100% |████████████████████████████████| 102kB 8.6MB/s 
[?25hRequirement already satisfied: click in /home/jupyterlab/conda/lib/python3.6/site-packages (from geocoder) (7.0)
Requirement already satisfied: six in /home/jupyterlab/conda/lib/python3.6/site-packages (from geocoder) (1.12.0)
Requirement already satisfied: requests in /home/jupyterlab/conda/lib/python3.6/site-packages (from geocoder) (2.21.0)
Collecting ratelim (from geocoder)
  Downloading https://files.pythonhosted.org/packages/f2/98/7e6d147fd16a10a5f821db6e25f192265d6ecca3d82957a4fdd592cad49c/ratelim-0.1.6-py2.py3-none-any.whl
Requirement already satisfied: future in /home/jupyterlab/conda/lib/python3.6/site-packages (from geocoder) (0.17.1)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/jupyterlab/conda/lib/pyth[...]<jupyter_text>Los Angeles County Zipcodes  Retrieve/Format Table of CA Zip Codes<jupyter_code>#Download & Format Available California Zipcodes
url = 'https://www.geonames.org/postal-codes/US/CA/california.html'
laData = pd.read_html(url)
laData = laData[2]  #Select Correct Table from url
laData
laData = laData.drop(laData.columns[0],axis=1) 
laData = laData.drop(laData.columns[5:len(laData.columns)],axis=1).reset_index(drop=True) #Drop extraneous columns
#Rename Columns
cols = list(laData.columns)
cols[3] = 'State'
cols[4] = 'County'
laData.columns = cols
laData = laData.drop(len(laData.index)-1).reset_index(drop=True) #Drop extraneous row (last row)
#extract coordinates fromm odd rows 
#reinsert as columns 
lat = [] 
long = [] 
for count, n in enumerate(laData['Place']):
    if (count % 2 != 0):
        temp = n.split('/')
        lat.append(temp[0])
        long.append(temp[1])
        laData = laData.drop(count)
laData = laData.reset_index(drop=True)

#add Latitude and Logitude to LA Dataframe
data = {'Latitude':lat,'Longitude':long}
temp = pd.DataFrame(data,dtype='float')
laData = laData.join(temp)
laData = laData[laData['County'].str.contains('Los Angeles')] #Select only zip codes in Los Angeles<jupyter_output><empty_output><jupyter_text>Remove Zipcodes with Duplicate Coordinates<jupyter_code>laData = laData.sort_values(by = ['Latitude','Longitude']).reset_index(drop=True)

for (row) in range(len(laData.index)-1):
    if laData.loc[row,'Latitude'] == laData.loc[row+1,'Latitude']:
        if laData.loc[row,'Longitude'] == laData.loc[row+1,'Longitude']:
            laData = laData.drop(row)

laData = laData.sort_values('Code').reset_index(drop=True) #resort by zipcode<jupyter_output><empty_output><jupyter_text>Complete LA County Zip Codes<jupyter_code>#Display Corrected Dataframe & its shape
print('Rows:',laData.shape[0],'\nColumns:',laData.shape[1])
laData<jupyter_output>Rows: 123 
Columns: 7<jupyter_text>Map of Zip Code Locations<jupyter_code>#Get Coordinates for Map
address = 'Los Angeles, CA'
geolocator = Nominatim(user_agent="LA_agent")
location = geolocator.geocode(address)
latitude = location.latitude
longitude = location.longitude
print(address,'\nLatitude:',latitude,'\nLongitude:',longitude)
#Create Map
laMap = folium.Map(location=[latitude, longitude],tiles = 'Stamen Toner', zoom_start=10)

#add Zipcode labels to map
for lat, lng, code, place in zip(laData['Latitude'], laData['Longitude'], laData['Code'], laData['Place']):
    label = '{}: {}'.format(place, code)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='blue',
        fill=True,
        fill_color='blue',
        fill_opacity=0.5,
        parse_html=False).add_to(laMap)

#display map
display(laMap)<jupyter_output><empty_output><jupyter_text>Retrieve Venues via Foursquare API<jupyter_code>#Define Foursquare Credentials
CLIENT_ID = 'Insert Your Foursquare ID' #Foursquare ID
CLIENT_SECRET = 'Insert Your Foursquare Secret' #Foursquare Secret
VERSION = '20180605'
#Define Foursquare Credentials
CLIENT_ID = 'V5GTPEARUI4LNNVRL2Z2CBTSKQX32TZB14ZUP4KKDB0DS01L' #Foursquare ID
CLIENT_SECRET = '<KEY>' #Foursquare Secret
VERSION = '20180605'
#Function to Create Dataframe of Venues from Foursquare API
def getVenues(code, latitudes, longitudes, radius=500):
    
    venues_list=[]
    dropList = []  
    for code, lat, lng in zip(code, latitudes, longitudes):
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            500)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        if results == []:
            print('Warning no venues returned for: ',code)
            dropList.append(code)
        # return only relevant information for each nearby venue
        venues_list.append([(
            code, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Zipcode', 
                  'Zipcode Latitude', 
                  'Zipcode Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']  
    return(nearby_venues, dropList)
laVenues, laDrop = getVenues(laData['Code'],laData['Latitude'],laData['Longitude'])
#Display Venues
laVenues.head()<jupyter_output><empty_output><jupyter_text>K-Clusters Analysis<jupyter_code>#Create Dataframe for kClusters Analysis
laEncode = pd.get_dummies(laVenues[['Venue Category']], prefix="", prefix_sep="")
laEncode['Zipcode'] = laVenues['Zipcode']
fixed_columns = [laEncode.columns[-1]] + list(laEncode.columns[:-1])
laEncode = laEncode[fixed_columns]
laSorted = laEncode.groupby(by='Zipcode').mean().reset_index()
laCluster = laSorted.drop('Zipcode', axis=1)<jupyter_output><empty_output><jupyter_text>Average Venue Category by Zip Code<jupyter_code>#Show mean values of category types sorted by zipcode
laSorted.head(10)<jupyter_output><empty_output><jupyter_text>Top 5 Venue Categories for each Zipcode<jupyter_code>#Create DataFrame of Top 5 Venue Categories for each Zipcode
topVen = pd.DataFrame()
#Get top 5 venues for each Zipcode
for count, z in enumerate(laSorted['Zipcode']):
    temp = laSorted[laSorted['Zipcode'] == z]  #Select all of a single zipcode
    temp = temp.drop('Zipcode',axis=1)
    temp.reset_index()
    temp = temp.sort_values(by=count,axis=1,ascending=False)
    topVen = topVen.append(pd.Series(temp.iloc[:,0:5].columns),ignore_index = True)
    
#Rename Columns & Display first 5 rows
topVen.columns = ['#1 Venue','#2 Venue','#3 Venue','#4 Venue','#5 Venue'] 
print(topVen.shape)
topVen.head()<jupyter_output>(121, 5)<jupyter_text>Create k Clusters from Category Data<jupyter_code>#Fit Data
k = 8
kmeans = KMeans(k, random_state=0).fit(laCluster)
#add Cluster Labels to Top Venues Dataframe
topVen.insert(0,'Cluster Labels',kmeans.labels_ + 1)<jupyter_output><empty_output><jupyter_text>Create Final Dataframe: Zipcode, Location Name, Coordinates, k Cluster Group, Top 5 Venues<jupyter_code># Join Zipcodes & Coordinates with Cluster Labels and Top Venues
laFinal = pd.DataFrame()
laFinal = laData.loc[:,['Code','Place','Latitude','Longitude']].sort_values(by='Code').reset_index(drop=True)
for x in laDrop:
    if laFinal['Code'].str.contains(x).any():
        laFinal = laFinal.drop(laFinal.index[laFinal['Code']==x])
laFinal = laFinal.reset_index(drop=True)
laFinal = laFinal.join(topVen)
laFinal = laFinal.sort_values('Cluster Labels').reset_index(drop=True)
laFinal<jupyter_output><empty_output><jupyter_text>Create Map of Clusters<jupyter_code>##### initialize map
clusterMap = folium.Map(location=[latitude, longitude],tiles='Stamen Toner', zoom_start=11)

# set color scheme for the clusters
x = np.arange(k)
ys = [i + x + (i*x)**4 for i in range(k)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]
random.shuffle(rainbow)
# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(laFinal['Latitude'], laFinal['Longitude'], laFinal['Code'], laFinal['Cluster Labels']):
    label = folium.Popup(str(poi) + ': Cluster ' + str(cluster), parse_html=True)
    folium.Circle(
        [lat, lon],
        radius=500,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.75).add_to(clusterMap)
print('K    Count\n-----------')
print(laFinal['Cluster Labels'].value_counts(sort=False))
display(clusterMap)<jupyter_output>K    Count
-----------
1    10
2     2
3     1
4     1
5    90
6     1
7     3
8    13
Name: Cluster Labels, dtype: int64
{'fraction_non_alphanumeric': 0.13875499948723208, 'fraction_numerical': 0.02656137832017229, 'mean_word_length': 4.451089994410285, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '205089', 'license': 'Unlicense', 'n_tokens_mistral': 3599, 'n_tokens_neox': 3389, 'n_words': 705, 'path': 'CapstoneNotebook.ipynb', 'repo_name': 'YTyler/CourseraCapstoneFinal', 'size': '9794'}