<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import pandas as pd
import requests
from pandas.io.json import json_normalize
import numpy as np
from sklearn.cluster import DBSCAN 
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Gather initial data - Boston Districts  Foursquare Credentials<jupyter_code>labelResteraunt = "Mexican Restaurant"
LIMIT = 100
CLIENT_ID = 'JFYULM12BC04XJ0S2QSZ5UZCLJPRYHJCNDSFTHSNFXE4ZGP2' # your Foursquare ID
CLIENT_SECRET = '<KEY>' # your Foursquare Secret
VERSION = '20180604'
print('Your credentails:')
print('CLIENT_ID: ' + CLIENT_ID)
print('CLIENT_SECRET:' + CLIENT_SECRET)<jupyter_output>Your credentails:
CLIENT_ID: JFYULM12BC04XJ0S2QSZ5UZCLJPRYHJCNDSFTHSNFXE4ZGP2
CLIENT_SECRET:IEF3IIMXP0VWRL13B4VP4OS2MB1JBOZLCNPT3Q2ZMNGZ0ZOT<jupyter_text>Load in Boston District Data<jupyter_code>#Loads data from a csv file and puts it into a table using pandas
boston_nh = pd.read_csv('bostonneighborhoods.csv')
boston_nh.head()
#Loads data from a csv file and puts it into a table using pandas
#boston_pop = pd.read_csv('bostonpop.csv')
#boston_pop.head()<jupyter_output><empty_output><jupyter_text>Combine the two datasets together<jupyter_code>#boston_combined = boston_nh.join(boston_pop.set_index('Neighborhood'), on='Neighborhood')
#boston_combined.head()<jupyter_output><empty_output><jupyter_text>List of Resteraunt types<jupyter_code># Url : https://developer.foursquare.com/docs/build-with-foursquare/categories

food_venues = {
'foursquareID_food'     : "4d4b7105d754a06374d81259",
'foursquareID_african'  : "4bf58dd8d48988d1c8941735",
'foursquareID_american' : "4bf58dd8d48988d14e941735",
'foursquareID_asian'    : "4bf58dd8d48988d142941735",
'foursquareID_chinese'  : "4bf58dd8d48988d145941735",
'foursquareID_japanese' : "4bf58dd8d48988d111941735",
'foursquareID_korean'   : "4bf58dd8d48988d113941735",
'foursquareID_thai'     : "4bf58dd8d48988d149941735",
'foursquareID_bbq'      : "4bf58dd8d48988d1df931735",
'foursquareID_buffet'   : '52e81612bcbc57f1066b79f4',
'foursquareID_burger'   : '4bf58dd8d48988d16c941735',
'foursquareID_fish'     : '4edd64a0c7ddd24ca188df1a',
'foursquareID_french'   : '4bf58dd8d48988d10c941735',
'foursquareID_german'   : '4bf58dd8d48988d10d941735',
'foursquareID_greek'    : '4bf58dd8d48988d10e941735',
'foursquareID_indian'   : '4bf58dd8d48988d10f941735',
'foursquareID_italian'  : '4bf58dd8d48988d110941735',
'foursquareID_latin'    : '4bf58dd8d48988d1be941735',
'foursquareID_mexian'   : '4bf58dd8d48988d1c1941735',
'foursquareID_med'      : '4bf58dd8d48988d1c0941735',
'foursquareID_middleE'  : '4bf58dd8d48988d115941735',
'foursquareID_pizza'    : '4bf58dd8d48988d1ca941735',
'foursquareID_southern' : '4bf58dd8d48988d14f941735',
'foursquareID_spanish'  : '4bf58dd8d48988d150941735',
'foursquareID_steak'    : '4bf58dd8d48988d1cc941735',
'foursquareID_turkish'  : '4f04af1f2fb6e1c99f3db0bb'
}

venue_ids = (list(food_venues.values()))
#venue_ids = ["4d4b7105d754a06374d81259",'4bf58dd8d48988d142941735']<jupyter_output><empty_output><jupyter_text>Pull data from Foursquare<jupyter_code>def getVenues(names, latitudes, longitudes, radius, category):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
        
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}&categoryId={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT,
            category)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    boston_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    boston_venues.columns = ['Neighborhood', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(boston_venues)
venueInfo = []
LIMIT = 100
radius = 1000
for cID in venue_ids:
    print(cID)
    try:
        boston_venues = getVenues(names=boston_nh['Neighborhood'],
                                           latitudes=boston_nh['Latitude'],
                                           longitudes=boston_nh['Longitude'],
                                           radius = radius,
                                           category = cID
                                          )
        venueInfo.append(boston_venues)
    except: pass<jupyter_output>4d4b7105d754a06374d81259
4bf58dd8d48988d1c8941735
4bf58dd8d48988d14e941735
4bf58dd8d48988d142941735
4bf58dd8d48988d145941735
4bf58dd8d48988d111941735
4bf58dd8d48988d113941735
4bf58dd8d48988d149941735
4bf58dd8d48988d1df931735
52e81612bcbc57f1066b79f4
4bf58dd8d48988d16c941735
4edd64a0c7ddd24ca188df1a
4bf58dd8d48988d10c941735
4bf58dd8d48988d10d941735
4bf58dd8d48988d10e941735
4bf58dd8d48988d10f941735
4bf58dd8d48988d110941735
4bf58dd8d48988d1be941735
4bf58dd8d48988d1c1941735
4bf58dd8d48988d1c0941735
4bf58dd8d48988d115941735
4bf58dd8d48988d1ca941735
4bf58dd8d48988d14f941735
4bf58dd8d48988d150941735
4bf58dd8d48988d1cc941735
4f04af1f2fb6e1c99f3db0bb<jupyter_text>Combine the total datasets into one Frame, remove non-unique's<jupyter_code>header = ['Neighborhood','Venue','Venue Latitude','Venue Longitude','Venue Category']
totalData = pd.DataFrame(columns=header)
#-----------------------------------------------
for k in venueInfo:
    totalData = pd.concat([k, totalData], ignore_index=True,sort=True)

totalData = pd.DataFrame.drop_duplicates(totalData)
totalData.reset_index(drop=True, inplace=True)

len(totalData)<jupyter_output><empty_output><jupyter_text>Setup the X values used for Clustering<jupyter_code>clusterList,longList,latList = [], [], []
labels = []
#clusterArray = np.array([])

for row in range(len(totalData)):
    long = totalData.at[row, "Venue Longitude"]
    lat  = totalData.at[row, "Venue Latitude"]
    #---------------------------------
    labels.append(totalData.at[row, "Venue Category"])
    clusterList.append( (long,lat) )
    longList.append(long) # Used for making maps later
    latList.append(lat) # Used for making maps later<jupyter_output><empty_output><jupyter_text>Graph Current Locations<jupyter_code>#list1 = arr.tolist()
plt.scatter(longList, latList)<jupyter_output><empty_output><jupyter_text>DBScan  Now we run the code through DB Scan to make the groups<jupyter_code># Epsilon dictates how tight the density groups can be.
# Min samples says how few data points are needed for a cluster to form
# Cluster list is a numpy array containing the X and Y coordinates of every Venue
dbScan = DBSCAN(eps=0.001, min_samples=5).fit_predict(clusterList) 
plt.scatter(longList, latList, c=dbScan)<jupyter_output><empty_output><jupyter_text>Now that we have the clusters, we can figure out how many venue's are in each<jupyter_code># Lets figure out how many clusters have been made
clusterResultList = []
for k in dbScan:
    clusterResultList.append(str(k))

clusterList = list(set(clusterResultList))
if "-1" in clusterList: clusterList.remove("-1")

print(len(clusterList))<jupyter_output>94<jupyter_text>Lets get the amount of resteraunts per district<jupyter_code>numberOfResterauntsPerCluster = []
for c in clusterList:
    counter = 0
    for k in clusterResultList:
        if k == c:
            counter+=1
    numberOfResterauntsPerCluster.append(counter)
#numberOfResterauntsPerCluster
numberOfMexicanPerCluster = []

for c in clusterList:
    counter = 0
    for idx in range(len(clusterResultList)):
        if c == clusterResultList[idx]: # This means that the district and catagory algin
            if labels[idx] == labelResteraunt:
                counter+=1
    numberOfMexicanPerCluster.append(counter)
#numberOfMexicanPerCluster<jupyter_output><empty_output><jupyter_text>Now, we generate a table to show value of each cluster<jupyter_code>totalVenues = len(clusterResultList)
finalResults = []

for idx in range(len(clusterList)):
    weight = numberOfResterauntsPerCluster[idx]/totalVenues
    percentX = numberOfMexicanPerCluster[idx]/numberOfResterauntsPerCluster[idx]
    value = weight*percentX    
    if percentX !=0:
        finalResults.append( (clusterList[idx],value) )
        
finalResults<jupyter_output><empty_output><jupyter_text>Find the largest sector<jupyter_code>maxValue, maxName = 0.000001, ''
for k in finalResults:
    print(k[1])
    if k[1] > maxValue:
        maxValue = k[1]
        maxName  = k[0]
        
print("Max id is : " + maxName)<jupyter_output>0.0005058168942842691
0.00025290844714213456
0.0007587253414264037
0.00025290844714213456
0.00025290844714213456
0.0007587253414264037
0.0005058168942842691
0.00025290844714213456
0.00025290844714213456
0.0015174506828528073
0.002529084471421346
0.019473950429944358
0.0005058168942842691
0.0005058168942842691
0.0010116337885685382
0.0005058168942842691
0.00025290844714213456
0.00025290844714213456
0.0032878098128477492
0.00025290844714213456
0.0005058168942842691
0.0010116337885685382
0.0005058168942842691
0.0030349013657056142
0.0005058168942842691
0.00025290844714213456
0.0005058168942842691
Max id is : 1<jupyter_text>Find the X/Y coordinates<jupyter_code>counter, longCounter, latCounter = 0, 0.0, 0.0


for idx in range(len(clusterResultList)):
    if clusterResultList[idx] == maxName:
        counter+=1
        longCounter = longCounter + longList[idx]
        latCounter  = latCounter  + latList[idx]
        
print(f"The Area that could use additional {labelResteraunt} is around '{longCounter/counter}' Longitude and '{latCounter/counter}' Latitude in Boston")<jupyter_output>The Area that could use additional Mexican Restaurant is around '-71.0588586089431' Longitude and '42.35557291694653' Latitude in Boston
{'fraction_non_alphanumeric': 0.10155023547880691, 'fraction_numerical': 0.15384615384615385, 'mean_word_length': 4.539673913043479, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 58, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '36017', 'license': 'MIT', 'n_tokens_mistral': 4650, 'n_tokens_neox': 3809, 'n_words': 632, 'path': 'CorseraFinalProject.ipynb', 'repo_name': 'aevear/Coursera-Final-Project', 'size': '10235'}