<jupyter_start><jupyter_text>Battle of the Neighborhoods  Introduction This project is for who is planning to open a Coffee House in Ankara, Turkey. This project suggests the best locations for Coffee Houses in Ankara. Ankara is the capital of Turkey with a population of 5M. Turkish culture has deep roots in coffee. Also, it is getting popular because of its affordable prices. Coffee consumption has increased 13% and average consumption is 1.1 kg/person in 2018. This report explores which neighborhoods of Ankara have the most as well as the best Coffee Houses. Also, this project answers the questions “Where should I open an Coffee House?” and “Where should I stay If I want a tasty coffee?”  Data District of Ankara are obtained from https://en.wikipedia.org/wiki/Ankara_ProvinceLatitude and Longitude values are obtained by using "geocoder".All data related to locations will be obtaine by using FourSquare API and Python Libraries.<jupyter_code>import requests
import pandas as pd
from bs4 import BeautifulSoup
#!conda install -c conda-forge geopy --yes
import geocoder
wiki_link = 'https://en.wikipedia.org/wiki/Ankara_Province'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0'}
wiki_page = requests.get(wiki_link, headers = headers)
wiki_page
soup = BeautifulSoup(wiki_page.content, 'html.parser')
table = soup.find('table', {'class':'wikitable sortable'}).tbody
rows = table.find_all('tr')
columns = [i.text.replace('\n', '') for i in rows[0].find_all('th')]
columns
df_ankara = pd.DataFrame(columns = columns)
for i in range(1, len(rows)):
    tds = rows[i].find_all('td')
    
    if len(tds) == 7:
        values = [tds[0].text, tds[1].text, tds[2].text.replace('\n', ''.replace('\xa0','')), tds[3].text, tds[4].text.replace('\n', ''.replace('\xa0','')), tds[5].text.replace('\n', ''.replace('\xa0','')), tds[6].text.replace('\n', ''.replace('\xa0',''))]
    else:
        values = [td.text.replace('\n', '').replace('\xa0','') for td in tds]
        
        df_ankara = df_ankara.append(pd.Series(values, index = columns), ignore_index = True)

        df_ankara
df_ankara.head()
def get_latlng(arcgis_geocoder):
    
    lat_lng_coords = None
    
    while(lat_lng_coords is None):
        g = geocoder.arcgis('{}, Ankara, Turkey'.format(arcgis_geocoder))
        lat_lng_coords = g.latlng
    return lat_lng_coords
districts = df_ankara['District']    
coordinates = [get_latlng(districts) for districts in districts.tolist()]
df_ankara_loc = df_ankara

df_ankara_coordinates = pd.DataFrame(coordinates, columns = ['Latitude', 'Longitude'])
df_ankara_loc['Latitude'] = df_ankara_coordinates['Latitude']
df_ankara_loc['Longitude'] = df_ankara_coordinates['Longitude']
df_ankara_loc.drop(columns="Population (2017)", axis=1, inplace=True)
df_ankara_loc.drop(columns="Density (per km²)", axis=1, inplace=True)
df_ankara_loc.drop(columns="Area (km²)", axis=1, inplace=True)
df_ankara_loc.head()
import numpy as np
import json 
from geopy.geocoders import Nominatim 

import requests 
from pandas.io.json import json_normalize

import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib.colors as colors

from sklearn.cluster import KMeans

#!conda install -c conda-forge folium=0.5.0 --yes 
import folium

print("Libraries imported")
from geopy.geocoders import Nominatim 

address = "Çankaya, Ankara"

geolocator = Nominatim(user_agent = "Ankara_explorer")

location = geolocator.geocode(address)
latitude = location.latitude
longitude = location.longitude

print("The geographical coordinates of Ankara are {}, {}.".format(latitude, longitude))
map_ankara = folium.Map(location=[latitude, longitude], zoom_start=10)

for lat, lng, label in zip(df_ankara_loc["Latitude"], df_ankara_loc["Longitude"], df_ankara_loc["District"]):
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=25,
        popup=label,
        color='blue',
        fill=True,
        fill_color='#3186cc',
        fill_opacity=0.3,
        parse_html=False).add_to(map_ankara)  
    
map_ankara
CLIENT_ID = "FBMIJHTR42BI0I5JUWFNREL3NTHT553IQUQ3AM1NGMQDDSHJ"
CLIENT_SECRET = "<KEY>" 
VERSION = "20180605"
df_ankara_loc.loc [0, "District"]
df_ankara_loc.loc [0, "District"]
neighborhood_latitude = df_ankara_loc.loc[0, "Latitude"]
neighborhood_longitude = df_ankara_loc.loc[0, "Longitude"] 

neighborhood_name = df_ankara_loc.loc[0, "District"] 

print("Latitude and longitude values of the neighborhood {} are {}, {}.".format(neighborhood_name, neighborhood_latitude, neighborhood_longitude))
LIMIT = 100
radius = 500

url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
    CLIENT_ID, 
    CLIENT_SECRET, 
    VERSION, 
    neighborhood_latitude, 
    neighborhood_longitude, 
    radius, 
    LIMIT)
url
results = requests.get(url).json()
results
def get_category_type(row):
    try:
        categories_list = row['categories']
    except:
        categories_list = row['venue.categories']
        
    if len(categories_list) == 0:
        return None
    else:
        return categories_list[0]['name']
venues = results['response']['groups'][0]['items']
    
nearby_venues = json_normalize(venues) # flatten JSON


filtered_columns = ['venue.name', 'venue.categories', 'venue.location.lat', 'venue.location.lng']
nearby_venues =nearby_venues.loc[:, filtered_columns]


nearby_venues['venue.categories'] = nearby_venues.apply(get_category_type, axis=1)


nearby_venues.columns = [col.split(".")[-1] for col in nearby_venues.columns]

nearby_venues.head(20)
print('{} venues were returned by Foursquare.'.format(nearby_venues.shape[0]))
def getNearbyVenues(names, latitudes, longitudes, radius=1500):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
        print(name)
            
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['District', 
                  'Latitude', 
                  'Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)
ankara_venues = getNearbyVenues(names=df_ankara_loc['District'],
                                   latitudes=df_ankara_loc['Latitude'],
                                   longitudes=df_ankara_loc['Longitude']
                                  )
print(ankara_venues.shape)
ankara_venues.head(250)
ankara_venues.groupby("District").count()
print('There are {} unique venue categories.'.format(len(ankara_venues['Venue Category'].unique())))
ankara_onehot = pd.get_dummies(ankara_venues[['Venue Category']], prefix="", prefix_sep="")

ankara_onehot['Neighborhood'] = ankara_venues['District'] 

fixed_columns = [ankara_onehot.columns[-1]] + list(ankara_onehot.columns[:-1])
ankara_onehot = ankara_onehot[fixed_columns]

ankara_onehot
ankara_onehot.shape
ankara_grouped = ankara_onehot.groupby('Neighborhood').mean().reset_index()
ankara_grouped
ankara_grouped.shape
num_top_venues = 10

for hood in ankara_grouped['Neighborhood']:
    print("----"+hood+"----")
    temp = ankara_grouped[ankara_grouped['Neighborhood'] == hood].T.reset_index()
    temp.columns = ['venue','freq']
    temp = temp.iloc[1:]
    temp['freq'] = temp['freq'].astype(float)
    temp = temp.round({'freq': 2})
    print(temp.sort_values('freq', ascending=False).reset_index(drop=True).head(num_top_venues))
    print('\n')
def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]
num_top_venues = 10
indicators = ['st', 'nd', 'rd']

columns = ['Neighborhood']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))
        
ankara_venues_sorted = pd.DataFrame(columns=columns)
ankara_venues_sorted['Neighborhood'] = ankara_grouped['Neighborhood']

for ind in np.arange(ankara_grouped.shape[0]):
    ankara_venues_sorted.iloc[ind, 1:] = return_most_common_venues(ankara_grouped.iloc[ind, :], num_top_venues)

ankara_venues_sorted.head(27)
from sklearn.cluster import KMeans
ankara_merged = df_ankara
ankara_merged['Cluster Labels'] = kmeans.labels_

ankara_merged = ankara_merged.join(ankara_venues_sorted.set_index('Neighborhood'), on='District')
ankara_venues_sorted.head(10)
ks = 3

ankara_grouped_clustering = ankara_grouped.drop('Neighborhood', 1)

kmeans = KMeans(n_clusters=ks, random_state=0).fit(ankara_grouped_clustering)

kmeans.labels_[1:10]
map_clusters = folium.Map(location=[latitude, longitude], zoom_start=11)

# set color scheme for the clusters
x = np.arange(ks)
ys = [i+x+(i*x)**2 for i in range(ks)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(ankara_merged['Latitude'], ankara_merged['Longitude'], ankara_merged['District'], ankara_merged['Cluster Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters
ankara_merged.loc[ankara_merged['Cluster Labels'] == 0, ankara_merged.columns[[0] + list(range(4, ankara_merged.shape[1]))]]
ankara_merged.loc[ankara_merged['Cluster Labels'] == 1, ankara_merged.columns[[0] + list(range(4, ankara_merged.shape[1]))]]
ankara_merged.loc[ankara_merged['Cluster Labels'] == 2, ankara_merged.columns[[0] + list(range(4, ankara_merged.shape[1]))]]
ankara_merged.loc[ankara_merged['Cluster Labels'] == 3, ankara_merged.columns[[0] + list(range(5, ankara_merged.shape[1]))]]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13844160651489912, 'fraction_numerical': 0.015454377197853045, 'mean_word_length': 4.230880929332042, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '207016', 'license': 'Apache-2.0', 'n_tokens_mistral': 3872, 'n_tokens_neox': 3659, 'n_words': 734, 'path': 'Capstone-Project---The-Battle-of-Neighborhoods-Week-2--master/Battle of the Neighborhoods Week2-Copy1.ipynb', 'repo_name': 'kananiarpan/APPLIED-DATA-SCIENCE-PROJECT-CAPSTONE', 'size': '10849'}