<jupyter_start><jupyter_text>Chapter7 - Getting Location Recommender Systems  InstallationsThe geospatial libraries are not pre installed in google colab as standard python library, therefore we need to install some libraries to use. Luckily this is an easy process. You can use either apt install or pip install. You can also create anaconda environment, but that is a bit complex so pip and apt are enough in our case to get the libraries we need. These are the libraries we need to install in this tutorial:GdalGeopandasFoliumThe installation might take 1 minute.<jupyter_code>%%time 
#!apt update
#!apt upgrade

# GDAL
!apt install gdal-bin python-gdal python3-gdal 
# Install rtree - Geopandas requirment
!apt install python3-rtree 
# Install Geopandas
!pip install git+git://github.com/geopandas/geopandas.git
# Install descartes - Geopandas requirment
!pip install descartes 
# Install Folium for Geographic data visualization
!pip install folium 
# Install Scikit-learn Recommender library Surprise
!pip install scikit-surprise
import pandas as pd
import numpy as np
import random

import geopandas as gpd


import matplotlib.pyplot as plt
import seaborn as sns
import folium

from surprise import Reader
from surprise import Dataset
from surprise import dump

from surprise import BaselineOnly, SVD, SVDpp,  KNNWithMeans,KNNWithZScore, CoClustering, NMF

from surprise.model_selection import cross_validate
from surprise.model_selection import KFold
from surprise.model_selection import train_test_split
from surprise.accuracy import rmse
from surprise import accuracy

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis (EDA<jupyter_code># Get the data and unzip it
!wget https://www.dropbox.com/s/lust8m4h5qycjsd/RCdata.zip
!unzip RCdata.zip<jupyter_output>--2019-05-15 10:37:35--  https://www.dropbox.com/s/lust8m4h5qycjsd/RCdata.zip
Resolving www.dropbox.com (www.dropbox.com)... 172.16.58.3, fdf8:f53e:61e4::18
Connecting to www.dropbox.com (www.dropbox.com)|172.16.58.3|:443... connected.
HTTP request sent, awaiting response... 301 Moved Permanently
Location: /s/raw/lust8m4h5qycjsd/RCdata.zip [following]
--2019-05-15 10:37:35--  https://www.dropbox.com/s/raw/lust8m4h5qycjsd/RCdata.zip
Reusing existing connection to www.dropbox.com:443.
HTTP request sent, awaiting response... 302 Found
Location: https://uc276a8dd987985873354260bcd8.dl.dropboxusercontent.com/cd/0/inline/Ag4LDbb5aMhbj6revpdbGg72PNzJ5iGjWiQP2uxN9G68OU4miaydVKTbjxwOKEGQm_4HevNyIdfTnbxz86qPVb_cEekes7e9Fo8hqC89CV-42A/file# [following]
--2019-05-15 10:37:35--  https://uc276a8dd987985873354260bcd8.dl.dropboxusercontent.com/cd/0/inline/Ag4LDbb5aMhbj6revpdbGg72PNzJ5iGjWiQP2uxN9G68OU4miaydVKTbjxwOKEGQm_4HevNyIdfTnbxz86qPVb_cEekes7e9Fo8hqC89CV-42A/file
Resolving uc276a8dd987985[...]<jupyter_text>Rating<jupyter_code>ratings = pd.read_csv('RCdata/rating_final.csv')
ratings.head()
ratings.shape
print(ratings['rating'].mean())
fig, ax = plt.subplots(figsize=(12,10))
sns.countplot(ratings['rating'], ax=ax, color='grey')
plt.title('Rating Countplot')
plt.savefig('ratingCountPlot.png')
plt.show()<jupyter_output>1.1998277347114557<jupyter_text>Unique userID and placeID in the dataset.<jupyter_code>print("There are {} unique userID in the dataset".format(ratings['userID'].nunique()))
print("There are {} unique placeID in the dataset".format(ratings['placeID'].nunique()))<jupyter_output>There are 138 unique userID in the dataset
There are 130 unique placeID in the dataset<jupyter_text>The following are the 10 users with the highest number of ratings. The highest two users have 18 ratings.<jupyter_code>fig, ax = plt.subplots(figsize=(12,10))
sns.countplot(x="userID",data=ratings, color="grey", order=ratings['userID'].value_counts().index[:10], ax=ax)
plt.show()<jupyter_output><empty_output><jupyter_text>The following are the 10 users with the highest number of ratings. The restaurant with the highest rating count has 36 ratings.<jupyter_code>fig, ax = plt.subplots(figsize=(18,12))
sns.countplot(x="placeID",data=ratings, color="grey", order=ratings['placeID'].value_counts().index[:10], ax=ax)
plt.show()<jupyter_output><empty_output><jupyter_text>Restaurants<jupyter_code># Read the data as GeoDataFrame
geoplaces = gpd.read_file('RCdata/geoplaces.geojson')
geoplaces.head(2).T

geoplaces.shape
# Display in an interactive map - Folium
m = folium.Map (
    location = [18.915421, -99.184871],
    
    zoom_start = 7
)
folium.GeoJson(geoplaces).add_to(m)

m
from folium.plugins import FastMarkerCluster

lons = geoplaces['longitude'] 
lats = geoplaces['latitude']
m = folium.Map(
    location = [np.mean(geoplaces.latitude), np.mean(geoplaces.longitude)],
    tiles= 'CartoDB dark_matter',
    zoom_start=6
    )
FastMarkerCluster(data=list(zip(lats, lons))).add_to(m)
folium.LayerControl().add_to(m)

m
print("Unique Cities in the dataset is {}".format(geoplaces.city.nunique()))

# Display cities in countplot
fig, ax = plt.subplots(figsize=(18,12))
sns.countplot(x="city",data=geoplaces, color="grey", order=geoplaces['city'].value_counts().index, ax=ax)
plt.savefig('citiesCountplot.png')
plt.show()<jupyter_output><empty_output><jupyter_text>As you can see there are some mistyped cities with the same name.We need to clean the data.<jupyter_code># Get the names of unique cities in the dataset
geoplaces['city'].unique()
# Group each city different names
cuer = ['Cuernavaca', 'cuernavaca', ]
slp = ['s.l.p.',  'San Luis Potosi', 'san luis potosi', 'slp', 'san luis potos', 'san luis potosi ', 's.l.p'] 
ciudad = ['victoria ', 'victoria', 'Cd Victoria', 'Ciudad Victoria', 'Cd. Victoria']
# Replace mistyped names with the right name
geoplaces['city']=geoplaces['city'].replace(slp,'San Luis Potosi' )
geoplaces['city']=geoplaces['city'].replace(ciudad,'Ciudad Victoria')
geoplaces['city']=geoplaces['city'].replace(cuer,'Cuernavaca')
# Display cleaned cities in countplot
fig, ax = plt.subplots(figsize=(18,12))
sns.countplot(x="city",data=geoplaces, color="grey", order=geoplaces['city'].value_counts().index, ax=ax)
plt.show()
# Subplots for four countplots
fig, ax = plt.subplots(2,2, figsize=(15,12))
sns.countplot(x="alcohol",data=geoplaces, color="grey", ax=ax[0][0])
ax[0][0].set_title('Alcohol Service')
sns.countplot(x="price",data=geoplaces, color="grey", ax=ax[0][1])
ax[0][1].set_title('Price Categories')
sns.countplot(x="accessibility",data=geoplaces, color="grey", ax=ax[1][0])
ax[1][0].set_title('Accessibility Categories')
sns.countplot(x="Rambience",data=geoplaces, color="grey", ax=ax[1][1])
ax[1][1].set_title('Rambience')
#fig.subplots_adjust(hspace=0.5)
plt.tight_layout()
plt.savefig('countplot_m.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Merge Ratings & Geoplaces<jupyter_code>df = pd.merge(ratings, geoplaces, on='placeID')
df.head()
df.shape
df.isnull().sum()<jupyter_output><empty_output><jupyter_text>Recommender Systems We will use Surprise, a Python Scikit library for building and analyzing recommender systesm<jupyter_code># Set rating scale of the dataset
reader = Reader(rating_scale=(0, 2))

# Load the dataframe with ratings.
data = Dataset.load_from_df(df[['userID', 'placeID', 'rating']], reader)
benchmark_scores = []


random.seed(114)
np.random.seed(114)

# Iterate selected algorithms
for algorithm in [BaselineOnly(), SVD(), SVDpp(),  KNNWithMeans(),KNNWithZScore(), CoClustering(), NMF()]:
    # Cross-validation
    cv = cross_validate(algorithm, data, cv=5, verbose=False)
    
    # create df with cv results
    df_cv = pd.DataFrame.from_dict(cv).mean(axis=0)
    df_cv = df_cv.append(pd.Series([str(algorithm).split(' ')[0].split('.')[-1]], index=['Algorithm']))
    benchmark_scores.append(df_cv)

# Create results DataFrame from the benckmark scores
results = pd.DataFrame(benchmark_scores).set_index('Algorithm').sort_values('test_rmse')
results<jupyter_output><empty_output><jupyter_text>Cross Validation K-Fold<jupyter_code># define a cross-validation iterator
kf = KFold(n_splits=5)

# Define algorithms (Top 2)
algo_knnwithMeans = KNNWithMeans()
algo_svdpp = SVDpp()<jupyter_output><empty_output><jupyter_text>KNNWithMeans<jupyter_code>random.seed(114)
np.random.seed(114)

for trainset, testset in kf.split(data):

    # train and test algorithm with KNNWithMeans.
    algo_knnwithMeans.fit(trainset)
    predictions_knnwithmeans = algo_knnwithMeans.test(testset)
    # Compute and print Root Mean Squared Error
    accuracy.rmse(predictions_knnwithmeans, verbose=True)
    
    dump.dump('./dump_KNNWithMeans', predictions_knnwithmeans, algo_knnwithMeans)

print("Average RMSE of the CV is: ", np.mean(accuracy.rmse(predictions_knnwithmeans, verbose=False)))<jupyter_output>Average RMSE of the CV is:  0.688647918600383<jupyter_text>Create DataFrame of KNNwithMeans Reuslts<jupyter_code># Load the dump file
predictions_knnwithmeans, algo_knnwithMeans = dump.load('./dump_KNNWithMeans')
df_knnithmeans = pd.DataFrame(predictions_knnwithmeans, columns=['uid', 'iid', 'rui', 'est', 'details'])  
df_knnithmeans.head()
df_knnithmeans.rui.unique()
df_knnithmeans.shape
# Calculate the error
df_knnithmeans['err'] = abs(df_knnithmeans.est - df_knnithmeans.rui)
df_knnithmeans.head()<jupyter_output><empty_output><jupyter_text>SVDpp<jupyter_code>my_seed = 114
random.seed(my_seed)
np.random.seed(my_seed)

for trainset, testset in kf.split(data):

    # train and test algorithm with SVDpp.
    algo_svdpp.fit(trainset)
    predictions_svdpp = algo_svdpp.test(testset)
    # Compute and print Root Mean Squared Error
    accuracy.rmse(predictions_svdpp, verbose=True)
    # Dump the prediction into dataframe
    dump.dump('./dump_SVDpp', predictions_svdpp, algo_svdpp)


print("Average RMSE of the CV is: ", np.mean(accuracy.rmse(predictions_svdpp, verbose=False)))<jupyter_output>Average RMSE of the CV is:  0.667695280346118<jupyter_text>Create DataFrame of SVDpp Reuslts<jupyter_code># Load the dump file
predictions_svdpp, algo_svdpp = dump.load('./dump_SVDpp')
df_svdpp = pd.DataFrame(predictions_svdpp, columns=['uid', 'iid', 'rui', 'est', 'details'])  
df_svdpp.head()

df_svdpp['err'] = abs(df_svdpp.est - df_svdpp.rui)
df_svdpp.head()<jupyter_output><empty_output><jupyter_text>Algorithm Comparison and Interpretations<jupyter_code># 10 Worst predictions for KNNwithMeans
df_knnithmeans.sort_values(by='err')[-10:]
# 10 Worst predictions for SVDpp
df_svdpp.sort_values(by='err')[-10:]
fig, ax = plt.subplots(1,2, figsize=(10,8))
sns.distplot(df_svdpp.err, ax=ax[0])
sns.distplot(df_knnithmeans.err, ax=ax[1])
ax[0].set_title('SVDpp')
ax[1].set_title('KNNwithMeans')
plt.savefig('comparison.png')
plt.show()<jupyter_output><empty_output><jupyter_text>* It seems both algorithms are right-skewed. and have higher predictions around zero. So what happens when a user has a smaller number of ratings. Remember we had a mean rating of 1.19. Let us say users with Rating less than 5<jupyter_code>def get_Iu(uid):
    """Return the number of items rated by given user
    
    Args:
        uid: The raw id of the user.
    Returns:
        The number of items rated by the user.
    """
    
    try:
        return len(trainset.ur[trainset.to_inner_uid(uid)])
    except ValueError:  # user was not part of the trainset
        return 0
    
df_knnithmeans['Iu'] = df_knnithmeans.uid.apply(get_Iu)
df_svdpp['Iu'] = df_svdpp.uid.apply(get_Iu)

df_knnithmeans[df_knnithmeans.Iu < 5].err.mean(), df_svdpp[df_svdpp.Iu < 5].err.mean()<jupyter_output><empty_output><jupyter_text>* It seems that KNNwithMeans is doing better in this case than svdpp  Location Based Recommenders<jupyter_code>kmeans = KMeans(n_clusters=24, init='k-means++')


X = df[['longitude','latitude']]
kmeans.fit(X)
y = kmeans.labels_
print("k = 24", " silhouette_score ", silhouette_score(X, y, metric='euclidean'))
df['cluster'] = kmeans.predict(df[['longitude','latitude']])
df[['userID','latitude','longitude','placeID','cluster']].sample(10)
topvenues_df = df.sort_values(by=['rating', 'food_rating',	'service_rating'], ascending=False)

def recommend_restaurants(df, longitude, latitude):
    # Predict the cluster for longitude and latitude provided
    cluster = kmeans.predict(np.array([longitude,latitude]).reshape(1,-1))[0]
   
    # Get the best restaurant in this cluster
    name = df[df['cluster']==cluster].iloc[0]['name']
   
    print('"{}" is recommended'.format(name))
   
def create_folium_map(df, user_coords, restaurant_name):
    m = folium.Map(
    location=user_coords,
    zoom_start=10,
    tiles='Stamen Terrain'
    )
    
    folium.Marker(
    location=user_coords,
    popup='User Location',
    icon=folium.Icon(icon='cloud')
    ).add_to(m)
    
    
    folium.Marker(
    location=list(df[df['name'] == restaurant_name][['latitude', 'longitude']].iloc[0]),
    popup='Restaurant Location',
    icon=folium.Icon(color='red', icon='info-sign')
    ).add_to(m)
    
    return m
    
recommend_restaurants(topvenues_df,-99.145185, 23.730925)

recommend_restaurants(topvenues_df, -100.939752, 22.150849)
user_coords = [22.120849, -100.839752]

restaurant_name = "<NAME>co"
create_folium_map(df, user_coords, restaurant_name)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12939309198095383, 'fraction_numerical': 0.03234827299523846, 'mean_word_length': 6.187398153177621, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 69, 'https://': 5, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '221539', 'license': 'MIT', 'n_tokens_mistral': 5191, 'n_tokens_neox': 4834, 'n_words': 1007, 'path': 'Chapter07/Chapter7.ipynb', 'repo_name': 'AdityaAnuragMishra/Geospatial-Data-Science-Quick-Start-Guide', 'size': '13245'}