<jupyter_start><jupyter_text>Exploratory Data Analysis - TripAdvisor data<jupyter_code>import pandas as pd
import seaborn as sns
import numpy as np
import os
import matplotlib.pyplot as plt
import requests
from logzero import logger
from wordcloud import WordCloud
from PIL import Image

%matplotlib inline
pd.set_option('display.max_columns', None)
# PARAMS
ALREADY_SCRAPED_ROOFTOPS = [10216, 13639, 12755, 14787, 14657, 10068, 13460, 13836, 12125, 9796, 11264]
DATA_STUDIO = True
BOKAN_LAT = 51.501244
BOKAN_LON = -0.023363
logger.info(f"In merger.py, pwd returns {os.getcwd()}")
path_list = os.getcwd().split(os.sep)
target_index = path_list.index('nlp_consulting_project')
path_list = path_list[:target_index + 1]
if os.name=='nt':
    path_list[0] += '/'
os.chdir(os.path.join(os.sep, *path_list))

logger.info(f"In merger.py, changing working directory to {os.getcwd()}")<jupyter_output>[I 210228 19:01:24 <ipython-input-3-237813fee1cb>:1] In merger.py, pwd returns D:\Users\Louis\Docs\Scolaire\Polytechnique\Capgemini NLP\nlp_consulting_project\cleaner\notebooks
[I 210228 19:01:24 <ipython-input-3-237813fee1cb>:9] In merger.py, changing working directory to D:\Users\Louis\Docs\Scolaire\Polytechnique\Capgemini NLP\nlp_consulting_project<jupyter_text>To-do list- Cleaning des rating : OK- Ajout option data studio : OK- Gérer les paths de chargement- Ecrire les distances sur disque- Ecrire une fonction de processing et de jointure avec les rooftops- Réparer la fonction score- Ajouter une ligne d'identification du Bokan dans les distances  Preprocessing<jupyter_code>def flag_rooftops(df, already_scraped):

    df['has_rooftop'] = False

    rooftops_list = [idx for idx in df.index if idx < 0] + list(already_scraped)
    current_rooftops = set(df.index).intersection(set(rooftops_list))
    if len(current_rooftops)!=0:
        df.loc[current_rooftops, 'has_rooftop'] = True
    return df

def freq_dict_generator(keywords, keywords_dict):
    """Creation of a dict containing all of the keywords. """
    if len(keywords)!=0:
        for keyword in keywords:
            if keyword not in keywords_dict:
                keywords_dict[keyword] = 1
            else:
                keywords_dict[keyword] += 1

def col_evaluation(ls, col_name):
    """ Function used to check if a given cuisine is present in the list of the 'cuisine' column. It is
    used in a parallelised manner."""
    if col_name in ls:
        val = 1
    else:
        val = 0
    return val
            
def generate_cuisine_cols(df, cuisine_dict):
    """ Creation of the columns containing all of the cuisine types in a dataframe. """
    for col_name in cuisine_dict:
        df[col_name] = df['cuisine'].apply(lambda x: col_evaluation(x, col_name))
    return df

def keywords_list_parser(ls):
    """Parses a given list of 3 keywords to determine the main cuisine (ex: Indian, British), the food specificity 
    (Vegan, serving seafood...) and whether the restaurant has unique features (ex: if it is a bar, a pub...). Each
     keyword is read in order to first find the specificity and the feature. For the cuisine, the algorithm first
     tries to find a 'tier one cuisine' (i.e a very specific type of cuisine) before moving on to a 'tier two cuisine' 
     (i.e a more global type of cuisine such as 'European'). For each category, a hardcoded list of entries is read. """
    
    place_type = None
    food_spec = None
    cuisine_t1 = None
    cuisine_t2 = None

    found_place = False
    found_spec = False
    found_cuisine_t1 = False
    found_cuisine_t2 = False

    place_type_ls = ['Bar', 'Pub', 'Cafe', 'Street food', 'Steakhouse']

    food_specifity = ['Vegan', 'Gluten Free Options', 'Healthy', 'Fast food', 'Barbecue', 'Seafood',
                      'Fusion', 'Contemporary', 'Halal']
    
    cuisine_tier_one = ['British', 'French', 'Italian', 'Indian', 'Spanish', 'Turkish',
                        'South American', 'Other European', 'Lebanese', 'Moroccan', 'American',
                        'Thai', 'African'] 

    cuisine_tier_two = ['Asian', 'European', 'Other European', 'Middle Eastern', 'International']
    
    for word in ls:
        
        if (word in place_type_ls) and (found_place == False):
            place_type = word
            found_place = True
        if (word in food_specifity) and (found_spec == False):
            food_spec = word
            found_spec = True
        if (word in cuisine_tier_one) and (found_cuisine_t1 == False):
            cuisine_t1 = word
            found_cuisine_t1 = True
        if (word in cuisine_tier_two) and (found_cuisine_t2 == False):
            cuisine_t2 = word
            found_cuisine_t2 = True
        
    if cuisine_t1 == None:
        cuisine = cuisine_t2
    else:
        cuisine = cuisine_t1
            
    return [cuisine, food_spec, place_type]
    

def simplify_keywords(keyword_ls, keyword_dict):
    """Simplifies keywords thanks to a hardcoded dict given as input. It is used to assemble close cuisines and 
    features into a single entry to enhance the analysis (ex : Sri Lankan and Indian are grouped)."""
    new_keyword_ls = [keyword_dict.get(item,item) for item in keyword_ls]
    return new_keyword_ls
    
def score_builder(text):
    """A unified ranking score is built by considering this formula : 
    
    score = (total_entries - rank + 1)/total_entries
    
    Therefore, restaurants ranking can be compared across different locations. """
    
    if text == 'Ranking not found':
        return None
    else:
        elements_list = text.split(' ')
        # Gets the first element of the list, removes the first '#' character and cleans it
        rank = int(elements_list[0][1:].replace(',', ''))
        # Gets the third element of the list and cleans it
        total_entries = int(elements_list[2].replace(',',''))
        score = (total_entries - rank + 1) / total_entries
        return round(score, 3)
    
def location_extractor(text):
    """Used to extract the neighborhood where the restaurant is located."""
    if text != 'Ranking not found':
        return text.split()[-1]
    else:
        return None

def identify_coordinates(address, postcode=False):
    """This functions performs an API call with the scrapped address of the restaurant to the OpenStreetMap API.
    A longitude and a latitude are then returned."""
    
    if postcode:
        postcode = get_postcode(address)
        if postcode == None:
            return (None, None, False)
        target_url = 'https://nominatim.openstreetmap.org/search?q=' + postcode + '&format=json'
    else:
        target_url = 'https://nominatim.openstreetmap.org/search?q=' + address + '&format=json'
        
    target_url = target_url.replace(',', '%2C')
    target_url = target_url.replace(' ', '+')

    r = requests.get(target_url)
    found_coordinates = True
    if r.status_code == 200:
        try:
            coordinates = r.json()[0]
            latitude, longitude = coordinates['lat'], coordinates['lon']
            
        except:
            #print(f'Address not found for: {target_url}')
            latitude, longitude = None, None
            found_coordinates = False
            
    else:
        print(f'Error {status_code} ocurred on the request')
        latitude, longitude, found_coordinates = None, None, False
    return latitude, longitude, found_coordinates 
    
def get_postcode(address):
    f = lambda x, y: x+' '+y
    splits = address.split(' ')
    splits_lagged = splits[1:] + ['']
    total_splits = set(map(f, splits, splits_lagged))
    postcode_ls = [split for split in total_splits if (split.upper() == split) and (len(split)>=7 and len(split)<=9)]
    if postcode_ls == []:
        postcode = None
    else:
        postcode = postcode_ls[0]
    return postcode

def haversine_dist_to_bokan(lat, lon, bokan_lat = BOKAN_LAT, bokan_lon=BOKAN_LON):
    """Computes the Haversine distance of a given restaurant to the Bokan, given a longitude and a latitude."""

    deg_to_rad = lambda x: x * np.pi/180
    
    R = 6371 # Radius of the earth in km
    deg_lat = deg_to_rad(lat - bokan_lat)
    deg_lon = deg_to_rad(lon - bokan_lon) 
    a = np.sin(deg_lat/2) * np.sin(deg_lat/2) + np.cos(deg_to_rad(lat)) * np.cos(deg_to_rad(bokan_lat)) * np.sin(deg_lon/2) * np.sin(deg_lon/2)
    c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
    d = R * c ## Distance in km
    return np.round(d, 3)

def process_distances(df, print_output=25, bokan_lat = BOKAN_LAT, bokan_lon=BOKAN_LON):
    """Determines all of the distances between the restaurants and the bokan. It also returns the number of 
    restaurants which address couldn't be used to find the latitude and the longitude."""
   # NOTE : vérifier index & address list 
    df['distance_to_bokan'] = None
    distances_file_path = os.path.join(os.getcwd(), 'cleaner', 'notebooks', 'distances_to_bokan.json')
    if os.path.exists(distances_file_path):
        df_already_processed = pd.read_json(distances_file_path, orient='index')
        unprocessed_index = set(df.index) - set(df_already_processed.index)
        processed_index = set(df.index).intersection(df_already_processed.index)
        df.loc[processed_index, 'distance_to_bokan'] = df_already_processed['distance_to_bokan']
        target_addresses = df.loc[unprocessed_index, 'address']
        address_list, loc_list = target_addresses.tolist(), target_addresses.tolist()
    else:
        df_already_processed = pd.DataFrame(columns=['distance_to_bokan'])
        address_list, loc_list = df['address'].tolist(), df['address'].tolist()
        unprocessed_index = df.index
    print(f'{len(address_list)} new addresses have been detected.')
    if len(address_list) != 0:
        not_found_counter = 0
        for counter, address in enumerate(address_list):
            if address == 'Floor 37-39, 40 Marsh Wall, London E14 9TP England':
                lat, lon = BOKAN_LAT, BOKAN_LON
            else:

                lat, lon, found_coordinates = identify_coordinates(address)
                if found_coordinates == False:
                    lat, lon, found_coordinates = identify_coordinates(address, postcode=True)
                    if found_coordinates == False:  
                        not_found_counter += 1
                if counter%print_output == 0:
                    print(f'{counter}/{len(address_list)} restaurant coordinates have been processed.')
            loc_list[counter] = [lat, lon]
        print(f'Finished determining coordinates. {not_found_counter}/{len(address_list)} coordinates have not been found.')
            
        np_coords = np.asarray(loc_list, dtype=np.float32)
        np_lat, np_lon = np_coords[:,0], np_coords[:,1]
        distances = haversine_dist_to_bokan(np_lat, np_lon)
        df.loc[unprocessed_index, 'distance_to_bokan'] = distances
        
        df_to_append = df.loc[unprocessed_index, 'distance_to_bokan']
        df_already_processed = df_already_processed.append(df_to_append)
        df_already_processed.to_json(distances_file_path, orient='index')

    return df
 

def handle_duplicate_addresses(df):
    while (df[['address']].duplicated()).any():
        df.loc[df[['address']].duplicated(), 'address'] = df.loc[df[['address']].duplicated(), 'address'] + ','
    return df

def clean_ratings(df):
    df.loc[df['rating']==-1.0, 'rating'] = None
    df = df[df['rating'].notna()]
    return df
    

def cleaning_pipeline(df, keywords_renaming_dict, parse_distances=False, data_studio=True, scraped_rooftops = ALREADY_SCRAPED_ROOFTOPS):
    """Performs all of the cleaning operations. The process_distances parameter is set on False by default as the 
    number of API calls can run for a substantial amount of time."""
    cuisine_dict = {}
    df_restaurants = df.copy()
    df_restaurants = df_restaurants.rename(columns={'cuisine':'keywords'})                                                                                   
    # Creates a frequency dictionary containing the occurence of all types of cuisine
    df_restaurants['keywords'].apply(lambda x: freq_dict_generator(x, cuisine_dict))
    
    # Converting reviews as int
    df_restaurants['nb_reviews'] = df_restaurants['nb_reviews'].str.replace(',','').astype('int64')
    
    # Creates a ranking score in order to compare restaurants competing among a different amount of restaurants
    # in a neighborhood. The score of the restaurant ranked i among n restaurants is as such : 
    # Score = (n - i + 1) / n
    df_restaurants['score'] = df_restaurants['ranking'].apply(lambda x: score_builder(x))
    
    # Extracts the neighborhood where the restaurant is located
    df_restaurants['neighborhood'] = df_restaurants['ranking'].apply(lambda x: location_extractor(x))
    
    # Simplifies and groups the original keywords to enhance the analysis
    df_restaurants['keywords'] = df_restaurants['keywords'].apply(lambda x: simplify_keywords(x, 
                                                                                            keywords_renaming_dict))
    
    # Separates the keywords into useful categories used to performed detailled analysis
    df_restaurants['cuisine'] = None
    df_restaurants['food_specificity'] = None
    df_restaurants['place_type'] = None
    key_res = df_restaurants['keywords'].apply(keywords_list_parser)
    df_restaurants[['cuisine','food_specificity','place_type']] = pd.DataFrame(key_res.tolist(), index=df_restaurants.index)
    
    # Clean the ratings in order to remove '-1' values
    df_restaurants = clean_ratings(df_restaurants)
    
    # Computes the distance in kilometers between the Bokan and the scrapped restaurant
#     if parse_distances:
    df_restaurants = process_distances(df_restaurants)
    
    # Detects duplicate addresses and adds a '.' string to each in order to output a data studio-compatible file
    if data_studio:
        df_restaurants = handle_duplicate_addresses(df_restaurants)
    

    if len(scraped_rooftops)!=0:
        df_restaurants = flag_rooftops(df_restaurants, scraped_rooftops)


    # Used to generate the columns indicating the cuisine of the restaurant - DEPRECATED
    #df_restaurants = generate_cuisine_cols(df_restaurants, cuisine_dict)
    
    return df_restaurants, cuisine_dict

def cuisine_stats(df, target_col='cuisine', stats='mean', 
                  cols=['cuisine', 'nb_reviews', 'min_price', 'max_price', 'rating', 'score', 'neighborhood']):
    """ Generate statistics on restaurants, using a column such as the type of cuisine or the special features. """
    df_res = df[cols].groupby(by=target_col)
    temp = df_res.size()
    if stats == 'mean':
        df_res = df_res.mean()
        rename_dict = {col:'mean_' + col for col in df_res.columns}
    elif stats == 'median':
        df_res = df_res.median()
        rename_dict = {col:'median_' + col for col in df_res.columns}
    elif stats == 'std':
        df_res = df_res.std()
        rename_dict = {col:'std_' + col for col in df_res.columns}
    df_res = df_res.rename(columns=rename_dict)
    df_res['nb_restaus'] = temp.tolist()
    cols = df_res.columns.tolist()
    cols = cols[-1:] + cols[:-1]
    df_res = df_res[cols]
#     f = lambda x: cuisine_name if x==1 else 'Other cuisines'
#     df_res.index = list(map(f, df_res.index))
    return df_res
root_path = os.path.join(os.getcwd(), 'scraper', 'scraped_data', 'merged_data')

restaurants_path = os.path.join(root_path, 'merged_restaurants.json')
bulk_restaurants_path = os.path.join(root_path, 'merged_bulk_restaurants.json')
review_path = os.path.join(root_path, 'merged_reviews.json')
users_path = os.path.join(root_path, 'merged_users.json')

df_restaurants = pd.read_json(restaurants_path, lines=True)
restaurants = list(df_restaurants['restaurant_id'])
df_restaurants = df_restaurants.set_index('restaurant_id')

df_bulk_restaurants = pd.read_json(bulk_restaurants_path, lines=True)
df_bulk_restaurants = df_bulk_restaurants.set_index('restaurant_id')

# df_reviews = pd.read_json(review_path, lines=True)
# df_users = pd.read_json(users_path, lines=True)
df_restaurants[df_restaurants.index.duplicated()]
print(sum(df_restaurants.index.duplicated()))
#df_bulk_restaurants[df_bulk_restaurants.index.duplicated()]
keywords_renaming_dict = {
    'Southern-Italian':'Italian', 'Sicilian':'Italian', 'Tuscan':'Italian', 'Neapolitan':'Italian',
    'Central-Italian':'Italian', 'Pizza':'Italian', 'Polish':'European', 'Belgian':'European',
    'German':'European', 'Eastern European':'European', 'Portuguese':'European',
    'Greek':'European', 'Hong Kong':'Chinese','Cantonese':'Chinese', 'Sushi':'Japanese', 
    'Malaysian':'Asian', 'Tibetan':'Asian', 'Vietnamese':'Asian', 'Latin':'South American', 
    'Argentinian':'South American', 'Mexican':'South American', 'Brazilian':'South American',
    'Colombian':'South American', 'Vegetarian Friendly':'Vegan', 'Vegan Options':'Vegan', 'Persian':'Middle Eastern',
    'Afghani':'Middle Eastern', 'Deli':'Indian', 'Sri Lankan':'Indian', 'Pakistani':'Indian', 'Balti':'Indian',
    'Nepalese':'Indian', 'Jamaican':'African', 'Carribean':'African', 'Cajun & Creole':'African',
    'Moroccan':'African', 'Brew Pub':'Pub', 'Gastropub':'Pub', 'Grill':'Barbecue'}

#df_restaurants, cuisine_dict = cleaning_pipeline(df_restaurants, keywords_renaming_dict, parse_distances=False, data_studio=DATA_STUDIO, scraped_rooftops=[])
df_restaurants_bulk, cuisine_dict = cleaning_pipeline(df_bulk_restaurants, keywords_renaming_dict, parse_distances=False, data_studio=DATA_STUDIO)
print(cuisine_dict)<jupyter_output>{'Grill': 302, 'Pakistani': 86, 'Afghani': 31, 'Indian': 1567, 'Asian': 2124, 'Vegetarian Friendly': 3165, 'Steakhouse': 212, 'Argentinian': 35, 'Latin': 64, 'Bar': 1698, 'Pizza': 1066, 'British': 3042, 'Italian': 1476, 'Deli': 88, 'Lebanese': 259, 'Cafe': 1384, 'Mediterranean': 1451, 'European': 1611, 'Balti': 295, 'Spanish': 155, 'Vegan Options': 573, 'International': 368, 'Contemporary': 159, 'Japanese': 422, 'Middle Eastern': 552, 'Sushi': 315, 'Thai': 357, 'Healthy': 212, 'Nepalese': 60, 'French': 311, 'Gluten Free Options': 206, 'Fast food': 926, 'Chinese': 638, 'Cantonese': 140, 'African': 66, 'Moroccan': 46, 'Turkish': 469, 'Persian': 97, 'Seafood': 407, 'Barbecue': 190, 'Pub': 1149, 'Caribbean': 123, 'Mexican': 106, 'Brazilian': 53, 'Bakeries': 4, 'Fusion': 80, 'Brew Pub': 47, 'Sri Lankan': 56, 'Greek': 111, 'Polish': 41, 'Eastern European': 58, 'Belgian': 12, 'German': 20, 'Sicilian': 15, 'Southern-Italian': 13, 'Tibetan': 3, 'American': 419, 'Gastropub': 51, 'Street Food': 5[...]<jupyter_text>Visualisation of a restaurant's reviews<jupyter_code>restaurant_id = "0"
while (restaurant_id not in restaurants):
    try:
        restaurant_id = int(input("Enter restaurant id from 1 to 143 : "))
    except:
        print("Please enter int")<jupyter_output>Enter restaurant id from 1 to 143 : 5<jupyter_text>TF-IDF matrix<jupyter_code>target_path = os.path.join(os.getcwd(), 'cleaner', 'cleaned_data', 'restaurant_tfidf_sparse.npz')
tf_idf_file = np.load(target_path)

indices = tf_idf_file['indices']
data = tf_idf_file['data']

indices = indices.reshape(indices.shape[0], 1)
data = data.reshape(data.shape[0], 1)

print(tf_idf_file.files)

tf_idf_array = np.concatenate((indices, data), axis=1)
target_csv = str(restaurant_id) + '_word_freq.csv'
target_csv_path = os.path.join('..', 'cleaned_data', 'restaurants_tfidf', target_csv)

df_tfidf = pd.read_csv(target_csv_path, index_col=0)
df_tfidf.head()
df_tfidf_mean = df_tfidf.mean().sort_values(ascending=False).to_frame(name='tfidf mean')
df_tfidf_mean.head()
df_tfidf_mean[:15].plot(kind='bar')
plt.show()<jupyter_output><empty_output><jupyter_text>Wordcloud visualisation<jupyter_code>target_img = str(restaurant_id) + '_word_cloud.png'
target_img_path = os.path.join('..', 'cleaned_data', 'restaurant_wordclouds', target_img)
img = plt.imread(target_img_path)

plt.figure(figsize=(12,16))
plt.imshow(img)
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Other wordclouds and TF-IDF can be found in the dedicated folder  Global restaurant analysis<jupyter_code># def filtering_dataframe(df, column, terms):
    
#     df = df[df[''].str.findall()]
#     return df

# def generate_wordcloud(df, idx, directory, mask):
#     #filename = directory + str(idx) + "_word_cloud.png"

#     df_mean = df.mean().sort_values(ascending=False).to_frame(name='tfidf mean')
#     dict_words_tfidf = df_mean[df_mean['tfidf mean'] != 0].to_dict()['tfidf mean']

#     wordcloud = WordCloud(height=600, width=800, background_color="white",
#         colormap='Blues', max_words=100, mask=mask,
#         contour_width=0.5, contour_color='lightsteelblue')
#     wordcloud.generate_from_frequencies(frequencies=dict_words_tfidf)
#     wordcloud.to_file(filename)  
df_restaurants_cuisine.info()
df_stats = cuisine_stats(df_restaurants_cuisine, stats='mean')
df_stats.head(20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12156302120562307, 'fraction_numerical': 0.018680009530617107, 'mean_word_length': 3.7869525547445257, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '625797', 'license': 'MIT', 'n_tokens_mistral': 7462, 'n_tokens_neox': 6824, 'n_words': 1694, 'path': 'cleaner/notebooks/EDA.ipynb', 'repo_name': 'elalamik/NLP_Capgemini_Data_Camp', 'size': '20985'}