<jupyter_start><jupyter_text>Data Collection and Storage  Importing libraries<jupyter_code># python os library
import json,os,re,datetime
import csv

# pandas library
import pandas as pd
from pandas import DataFrame
# path to un-processed data
path_to_folder = "raw_data"

# path to processed data
path_to_json = 'processed_data'

#looping in the folder "data" where the the json files are placed 
json_files = [pos_json for pos_json in os.listdir(path_to_folder) if pos_json.endswith('.json')] 

# printing list of json files 
print (json_files)<jupyter_output>['file2.json', 'file1.json', 'file5.json', 'file3.json', 'file4.json']<jupyter_text>`raw_data` folder contains json data fetched from Zomato API. It contains 10 files from different cities of California, USA.<jupyter_code>path=""


csvfile=open('zomato.csv', 'w',newline="")
writer=csv.writer(csvfile)  
writer.writerow(['restaurant_id','restaurant_name','country_code','city','address','locality','locality_verbose','longitude','latitude','cuisines','average_cost_for_two','has_table_booking','has_online_delivery','is_delivering_now','switch_to_orvder_menu','price_range','aggregate_rating','rating_color','rating_text','votes']) #writer to write the row


for js in json_files:
    with open(os.path.join(path_to_folder, js),'r', encoding='utf-8', errors="ignore") as json_file: #append the path of file to the source folde
        loaded_file = json.load(json_file,strict=False) #load the file to be selected
        for i in range(len(loaded_file)):
            if 'restaurants' in loaded_file[i].keys():
                for j in range(len(loaded_file[i]['restaurants'])):
                    if('restaurant' in loaded_file[i]['restaurants'][j].keys()):
                        
                        currency = loaded_file[i]['restaurants'][j]['restaurant']['currency']
                        city=loaded_file[i]['restaurants'][j]['restaurant']['location']['city']
                        locality=loaded_file[i]['restaurants'][j]['restaurant']['location']['locality']
                        rating_text=loaded_file[i]['restaurants'][j]['restaurant']['user_rating']['rating_text']
                        name_res=loaded_file[i]['restaurants'][j]['restaurant']['name'].replace('/'," ")
                        res_id=loaded_file[i]['restaurants'][j]['restaurant']['R']['res_id']
                        cuisines = loaded_file[i]['restaurants'][j]['restaurant']['cuisines']
                        has_table_booking = loaded_file[i]['restaurants'][j]['restaurant']['has_table_booking']
                        has_online_delivery = loaded_file[i]['restaurants'][j]['restaurant']['has_online_delivery']
                        city_id = loaded_file[i]['restaurants'][j]['restaurant']['location']['city_id']
                        address = loaded_file[i]['restaurants'][j]['restaurant']['location']['address']
                        locality_verbose = loaded_file[i]['restaurants'][j]['restaurant']['location']['locality_verbose']
                        longitude= loaded_file[i]['restaurants'][j]['restaurant']['location']['longitude']
                        latitude= loaded_file[i]['restaurants'][j]['restaurant']['location']['latitude']
                        is_delivering_now= loaded_file[i]['restaurants'][j]['restaurant']['is_delivering_now']                        
                        country_id=loaded_file[i]['restaurants'][j]['restaurant']['location']['country_id']
                        price_range=loaded_file[i]['restaurants'][j]['restaurant']['price_range']
                        switch_to_order_menu=loaded_file[i]['restaurants'][j]['restaurant']['switch_to_order_menu']
                        aggregate_rating=loaded_file[i]['restaurants'][j]['restaurant']['user_rating']['aggregate_rating']
                        rating_color=loaded_file[i]['restaurants'][j]['restaurant']['user_rating']['rating_color']
                        votes=loaded_file[i]['restaurants'][j]['restaurant']['user_rating']['votes']
                        avg_cost=loaded_file[i]['restaurants'][j]['restaurant']['average_cost_for_two']
                        
                        writer.writerow([res_id,name_res,country_id,city,address,locality,locality_verbose,longitude,
                                         latitude,cuisines,avg_cost,has_table_booking,has_online_delivery,
                                         is_delivering_now,switch_to_order_menu,price_range,aggregate_rating,
                                         rating_color,rating_text,votes]) 
                        
zomato=pd.read_csv('zomato.csv',sep=',', encoding='latin-1') #read csv
zomato.head()
zomato.info()
print("Total duplicated: " , zomato.duplicated().sum())
# Droping duplicates
zomato.drop_duplicates(inplace=True)
print(zomato.shape)

# After droping duplicates the number of data is 9519.
# # There are 9475 distinct restaurant
zomato['restaurant_id'].unique().shape<jupyter_output><empty_output><jupyter_text>Now we will get reviews for all the distinct restaurant<jupyter_code># Exporting restaurant id.
# To be used for making request to zomato API 
# based on restaurant id

path_rest_id = "processed_data"
out_file = open(os.path.join(path_rest_id,'restaurant_id.txt'), "w")
for line in zomato['Restaurant_ID'].unique():
    out_file.write(str(line))
    out_file.write("\n")
out_file.close()
# To read rest file
read_file = open(os.path.join(path_rest_id,"restaurant_id.txt"), "r")
rest_ID= []
for i in read_file:
    i = i.replace('\n', '')
    rest_ID.append(i)
# Once restarant_id is exported. It is used in zomatoWrapper package to fetch reviews for restaurnat.<jupyter_output><empty_output><jupyter_text>Get reviews for restaurant<jupyter_code># path to un-processed data
path_to_folder = "raw_data_reviews"

# path to processed data
path_to_json = 'processed_data'

#looping in the folder "data" where the the json files are placed 
json_files = [pos_json for pos_json in os.listdir(path_to_folder) if pos_json.endswith('.json')] 

# printing list of json files 
print (json_files) 
path=""

csvfile=open('review.csv', 'w',newline="")
writer=csv.writer(csvfile)  
writer.writerow(['restaurant_id','review']) 

review = []

for js in json_files:
    with open(os.path.join(path_to_folder, js),'r', encoding='utf-8', errors="ignore") as json_file: #append the path of file to the source folde
        loaded_file = json.load(json_file,strict=False) #load the file to be selected
        for i in loaded_file.items():
            review = i[0]
            restaurant_id = i[1]
            
            writer.writerow([restaurant_id,review])
data = pd.read_csv('review.csv')
data.head()
dd = data.groupby("restaurant_id")['review'].apply(list)
dd.shape
zomato_1= zomato.head(900)
df_merge_col = pd.merge(zomato_1,dd , on='restaurant_id')
df_merge_col.head()<jupyter_output><empty_output><jupyter_text>Saving merged dataframe to csv<jupyter_code>df_merge_col.to_csv(os.path.join(path_to_json,'zomato_final.csv'), encoding='utf-8', index=False)
df_merge_col.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1426148409893993, 'fraction_numerical': 0.003674911660777385, 'mean_word_length': 3.863230240549828, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '129668', 'license': 'MIT', 'n_tokens_mistral': 2245, 'n_tokens_neox': 2159, 'n_words': 371, 'path': 'research/001.data_storage_and_processing.ipynb', 'repo_name': 'ashish-gh/Exploratory_Data_Analysis_Zomato_Restaurant', 'size': '7075'}