<jupyter_start><jupyter_text>SetUP<jupyter_code># connect to google drive
import os
import numpy as np

# mount google drive
from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
# change root directory such that models are saved in google drive during training
root_dir = "/content/gdrive/My Drive/DeepFLY"
os.chdir(root_dir)
# print the contents
!ls<jupyter_output>apify_data	   poi_japan.json		 TA_scrape.ipynb
LP_Scrape.ipynb    poi_thailand.json		 Travel-Rec-Sys.ipynb
poi_database.json  scrape_data_with_apify.ipynb  Triposo_Scrape.ipynb<jupyter_text>Imports<jupyter_code>import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
import seaborn as sns
import os
from time import sleep
import sys
import re
import shutil
import json
from tqdm import tqdm_notebook
import operator
from collections import OrderedDict
import pickle
%matplotlib inline
import spacy
nlp = spacy.load('en_core_web_sm')<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>with open('apify_data/ghent.json','r') as f:
  ghent = json.load(f)<jupyter_output><empty_output><jupyter_text>Explore reviews<jupyter_code>def create_reviews_df(apify_json):
  all_reviews = {}
  count = 0

  for att in apify_json:
    for review in att['reviews']:
      one_review = {}
      one_review['user'] = review['user']['username']
      one_review['rating'] = review['rating']
      one_review['comment'] = review['text']
      one_review['title'] = review['title']
      one_review['location_id'] = att['location_id']
      one_review['location_name'] = att['name']
      all_reviews[count] = one_review
      count += 1
      
  ###
  reviews_df = pd.DataFrame(all_reviews).T 
  return reviews_df

#  A typical review
ghent[85]['reviews'][1]
# Make reviews dataframe
df_reviews = create_reviews_df(ghent)
df_reviews.sample(5)
## Group reviews by user
review_grps = df_reviews.groupby('user')
users_list = []
### Make a list of users with more than n reniews
n = 10
for name, grp in review_grps:
  if(len(grp)>n):
    users_list.append(name)
##
print('Number of users with more than {} reviews = {}'.format(n,len(users_list)))
users_list
## Show reviews of a user in the users list
review_grps.get_group('BradJill')<jupyter_output><empty_output><jupyter_text>Explore attractions<jupyter_code>### Attractions

def create_att_df(apify_json):
  ####
  def _extract_attr(cur_dict, key):
    val = None
    try:
      val = cur_dict[key]
    except:
      pass
    return val
  #########
  
  def _join_vals(x):
    
    try:
      L = []
      for i in x:
        L.append(i['name'])
      res = ','.join(L)
    except:
      return None
      
    return res
  
  #####
  def _join_keys(x):
    
    try:
      L = []
      for i in x:
        L.append(i['key'])
      res = ','.join(L)
    except:
      return None
    
    return res
    
    
  
  ###
  all_att = {}

  for i, att in enumerate(apify_json):
    
    one_att = {}
    one_att['location_id'] = _extract_attr(att,'location_id')
    one_att['name'] = _extract_attr(att,'name') 
    one_att['latitude'] = _extract_attr(att,'latitude') 
    one_att['longitude'] =  _extract_attr(att,'longitude') 
    one_att['num_reviews'] = _extract_attr(att,'num_reviews')
    one_att['location_string'] = _extract_attr(att,'location_string')
    one_att['raw_ranking'] = _extract_attr(att,'raw_ranking')
    one_att['ranking_geo'] = _extract_attr(att,'ranking_geo')
    #one_att['ranking_geo_id'] = _extract_attr(att,'ranking_geo_id')
    one_att['ranking_position'] =_extract_attr(att,'ranking_position')
    one_att['ranking'] = _extract_attr(att,'ranking')
    one_att['rating'] = _extract_attr(att,'rating')
    one_att['description'] = _extract_attr(att,'description')
    one_att['web_url'] = _extract_attr(att,'web_url')
    one_att['subcategory'] = _extract_attr(att,'subcategory')
    one_att['subtype'] = _extract_attr(att,'subtype')

    all_att[i] = one_att
  all_df = pd.DataFrame(all_att).T
  
  ## Post process
  

  all_df['category_names'] = all_df['subcategory'].apply(lambda x:  _join_vals(x))
  all_df['category_ids'] = all_df['subcategory'].apply(lambda x: _join_keys(x))

  all_df['subtype_names'] = all_df['subtype'].apply(lambda x:  _join_vals(x))
  all_df['subtype_ids'] = all_df['subtype'].apply(lambda x: _join_keys(x))

  all_df = all_df.drop('subcategory', 1)
  all_df = all_df.drop('subtype', 1)


  return all_df
### Create attractions dataframe for ghent
df_att = create_att_df(ghent)
df_att.sample(5)<jupyter_output><empty_output><jupyter_text>Trip Advisor > Scraped by Apify  Combine reviews and attractions<jupyter_code>## read in json file names in apify_data directory
ta_files = []
for root, dirs, files in os.walk("apify_data", topdown=False):
   for name in files:
      ta_files.append(os.path.join(root, name))
len(ta_files)
### Create attraction dataframe
frames = []
for i, fname in enumerate(ta_files):
  ##
  if i%25 == 0:
    print('Now extracting from {}'.format(fname))
  
  with open(fname,'r') as f:
    data = json.load(f)
  cur_df = create_att_df(data)
  frames.append(cur_df)
###
df_att = pd.concat(frames, ignore_index=True)
  
df_att.sample(5)
len(df_att)
%%time
### Create reviews dataframe
frames = []
tq = tqdm_notebook(ta_files)


for i, fname in enumerate(tq):
  ##

  with open(fname,'r') as f:
    data = json.load(f)
  cur_df = create_reviews_df(data)
  frames.append(cur_df)
###
df_reviews = pd.concat(frames, ignore_index=True)
  
df_reviews.sample(5)
len(df_reviews)
df_reviews.to_pickle('all_reviews.pkl')
df_att.to_pickle('all_attractions.pkl')
###### <<< ---- >>> ####<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13478796410346647, 'fraction_numerical': 0.002991377793418969, 'mean_word_length': 4.561643835616438, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '183155', 'license': 'Unlicense', 'n_tokens_mistral': 2130, 'n_tokens_neox': 2084, 'n_words': 412, 'path': 'scraping/scrape_data_with_apify.ipynb', 'repo_name': 'rbiswasfc/DeepFly', 'size': '5683'}