<jupyter_start><jupyter_text>In order to use dask1. Activate Conda Python 3 `source activate python3`2. Install dask `conda install dask -y`2. Install dask and s3fs `conda install s3fs -c conda-forge -y`  Load Buisness Dataset<jupyter_code>data_key = 'yelp_academic_dataset_business.json'
data_location = 's3://{}/{}/{}'.format(bucket, folder, data_key)
attributes = dd.read_json(data_location, lines=True)
attributes.head()<jupyter_output><empty_output><jupyter_text>Load Reviews Dataset<jupyter_code>data_key = 'yelp_academic_dataset_review.json'
data_location = 's3://{}/{}/{}'.format(bucket, folder, data_key)
reviews = dd.read_json(data_location, blocksize=32e6)
reviews.head()<jupyter_output><empty_output><jupyter_text>Load Users Dataset<jupyter_code>data_key = 'yelp_academic_dataset_user.json'
data_location = 's3://{}/{}/{}'.format(bucket, folder, data_key)
users = dd.read_json(data_location, blocksize=32e6)
users.head()<jupyter_output><empty_output><jupyter_text>Now we do EDA<jupyter_code># Next steps (Dask merge the datasets)
# Then filter for resturant
# Then export as .csv with restaurants
# Then upload to S3 Bucket<jupyter_output><empty_output><jupyter_text>In order to use dask1. Activate Conda Python 3 `source activate python3`2. Install fastparquet `conda install -c conda-forge fastparquet'3. Restart notebook kernal ensuring you are on conda_python3 environment<jupyter_code>## Inital Load of Dask Files (missing columns run)
from fastparquet import ParquetFile
rru = dd.read_parquet('s3://tally-ai-dspt3/yelp-kaggle-raw-data/restaurants_reviews_users.parquet.gzip') 
             
rru.head()
rru.columns
##################################################################################################################################<jupyter_output><empty_output><jupyter_text>Initial EDA Info  In order to use dask1. Activate Conda Python 3 `source activate python3`2. Install fastparquet `conda install -c conda-forge fastparquet'3. Restart notebook kernal ensuring you are on conda_python3 environment<jupyter_code>import boto3
import dask.dataframe as dd
from sagemaker import get_execution_role
from fastparquet import ParquetFile
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

role = get_execution_role()
bucket='tally-ai-dspt3'
folder = 'yelp-kaggle-raw-data'
#Loading data
data = 'final_combined.parquet.gzip'
data_location = 's3://{}/{}/{}'.format(bucket, folder, data)
df = dd.read_parquet(data_location)
df.head()

#Convert dd to pd dataframe
df = df.compute()
#There aare 5055992 entries
df.shape
df.columns
print('Number of unique categories of food, ', df.categories.nunique())
print('Number of unique cities with restaurants, ', df.city.nunique())
print('Number of unique reviews, ', df.text.nunique())
print('Number of unique reviews of stars of experience, ', df.stars_review.nunique())



print('Number of unique states with restaurants, ', df.state.nunique())
print('Number of unique users with reviews, ', df.user_id.nunique())
print('Number of unique restaurants, ', df.business_id.nunique())
df['average_stars'].describe()
df.full_name.nunique()

asteroid.diameter = df.diameter.astype(float)

sns.distplot(df.diameter)

#categorical column breakdown
df.select_dtypes(exclude='number').describe().T

#numerical columns
numeric_cols = df.select_dtypes(include='number').columns
df.dtypes
df.isna().sum()
df['stars_review'].unique
df.describe()
#Check the value of stars_reviews to see if it's different from stars_review
df['stars_reviews'].unique
#It appears to be the same, so I dropped the stars_reviews one
df = df.drop(['stars_reviews'], axis=1)
df.columns<jupyter_output><empty_output><jupyter_text>Categories<jupyter_code>#Check all kinds of restaurants
df['categories'].value_counts()
import matplotlib.pyplot as plt
import seaborn as sns
#Created a graph for top categories

x=df['categories'].value_counts()
print(f'There are {len(x)} different types of restaurants in Yelp')

#prep for chart
x=x.sort_values(ascending=False)
x=x.iloc[0:20]

#chart
plt.figure(figsize=(16,4))
ax = sns.barplot(x.index, x.values, alpha=0.8)
plt.title("Popular Cuisine", fontsize=30)
locs, labels = plt.xticks()
plt.setp(labels, rotation=80)
plt.ylabel('#Restaurants', fontsize=14)
plt.xlabel('Cuisine', fontsize=14)

#adding the text labels
rets = ax.patches
labels = x.values
for rect, label in zip(rets, labels):
    height = rect.get_height()
    ax.text(rect.get_x() + rect.get_width()/2, height + 5, label, ha = 'center', va = 'bottom')

plt.show()
        
df['categories'].value_counts()<jupyter_output><empty_output><jupyter_text>City with the Most Reviews<jupyter_code>#Sort city from top to bottom by rating counts
x=df['city'].value_counts()
x=x.sort_values(ascending=False)
x=x.iloc[0:20]
plt.figure(figsize=(16,4))
ax = sns.barplot(x.index, x.values, alpha=0.8)
plt.title("City with most reviews")
locs, labels = plt.xticks()
plt.setp(labels, rotation=45)
plt.ylabel('#restaurants', fontsize=14)
plt.xlabel('City', fontsize=14)

#adding the text labels
rects = ax.patches
labels = x.values
for rect, label in zip(rects, labels):
    height = rect.get_height()
    ax.text(rect.get_x() + rect.get_width()/2, height + 5, label, ha='center', va='bottom')
plt.show()<jupyter_output><empty_output><jupyter_text>Previous Tally-ai EDA LAPS19  Review Info<jupyter_code>df.columns
print(df['business_id'].value_counts())

# there are 63944 restaurants 

# review text will be the focus of NLP
print(df['user_id'].value_counts())

# there are 1399916 users 

# review text will be the focus of NLP
df.columns
print(df['postal_code'].value_counts())
print(df['state'].value_counts())

# there are 10944 zip codes 

# These are the states
#
#NV     1688628
#AZ     1442738
#ON      668862
#NC      340882
#OH      273514
#PA      248253
#QC      161133
#WI      106379
#AB       71907
#IL       32327
#SC       19766
#TX        1098
#NY         128
#CA          95
#AR          60
#HI          51
#CO          45
#MB          36
#VA          27
#NE          18
#FL          10
#OR           9
#BC           6
#VT           5
#WY          5
#AL           4
#WA           3
#HPL          3
mean_reviews = df['average_stars'].mean()
mean_reviews
df_tmp = df['stars_review']\
    .value_counts().reset_index()\
    .sort_values('index')
mean_stars = df['stars_review'].mean()
print('average stars:', mean_stars)
plt.bar(df_tmp['index'], df_tmp['stars_review'])
plt.axvline(mean_stars, c='b');
# wild guess:
# 1 and 5 stars are more inluenced by emotions
# 2-4 stars are relatively less influenced?
df.columns
cols = ['useful_reviews', 'cool_reviews', 'funny_reviews']
def plot1(colname):
    df_tmp = df[colname]\
        .value_counts().reset_index()\
        .sort_values('index')
    plt.plot(df_tmp['index'], df_tmp[colname])    

def plot2(colname):
    df_tmp = df[colname]\
        .value_counts().reset_index()\
        .sort_values('index')
    df_tmp = df_tmp[df_tmp['index']<10]
    plt.plot(df_tmp['index'], df_tmp[colname])

plt.figure(figsize=(12,4))    
plt.subplot(121)
for col in cols:
    plot1(col)    
plt.subplot(122)   
plt.xlim(0,9)
for col in cols:
    plot2(col)
plt.legend(cols);  
# all 3 review feedbacks have a long tail
# "funny" and "cool" have very similar shapes<jupyter_output><empty_output><jupyter_text>Datetime<jupyter_code>df['datetime'] = pd.to_datetime(df['date'])
df['datetime'].describe()
# the timespan is about 14 years
df['year'] = df['datetime'].apply(lambda x: x.year)
df['month'] = df['datetime'].apply(lambda x: x.month)
df['day'] = df['datetime'].apply(lambda x: x.day)
df['hour'] = df['datetime'].apply(lambda x: x.hour)
df['weekday'] = df['datetime'].apply(lambda x: x.weekday())
df_tmp = df['year'].value_counts().reset_index()
plt.bar(df_tmp['index'], df_tmp['year']);
# it was growing till 2017
df_tmp = df['month'].value_counts().reset_index()
plt.bar(df_tmp['index'], df_tmp['month']);
# monthly number of reviews are not very different
df_tmp = df['hour'].value_counts().reset_index()
plt.bar(df_tmp['index'], df_tmp['hour']);
# around 10 am users left the lest number of reviews... lol
# also from the business.csv we could see that 
# all data are about the US region
df_tmp = df['weekday'].value_counts().reset_index()
plt.bar(df_tmp['index'], df_tmp['weekday']);
# review numbers are slightly different for weekdays
types = df['text'].apply(lambda x: type(x))
types.value_counts()
# this is interesting... 
# sthere are 2 floats among all the strings.
df.columns<jupyter_output><empty_output><jupyter_text>Text Length<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# df['text_length'] = df['text'].fillna('').apply(lambda x: len(x))
df['text'].describe()
df['text_length'] = df['text'].str.len()
df['text_length']
mean_length = df['text_length'].mean()
print(mean_length)
df['text_length'].describe()
df_tmp = df['text_length'].value_counts().reset_index()
plt.figure(figsize=(12,8))
plt.subplot(221)
plt.bar(df_tmp['index'], df_tmp['text_length'])
plt.subplot(222)
plt.bar(df_tmp['index'], np.log(df_tmp['text_length']))
plt.subplot(223)
df_tmp1 = df_tmp[df_tmp['index']<500]
plt.bar(df_tmp1['index'], df_tmp1['text_length'])
plt.subplot(224)
df_tmp = df_tmp[df_tmp['index']>4500]
plt.bar(df_tmp['index'], df_tmp['text_length']);
# this is a very interesting distribution
# there is a discontinuity around 80(?) characters.
# it might be caused by the old limit of review length
# we could exam the text length time series.<jupyter_output><empty_output><jupyter_text>Monthly Max Text Length<jupyter_code>from datetime import datetime

# 4 records have NA value in the 'date' column
years = df['year'].dropna().unique().astype(int)
months = np.arange(1, 13, 1)
lengths = []
for year in years:
    for month in months:
        condition = ((df['year']==year) &
            (df['month']==month))
        length = df[condition]['text_length'].max()
        lengths.append([year, month, length])
lengths = pd.DataFrame(lengths, columns=['year', 'month', 'maximum'])
lengths = lengths.dropna().sort_values(['year', 'month'])
lengths['date'] = lengths.apply(lambda x: 
    datetime.strptime(
        str(int(x['year']))+str(int(x['month'])).zfill(2)+'01', 
        '%Y%m%d'),
    axis=1)
lengths
# monthly maximum length of reviews
plt.figure(figsize=(18, 3))
plt.plot(lengths['date'], lengths['maximum']);<jupyter_output>/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/plotting/_converter.py:129: FutureWarning: Using an implicitly registered datetime converter for a matplotlib plotting method. The converter was registered by pandas on import. Future versions of pandas will require you to explicitly register matplotlib converters.

To register the converters:
	>>> from pandas.plotting import register_matplotlib_converters
	>>> register_matplotlib_converters()
  warnings.warn(msg, FutureWarning)<jupyter_text>Sample of Text > 4000 Chars<jupyter_code># sample a review longer than 4,000 characters
print(df[df['text_length']>4000]['text'].sample().values[0])
df['text_length'].max()

#misc
import gc
import time
import warnings


#viz
import matplotlib.pyplot as plt
import seaborn as sns 
import matplotlib.gridspec as gridspec 
import matplotlib.gridspec as gridspec 

# graph viz
import plotly.offline as pyo
from plotly.graph_objs import *
import plotly.graph_objs as go

#map section
import imageio
import folium
import folium.plugins as plugins
from mpl_toolkits.basemap import Basemap


#graph section
import networkx as nx
import heapq  # for getting top n number of things from list,dict<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14748170469220834, 'fraction_numerical': 0.024192854068015496, 'mean_word_length': 5.2687533729087965, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '421921', 'license': 'MIT', 'n_tokens_mistral': 4232, 'n_tokens_neox': 3978, 'n_words': 997, 'path': 'notebooks/EDA-NLP-BL/kaggle_join_datasets.ipynb', 'repo_name': 'BloomTech-Labs/tally-ai-ds', 'size': '11615'}