<jupyter_start><jupyter_text>Explorative data analysis of cleaned data frame This notebook is meant for EDA of the dataframe we ended up with after cleaning the data from irrelevant attributes, missing values, duplicates and outliers<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
df = pd.read_csv('..\Yelp Dataset\merged_data.csv')
df.head()
df.info()
# Setting float format options to show floats with three desimals
pd.set_option('display.float_format', lambda x: '%.3f' % x)

df.describe(include='all')<jupyter_output><empty_output><jupyter_text>We want to only include states with over 1000 reviews and that are located in US or Canada. Therefore we subset only those states.<jupyter_code># Value counts into dataframe
state_values = pd.DataFrame(df.state.value_counts())

# Filtering states with over 1000 reviews
state_values = state_values[state_values['state'] > 1000]

# Dropping BW (Germany) and EDH (Edinburgh)
state_values.drop(['BW', 'EDH'], inplace=True)

# Getting indexes
state_idx = state_values.index

# Filtering df with state_idx
df = df[df['state'].isin(state_idx)]
print(df.state.value_counts())
df.shape
# Setting up seaborn for plotting
sns.set(palette="deep", font_scale=1.1)<jupyter_output><empty_output><jupyter_text>Categorical variables<jupyter_code># Reviews in each state
plt.figure(figsize=(14,8))
chart = sns.countplot(data=df, x='state', order = df['state'].value_counts().index)
chart = chart.set_xticklabels(labels=df['state'].value_counts().index)
# Reviews in each city
df.city.value_counts()
# Review counts of different cuisiness

plt.figure(figsize=(14,8))
chart = sns.countplot(data=df, x='category', order = df['category'].value_counts().index)
chart.set_xticklabels(labels=df['category'].value_counts().index, rotation=90)

df.category.value_counts()
df.shape
df.to_csv('..\Yelp Dataset\clean.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12096360840594567, 'fraction_numerical': 0.011788826242952332, 'mean_word_length': 6.839357429718875, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '297782', 'license': 'MIT', 'n_tokens_mistral': 675, 'n_tokens_neox': 635, 'n_words': 171, 'path': 'notebooks/data-exploration.ipynb', 'repo_name': 'jphietala/Yelp-InfoViz', 'size': '1951'}